1//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the PPCISelLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCISelLowering.h"
14#include "MCTargetDesc/PPCMCTargetDesc.h"
15#include "MCTargetDesc/PPCPredicates.h"
16#include "PPC.h"
17#include "PPCCallingConv.h"
18#include "PPCFrameLowering.h"
19#include "PPCInstrInfo.h"
20#include "PPCMachineFunctionInfo.h"
21#include "PPCPerfectShuffle.h"
22#include "PPCRegisterInfo.h"
23#include "PPCSelectionDAGInfo.h"
24#include "PPCSubtarget.h"
25#include "PPCTargetMachine.h"
26#include "llvm/ADT/APFloat.h"
27#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/APSInt.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SmallPtrSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/Statistic.h"
35#include "llvm/ADT/StringRef.h"
36#include "llvm/CodeGen/CallingConvLower.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/LivePhysRegs.h"
39#include "llvm/CodeGen/MachineBasicBlock.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachineJumpTableInfo.h"
45#include "llvm/CodeGen/MachineLoopInfo.h"
46#include "llvm/CodeGen/MachineMemOperand.h"
47#include "llvm/CodeGen/MachineModuleInfo.h"
48#include "llvm/CodeGen/MachineOperand.h"
49#include "llvm/CodeGen/MachineRegisterInfo.h"
50#include "llvm/CodeGen/SelectionDAG.h"
51#include "llvm/CodeGen/SelectionDAGNodes.h"
52#include "llvm/CodeGen/TargetInstrInfo.h"
53#include "llvm/CodeGen/TargetLowering.h"
54#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/ValueTypes.h"
57#include "llvm/CodeGenTypes/MachineValueType.h"
58#include "llvm/IR/CallingConv.h"
59#include "llvm/IR/Constant.h"
60#include "llvm/IR/Constants.h"
61#include "llvm/IR/DataLayout.h"
62#include "llvm/IR/DebugLoc.h"
63#include "llvm/IR/DerivedTypes.h"
64#include "llvm/IR/Function.h"
65#include "llvm/IR/GlobalValue.h"
66#include "llvm/IR/IRBuilder.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/Intrinsics.h"
69#include "llvm/IR/IntrinsicsPowerPC.h"
70#include "llvm/IR/Module.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/Use.h"
73#include "llvm/IR/Value.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/MC/MCExpr.h"
76#include "llvm/MC/MCSectionXCOFF.h"
77#include "llvm/MC/MCSymbolXCOFF.h"
78#include "llvm/Support/AtomicOrdering.h"
79#include "llvm/Support/BranchProbability.h"
80#include "llvm/Support/Casting.h"
81#include "llvm/Support/CodeGen.h"
82#include "llvm/Support/CommandLine.h"
83#include "llvm/Support/Compiler.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/Format.h"
87#include "llvm/Support/KnownBits.h"
88#include "llvm/Support/MathExtras.h"
89#include "llvm/Support/raw_ostream.h"
90#include "llvm/Target/TargetMachine.h"
91#include "llvm/Target/TargetOptions.h"
92#include <algorithm>
93#include <cassert>
94#include <cstdint>
95#include <iterator>
96#include <list>
97#include <optional>
98#include <utility>
99#include <vector>
100
101using namespace llvm;
102
103#define DEBUG_TYPE "ppc-lowering"
104
105extern cl::opt<bool> EnablePPCGenScalarMASSEntries;
106
107static cl::opt<bool> EnableP10StoreForward(
108 "ppc-p10-store-forward",
109 cl::desc("enable P10 store forward-friendly conversion"), cl::init(Val: true),
110 cl::Hidden);
111
112static cl::opt<bool> EnablePPCPreinc(
113 "ppc-preinc", cl::desc("enable preincrement load/store generation on PPC"),
114 cl::init(Val: true), cl::Hidden);
115
116static cl::opt<bool>
117 EnableILPPref("ppc-ilp-pref",
118 cl::desc("Set the node scheduling preference to ILP on PPC"),
119 cl::init(Val: true), cl::Hidden);
120
121static cl::opt<bool> EnablePPCUnaligned(
122 "ppc-unaligned", cl::desc("enable unaligned load/store generation on PPC"),
123 cl::init(Val: true), cl::Hidden);
124
125static cl::opt<bool>
126 EnableSCO("ppc-sco", cl::desc("enable sibling call optimization on ppc"),
127 cl::init(Val: true), cl::Hidden);
128
129static cl::opt<bool> EnableInnermostLoopAlign32(
130 "ppc-innermost-loop-align32",
131 cl::desc("Align innermost loops to 32 bytes on ppc"), cl::init(Val: true),
132 cl::Hidden);
133
134static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables",
135cl::desc("use absolute jump tables on ppc"), cl::Hidden);
136
137static cl::opt<bool>
138 EnablePerfectShuffle("ppc-perfect-shuffle",
139 cl::desc("enable vector permute decomposition"),
140 cl::init(Val: false), cl::Hidden);
141
142cl::opt<bool> EnableAutoPairedVecSt(
143 "ppc-auto-paired-vec-st",
144 cl::desc("enable automatically generated 32byte paired vector stores"),
145 cl::init(Val: false), cl::Hidden);
146
147static cl::opt<unsigned> PPCMinimumJumpTableEntries(
148 "ppc-min-jump-table-entries", cl::init(Val: 64), cl::Hidden,
149 cl::desc("Set minimum number of entries to use a jump table on PPC"));
150
151static cl::opt<unsigned> PPCMinimumBitTestCmps(
152 "ppc-min-bit-test-cmps", cl::init(Val: 3), cl::Hidden,
153 cl::desc("Set minimum of largest number of comparisons to use bit test for "
154 "switch on PPC."));
155
156static cl::opt<unsigned> PPCGatherAllAliasesMaxDepth(
157 "ppc-gather-alias-max-depth", cl::init(Val: 18), cl::Hidden,
158 cl::desc("max depth when checking alias info in GatherAllAliases()"));
159
160static cl::opt<unsigned> PPCAIXTLSModelOptUseIEForLDLimit(
161 "ppc-aix-shared-lib-tls-model-opt-limit", cl::init(Val: 1), cl::Hidden,
162 cl::desc("Set inclusive limit count of TLS local-dynamic access(es) in a "
163 "function to use initial-exec"));
164
165STATISTIC(NumTailCalls, "Number of tail calls");
166STATISTIC(NumSiblingCalls, "Number of sibling calls");
167STATISTIC(ShufflesHandledWithVPERM,
168 "Number of shuffles lowered to a VPERM or XXPERM");
169STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed");
170
171static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
172
173static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
174
175static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
176 unsigned OpIdx, bool IsByte,
177 const PPCInstrInfo *TII);
178
179// A faster local-[exec|dynamic] TLS access sequence (enabled with the
180// -maix-small-local-[exec|dynamic]-tls option) can be produced for TLS
181// variables; consistent with the IBM XL compiler, we apply a max size of
182// slightly under 32KB.
183constexpr uint64_t AIXSmallTlsPolicySizeLimit = 32751;
184
185// FIXME: Remove this once the bug has been fixed!
186extern cl::opt<bool> ANDIGlueBug;
187
188PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
189 const PPCSubtarget &STI)
190 : TargetLowering(TM, STI), Subtarget(STI) {
191 // Initialize map that relates the PPC addressing modes to the computed flags
192 // of a load/store instruction. The map is used to determine the optimal
193 // addressing mode when selecting load and stores.
194 initializeAddrModeMap();
195 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
196 // arguments are at least 4/8 bytes aligned.
197 bool isPPC64 = Subtarget.isPPC64();
198 setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4));
199 const MVT RegVT = Subtarget.getScalarIntVT();
200
201 // Set up the register classes.
202 addRegisterClass(VT: MVT::i32, RC: &PPC::GPRCRegClass);
203 if (!useSoftFloat()) {
204 if (hasSPE()) {
205 addRegisterClass(VT: MVT::f32, RC: &PPC::GPRCRegClass);
206 // EFPU2 APU only supports f32
207 if (!Subtarget.hasEFPU2())
208 addRegisterClass(VT: MVT::f64, RC: &PPC::SPERCRegClass);
209 } else {
210 addRegisterClass(VT: MVT::f32, RC: &PPC::F4RCRegClass);
211 addRegisterClass(VT: MVT::f64, RC: &PPC::F8RCRegClass);
212 }
213 }
214
215 setOperationAction(Op: ISD::UADDO, VT: RegVT, Action: Custom);
216 setOperationAction(Op: ISD::USUBO, VT: RegVT, Action: Custom);
217
218 // PowerPC uses addo_carry,subo_carry to propagate carry.
219 setOperationAction(Op: ISD::UADDO_CARRY, VT: RegVT, Action: Custom);
220 setOperationAction(Op: ISD::USUBO_CARRY, VT: RegVT, Action: Custom);
221
222 // On P10, the default lowering generates better code using the
223 // setbc instruction.
224 if (!Subtarget.hasP10Vector()) {
225 setOperationAction(Op: ISD::SSUBO, VT: MVT::i32, Action: Custom);
226 setOperationAction(Op: ISD::SADDO, VT: MVT::i32, Action: Custom);
227 if (isPPC64) {
228 setOperationAction(Op: ISD::SSUBO, VT: MVT::i64, Action: Custom);
229 setOperationAction(Op: ISD::SADDO, VT: MVT::i64, Action: Custom);
230 }
231 }
232
233 // Match BITREVERSE to customized fast code sequence in the td file.
234 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
235 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
236
237 // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended.
238 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i32, Action: Custom);
239
240 // Custom lower inline assembly to check for special registers.
241 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
242 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
243
244 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD.
245 for (MVT VT : MVT::integer_valuetypes()) {
246 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
247 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i8, Action: Expand);
248 }
249
250 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
251 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f128, Action: Expand);
252
253 if (Subtarget.isISA3_0()) {
254 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Legal);
255 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
256 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
257 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
258 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
259 } else {
260 // No extending loads from f16 or HW conversions back and forth.
261 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
262 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f128, Action: Expand);
263 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
264 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
265 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
266 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
267 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
268 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
269 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
270 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
271 }
272
273 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
274
275 // PowerPC has pre-inc load and store's.
276 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
277 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
278 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
279 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
280 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
281 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
282 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
283 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
284 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
285 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
286 if (!Subtarget.hasSPE()) {
287 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
288 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
289 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
290 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
291 }
292
293 if (Subtarget.useCRBits()) {
294 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
295
296 if (isPPC64 || Subtarget.hasFPCVT()) {
297 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Promote);
298 AddPromotedToType(Opc: ISD::STRICT_SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
299 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Promote);
300 AddPromotedToType(Opc: ISD::STRICT_UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
301
302 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
303 AddPromotedToType(Opc: ISD::SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
304 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
305 AddPromotedToType(Opc: ISD::UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
306
307 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i1, Action: Promote);
308 AddPromotedToType(Opc: ISD::STRICT_FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
309 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i1, Action: Promote);
310 AddPromotedToType(Opc: ISD::STRICT_FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
311
312 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
313 AddPromotedToType(Opc: ISD::FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
314 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
315 AddPromotedToType(Opc: ISD::FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
316 } else {
317 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Custom);
318 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Custom);
319 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Custom);
320 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Custom);
321 }
322
323 // PowerPC does not support direct load/store of condition registers.
324 setOperationAction(Op: ISD::LOAD, VT: MVT::i1, Action: Custom);
325 setOperationAction(Op: ISD::STORE, VT: MVT::i1, Action: Custom);
326
327 // FIXME: Remove this once the ANDI glue bug is fixed:
328 if (ANDIGlueBug)
329 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::i1, Action: Custom);
330
331 for (MVT VT : MVT::integer_valuetypes()) {
332 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
333 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
334 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
335 }
336
337 addRegisterClass(VT: MVT::i1, RC: &PPC::CRBITRCRegClass);
338 }
339
340 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
341 // PPC (the libcall is not available).
342 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
343 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
344 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
345 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
346
347 // We do not currently implement these libm ops for PowerPC.
348 setOperationAction(Op: ISD::FFLOOR, VT: MVT::ppcf128, Action: Expand);
349 setOperationAction(Op: ISD::FCEIL, VT: MVT::ppcf128, Action: Expand);
350 setOperationAction(Op: ISD::FTRUNC, VT: MVT::ppcf128, Action: Expand);
351 setOperationAction(Op: ISD::FRINT, VT: MVT::ppcf128, Action: Expand);
352 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::ppcf128, Action: Expand);
353 setOperationAction(Op: ISD::FREM, VT: MVT::ppcf128, Action: LibCall);
354
355 // PowerPC has no SREM/UREM instructions unless we are on P9
356 // On P9 we may use a hardware instruction to compute the remainder.
357 // When the result of both the remainder and the division is required it is
358 // more efficient to compute the remainder from the result of the division
359 // rather than use the remainder instruction. The instructions are legalized
360 // directly because the DivRemPairsPass performs the transformation at the IR
361 // level.
362 if (Subtarget.isISA3_0()) {
363 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Legal);
364 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Legal);
365 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Legal);
366 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Legal);
367 } else {
368 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Expand);
369 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Expand);
370 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Expand);
371 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Expand);
372 }
373
374 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM.
375 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i32, Action: Expand);
376 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i32, Action: Expand);
377 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i64, Action: Expand);
378 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i64, Action: Expand);
379 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Expand);
380 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Expand);
381 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i64, Action: Expand);
382 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i64, Action: Expand);
383
384 // Handle constrained floating-point operations of scalar.
385 // TODO: Handle SPE specific operation.
386 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f32, Action: Legal);
387 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f32, Action: Legal);
388 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f32, Action: Legal);
389 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f32, Action: Legal);
390 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
391
392 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f64, Action: Legal);
393 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f64, Action: Legal);
394 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f64, Action: Legal);
395 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f64, Action: Legal);
396
397 if (!Subtarget.hasSPE()) {
398 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f32, Action: Legal);
399 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f64, Action: Legal);
400 }
401
402 if (Subtarget.hasVSX()) {
403 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f32, Action: Legal);
404 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f64, Action: Legal);
405 }
406
407 if (Subtarget.hasFSQRT()) {
408 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f32, Action: Legal);
409 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f64, Action: Legal);
410 }
411
412 if (Subtarget.hasFPRND()) {
413 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f32, Action: Legal);
414 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f32, Action: Legal);
415 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f32, Action: Legal);
416 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f32, Action: Legal);
417
418 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f64, Action: Legal);
419 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f64, Action: Legal);
420 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f64, Action: Legal);
421 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f64, Action: Legal);
422 }
423
424 // We don't support sin/cos/sqrt/fmod/pow
425 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Expand);
426 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Expand);
427 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
428 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
429 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Expand);
430 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Expand);
431 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Expand);
432 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
433 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
434 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Expand);
435
436 // MASS transformation for LLVM intrinsics with replicating fast-math flag
437 // to be consistent to PPCGenScalarMASSEntries pass
438 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive) {
439 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Custom);
440 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Custom);
441 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Custom);
442 setOperationAction(Op: ISD::FLOG, VT: MVT::f64, Action: Custom);
443 setOperationAction(Op: ISD::FLOG10, VT: MVT::f64, Action: Custom);
444 setOperationAction(Op: ISD::FEXP, VT: MVT::f64, Action: Custom);
445 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Custom);
446 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Custom);
447 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Custom);
448 setOperationAction(Op: ISD::FLOG, VT: MVT::f32, Action: Custom);
449 setOperationAction(Op: ISD::FLOG10, VT: MVT::f32, Action: Custom);
450 setOperationAction(Op: ISD::FEXP, VT: MVT::f32, Action: Custom);
451 }
452
453 if (Subtarget.hasSPE()) {
454 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Expand);
455 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Expand);
456 } else {
457 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Legal);
458 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Legal);
459 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
460 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
461 }
462
463 if (Subtarget.hasSPE())
464 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
465
466 // If we're enabling GP optimizations, use hardware square root
467 if (!Subtarget.hasFSQRT() && !(Subtarget.hasFRSQRTE() && Subtarget.hasFRE()))
468 setOperationAction(Op: ISD::FSQRT, VT: MVT::f64, Action: Expand);
469
470 if (!Subtarget.hasFSQRT() &&
471 !(Subtarget.hasFRSQRTES() && Subtarget.hasFRES()))
472 setOperationAction(Op: ISD::FSQRT, VT: MVT::f32, Action: Expand);
473
474 if (Subtarget.hasFCPSGN()) {
475 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Legal);
476 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Legal);
477 } else {
478 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Expand);
479 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Expand);
480 }
481
482 if (Subtarget.hasFPRND()) {
483 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
484 setOperationAction(Op: ISD::FCEIL, VT: MVT::f64, Action: Legal);
485 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f64, Action: Legal);
486 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
487
488 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f32, Action: Legal);
489 setOperationAction(Op: ISD::FCEIL, VT: MVT::f32, Action: Legal);
490 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f32, Action: Legal);
491 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
492 }
493
494 // Prior to P10, PowerPC does not have BSWAP, but we can use vector BSWAP
495 // instruction xxbrd to speed up scalar BSWAP64.
496 if (Subtarget.isISA3_1()) {
497 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
498 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
499 } else {
500 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Expand);
501 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64,
502 Action: ((Subtarget.hasP8Vector()) && isPPC64) ? Custom
503 : Expand);
504 }
505
506 // CTPOP or CTTZ were introduced in P8/P9 respectively
507 if (Subtarget.isISA3_0()) {
508 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Legal);
509 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Legal);
510 } else {
511 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Expand);
512 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Expand);
513 }
514
515 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
516 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Legal);
517 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Legal);
518 } else {
519 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Expand);
520 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Expand);
521 }
522
523 // PowerPC does not have ROTR
524 setOperationAction(Op: ISD::ROTR, VT: MVT::i32 , Action: Expand);
525 setOperationAction(Op: ISD::ROTR, VT: MVT::i64 , Action: Expand);
526
527 if (!Subtarget.useCRBits()) {
528 // PowerPC does not have Select
529 setOperationAction(Op: ISD::SELECT, VT: MVT::i32, Action: Expand);
530 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Expand);
531 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Expand);
532 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Expand);
533 }
534
535 // PowerPC wants to turn select_cc of FP into fsel when possible.
536 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Custom);
537 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Custom);
538
539 // PowerPC wants to optimize integer setcc a bit
540 if (!Subtarget.useCRBits())
541 setOperationAction(Op: ISD::SETCC, VT: MVT::i32, Action: Custom);
542
543 if (Subtarget.hasFPU()) {
544 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
545 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
546 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Legal);
547
548 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
549 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
550 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Legal);
551 }
552
553 // PowerPC does not have BRCOND which requires SetCC
554 if (!Subtarget.useCRBits())
555 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
556
557 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
558
559 if (Subtarget.hasSPE()) {
560 // SPE has built-in conversions
561 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Legal);
562 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Legal);
563 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Legal);
564 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Legal);
565 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Legal);
566 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Legal);
567
568 // SPE supports signaling compare of f32/f64.
569 // But it doesn't comply IEEE-754 rules for comparing
570 // special values like NaNs, Infs.
571 setOperationAction(Op: ISD::SETCC, VT: MVT::f32, Action: Custom);
572 setOperationAction(Op: ISD::SETCC, VT: MVT::f64, Action: Custom);
573 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Custom);
574 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Custom);
575 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Custom);
576 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Custom);
577 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Custom);
578 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Custom);
579 } else {
580 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores.
581 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
582 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
583
584 // PowerPC does not have [U|S]INT_TO_FP
585 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Expand);
586 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Expand);
587 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Expand);
588 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Expand);
589 }
590
591 if (Subtarget.hasDirectMove() && isPPC64) {
592 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Legal);
593 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Legal);
594 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Legal);
595 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Legal);
596
597 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f64, Action: Custom);
598 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f32, Action: Custom);
599 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f64, Action: Custom);
600 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f32, Action: Custom);
601 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f64, Action: Custom);
602 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f32, Action: Custom);
603 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f64, Action: Custom);
604 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f32, Action: Custom);
605 } else {
606 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Expand);
607 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Expand);
608 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Expand);
609 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Expand);
610 }
611
612 // We cannot sextinreg(i1). Expand to shifts.
613 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
614
615 // Custom handling for PowerPC ucmp and scmp instructions
616 setOperationAction(Op: ISD::UCMP, VT: MVT::i32, Action: Custom);
617 setOperationAction(Op: ISD::UCMP, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
618 setOperationAction(Op: ISD::ABDU, VT: MVT::i32, Action: Custom);
619 setOperationAction(Op: ISD::ABDU, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
620 setOperationAction(Op: ISD::SCMP, VT: MVT::i32, Action: Custom);
621 setOperationAction(Op: ISD::SCMP, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
622
623 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
624 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
625 // support continuation, user-level threading, and etc.. As a result, no
626 // other SjLj exception interfaces are implemented and please don't build
627 // your own exception handling based on them.
628 // LLVM/Clang supports zero-cost DWARF exception handling.
629 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
630 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
631
632 // We want to legalize GlobalAddress and ConstantPool nodes into the
633 // appropriate instructions to materialize the address.
634 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
635 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
636 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
637 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
638 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
639 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i64, Action: Custom);
640 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i64, Action: Custom);
641 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i64, Action: Custom);
642 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i64, Action: Custom);
643 setOperationAction(Op: ISD::JumpTable, VT: MVT::i64, Action: Custom);
644
645 // TRAP is legal.
646 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
647
648 // TRAMPOLINE is custom lowered.
649 setOperationAction(Op: ISD::INIT_TRAMPOLINE, VT: MVT::Other, Action: Custom);
650 setOperationAction(Op: ISD::ADJUST_TRAMPOLINE, VT: MVT::Other, Action: Custom);
651
652 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
653 setOperationAction(Op: ISD::VASTART , VT: MVT::Other, Action: Custom);
654
655 if (Subtarget.is64BitELFABI()) {
656 // VAARG always uses double-word chunks, so promote anything smaller.
657 setOperationAction(Op: ISD::VAARG, VT: MVT::i1, Action: Promote);
658 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i1, DestVT: MVT::i64);
659 setOperationAction(Op: ISD::VAARG, VT: MVT::i8, Action: Promote);
660 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i8, DestVT: MVT::i64);
661 setOperationAction(Op: ISD::VAARG, VT: MVT::i16, Action: Promote);
662 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i16, DestVT: MVT::i64);
663 setOperationAction(Op: ISD::VAARG, VT: MVT::i32, Action: Promote);
664 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i32, DestVT: MVT::i64);
665 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
666 } else if (Subtarget.is32BitELFABI()) {
667 // VAARG is custom lowered with the 32-bit SVR4 ABI.
668 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
669 setOperationAction(Op: ISD::VAARG, VT: MVT::i64, Action: Custom);
670 } else
671 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
672
673 // VACOPY is custom lowered with the 32-bit SVR4 ABI.
674 if (Subtarget.is32BitELFABI())
675 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Custom);
676 else
677 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Expand);
678
679 // Use the default implementation.
680 setOperationAction(Op: ISD::VAEND , VT: MVT::Other, Action: Expand);
681 setOperationAction(Op: ISD::STACKSAVE , VT: MVT::Other, Action: Expand);
682 setOperationAction(Op: ISD::STACKRESTORE , VT: MVT::Other, Action: Custom);
683 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32 , Action: Custom);
684 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i64 , Action: Custom);
685 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i32, Action: Custom);
686 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i64, Action: Custom);
687 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
688 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i64, Action: Custom);
689
690 if (Subtarget.isISA3_0() && isPPC64) {
691 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v16i1, Action: Custom);
692 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v8i1, Action: Custom);
693 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v4i1, Action: Custom);
694 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v2i1, Action: Custom);
695 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v16i1, Action: Custom);
696 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v8i1, Action: Custom);
697 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v4i1, Action: Custom);
698 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v2i1, Action: Custom);
699 }
700
701 // We want to custom lower some of our intrinsics.
702 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
703 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::f64, Action: Custom);
704 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::ppcf128, Action: Custom);
705 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v4f32, Action: Custom);
706 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v2f64, Action: Custom);
707
708 // To handle counter-based loop conditions.
709 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i1, Action: Custom);
710 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
711
712 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i8, Action: Custom);
713 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i16, Action: Custom);
714 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
715 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
716
717 // Comparisons that require checking two conditions.
718 if (Subtarget.hasSPE()) {
719 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f32, Action: Expand);
720 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f64, Action: Expand);
721 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f32, Action: Expand);
722 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f64, Action: Expand);
723 }
724 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f32, Action: Expand);
725 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f64, Action: Expand);
726 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f32, Action: Expand);
727 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f64, Action: Expand);
728 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f32, Action: Expand);
729 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f64, Action: Expand);
730 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f32, Action: Expand);
731 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f64, Action: Expand);
732 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f32, Action: Expand);
733 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f64, Action: Expand);
734 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f32, Action: Expand);
735 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f64, Action: Expand);
736
737 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
738 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
739
740 if (Subtarget.has64BitSupport()) {
741 // They also have instructions for converting between i64 and fp.
742 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
743 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Expand);
744 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
745 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Expand);
746 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
747 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Expand);
748 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
749 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Expand);
750 // This is just the low 32 bits of a (signed) fp->i64 conversion.
751 // We cannot do this with Promote because i64 is not a legal type.
752 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
753 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
754
755 if (Subtarget.hasLFIWAX() || isPPC64) {
756 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
757 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
758 }
759 } else {
760 // PowerPC does not have FP_TO_UINT on 32-bit implementations.
761 if (Subtarget.hasSPE()) {
762 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Legal);
763 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Legal);
764 } else {
765 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Expand);
766 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Expand);
767 }
768 }
769
770 // With the instructions enabled under FPCVT, we can do everything.
771 if (Subtarget.hasFPCVT()) {
772 if (Subtarget.has64BitSupport()) {
773 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
774 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Custom);
775 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
776 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Custom);
777 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
778 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Custom);
779 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
780 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
781 }
782
783 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
784 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
785 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
786 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Custom);
787 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
788 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
789 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
790 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Custom);
791 }
792
793 if (Subtarget.use64BitRegs()) {
794 // 64-bit PowerPC implementations can support i64 types directly
795 addRegisterClass(VT: MVT::i64, RC: &PPC::G8RCRegClass);
796 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
797 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
798 // 64-bit PowerPC wants to expand i128 shifts itself.
799 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Custom);
800 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Custom);
801 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Custom);
802 } else {
803 // 32-bit PowerPC wants to expand i64 shifts itself.
804 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Custom);
805 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Custom);
806 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Custom);
807 }
808
809 // PowerPC has better expansions for funnel shifts than the generic
810 // TargetLowering::expandFunnelShift.
811 if (Subtarget.has64BitSupport()) {
812 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Custom);
813 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Custom);
814 }
815 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Custom);
816 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Custom);
817
818 if (Subtarget.hasVSX()) {
819 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
820 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
821 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
822 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
823 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
824 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
825 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
826 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
827 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
828 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
829 }
830
831 if (Subtarget.hasAltivec()) {
832 for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
833 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
834 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
835 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
836 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
837 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
838 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
839 }
840 // First set operation action for all vector types to expand. Then we
841 // will selectively turn on ones that can be effectively codegen'd.
842 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
843 // add/sub are legal for all supported vector VT's.
844 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
845 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
846
847 // For v2i64, these are only valid with P8Vector. This is corrected after
848 // the loop.
849 if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) {
850 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
851 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
852 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
853 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
854 }
855 else {
856 setOperationAction(Op: ISD::SMAX, VT, Action: Expand);
857 setOperationAction(Op: ISD::SMIN, VT, Action: Expand);
858 setOperationAction(Op: ISD::UMAX, VT, Action: Expand);
859 setOperationAction(Op: ISD::UMIN, VT, Action: Expand);
860 }
861
862 if (Subtarget.hasVSX()) {
863 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT, Action: Legal);
864 setOperationAction(Op: ISD::FMINNUM_IEEE, VT, Action: Legal);
865 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
866 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
867 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Legal);
868 }
869
870 // Vector instructions introduced in P8
871 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
872 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
873 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
874 }
875 else {
876 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
877 setOperationAction(Op: ISD::CTLZ, VT, Action: Expand);
878 }
879
880 // Vector instructions introduced in P9
881 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128))
882 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
883 else
884 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
885
886 // We promote all shuffles to v16i8.
887 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Promote);
888 AddPromotedToType (Opc: ISD::VECTOR_SHUFFLE, OrigVT: VT, DestVT: MVT::v16i8);
889
890 // We promote all non-typed operations to v4i32.
891 setOperationAction(Op: ISD::AND , VT, Action: Promote);
892 AddPromotedToType (Opc: ISD::AND , OrigVT: VT, DestVT: MVT::v4i32);
893 setOperationAction(Op: ISD::OR , VT, Action: Promote);
894 AddPromotedToType (Opc: ISD::OR , OrigVT: VT, DestVT: MVT::v4i32);
895 setOperationAction(Op: ISD::XOR , VT, Action: Promote);
896 AddPromotedToType (Opc: ISD::XOR , OrigVT: VT, DestVT: MVT::v4i32);
897 setOperationAction(Op: ISD::LOAD , VT, Action: Promote);
898 AddPromotedToType (Opc: ISD::LOAD , OrigVT: VT, DestVT: MVT::v4i32);
899 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
900 AddPromotedToType (Opc: ISD::SELECT, OrigVT: VT, DestVT: MVT::v4i32);
901 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
902 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Promote);
903 AddPromotedToType (Opc: ISD::SELECT_CC, OrigVT: VT, DestVT: MVT::v4i32);
904 setOperationAction(Op: ISD::STORE, VT, Action: Promote);
905 AddPromotedToType (Opc: ISD::STORE, OrigVT: VT, DestVT: MVT::v4i32);
906
907 // No other operations are legal.
908 setOperationAction(Op: ISD::MUL , VT, Action: Expand);
909 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
910 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
911 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
912 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
913 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
914 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
915 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
916 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
917 setOperationAction(Op: ISD::FLOG, VT, Action: Expand);
918 setOperationAction(Op: ISD::FLOG10, VT, Action: Expand);
919 setOperationAction(Op: ISD::FLOG2, VT, Action: Expand);
920 setOperationAction(Op: ISD::FEXP, VT, Action: Expand);
921 setOperationAction(Op: ISD::FEXP2, VT, Action: Expand);
922 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
923 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
924 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
925 setOperationAction(Op: ISD::FFLOOR, VT, Action: Expand);
926 setOperationAction(Op: ISD::FCEIL, VT, Action: Expand);
927 setOperationAction(Op: ISD::FTRUNC, VT, Action: Expand);
928 setOperationAction(Op: ISD::FRINT, VT, Action: Expand);
929 setOperationAction(Op: ISD::FLDEXP, VT, Action: Expand);
930 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Expand);
931 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Expand);
932 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Expand);
933 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Expand);
934 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
935 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
936 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
937 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
938 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
939 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
940 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Expand);
941 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
942 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
943 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT, Action: Expand);
944 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
945 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
946
947 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
948 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
949 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
950 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
951 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
952 }
953 }
954 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::v4i32, Action: Expand);
955 if (!Subtarget.hasP8Vector()) {
956 setOperationAction(Op: ISD::SMAX, VT: MVT::v2i64, Action: Expand);
957 setOperationAction(Op: ISD::SMIN, VT: MVT::v2i64, Action: Expand);
958 setOperationAction(Op: ISD::UMAX, VT: MVT::v2i64, Action: Expand);
959 setOperationAction(Op: ISD::UMIN, VT: MVT::v2i64, Action: Expand);
960 }
961
962 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
963 // with merges, splats, etc.
964 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v16i8, Action: Custom);
965
966 // Vector truncates to sub-word integer that fit in an Altivec/VSX register
967 // are cheap, so handle them before they get expanded to scalar.
968 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v8i8, Action: Custom);
969 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i8, Action: Custom);
970 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i8, Action: Custom);
971 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i16, Action: Custom);
972 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i16, Action: Custom);
973
974 setOperationAction(Op: ISD::AND , VT: MVT::v4i32, Action: Legal);
975 setOperationAction(Op: ISD::OR , VT: MVT::v4i32, Action: Legal);
976 setOperationAction(Op: ISD::XOR , VT: MVT::v4i32, Action: Legal);
977 setOperationAction(Op: ISD::LOAD , VT: MVT::v4i32, Action: Legal);
978 setOperationAction(Op: ISD::SELECT, VT: MVT::v4i32,
979 Action: Subtarget.useCRBits() ? Legal : Expand);
980 setOperationAction(Op: ISD::STORE , VT: MVT::v4i32, Action: Legal);
981 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
982 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
983 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
984 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
985 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
986 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
987 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
988 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
989 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
990 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
991 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
992 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
993
994 // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8.
995 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Custom);
996 // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w).
997 if (Subtarget.hasAltivec())
998 for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8})
999 setOperationAction(Op: ISD::ROTL, VT, Action: Legal);
1000 // With hasP8Altivec set, we can lower ISD::ROTL to vrld.
1001 if (Subtarget.hasP8Altivec())
1002 setOperationAction(Op: ISD::ROTL, VT: MVT::v2i64, Action: Legal);
1003
1004 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VRRCRegClass);
1005 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VRRCRegClass);
1006 addRegisterClass(VT: MVT::v8i16, RC: &PPC::VRRCRegClass);
1007 addRegisterClass(VT: MVT::v16i8, RC: &PPC::VRRCRegClass);
1008
1009 setOperationAction(Op: ISD::MUL, VT: MVT::v4f32, Action: Legal);
1010 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
1011
1012 if (Subtarget.hasVSX()) {
1013 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
1014 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
1015 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
1016 }
1017
1018 if (Subtarget.hasP8Altivec())
1019 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Legal);
1020 else
1021 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Custom);
1022
1023 if (Subtarget.isISA3_1()) {
1024 setOperationAction(Op: ISD::MUL, VT: MVT::v2i64, Action: Legal);
1025 setOperationAction(Op: ISD::MULHS, VT: MVT::v2i64, Action: Legal);
1026 setOperationAction(Op: ISD::MULHU, VT: MVT::v2i64, Action: Legal);
1027 setOperationAction(Op: ISD::MULHS, VT: MVT::v4i32, Action: Legal);
1028 setOperationAction(Op: ISD::MULHU, VT: MVT::v4i32, Action: Legal);
1029 setOperationAction(Op: ISD::UDIV, VT: MVT::v2i64, Action: Legal);
1030 setOperationAction(Op: ISD::SDIV, VT: MVT::v2i64, Action: Legal);
1031 setOperationAction(Op: ISD::UDIV, VT: MVT::v4i32, Action: Legal);
1032 setOperationAction(Op: ISD::SDIV, VT: MVT::v4i32, Action: Legal);
1033 setOperationAction(Op: ISD::UREM, VT: MVT::v2i64, Action: Legal);
1034 setOperationAction(Op: ISD::SREM, VT: MVT::v2i64, Action: Legal);
1035 setOperationAction(Op: ISD::UREM, VT: MVT::v4i32, Action: Legal);
1036 setOperationAction(Op: ISD::SREM, VT: MVT::v4i32, Action: Legal);
1037 setOperationAction(Op: ISD::UREM, VT: MVT::v1i128, Action: Legal);
1038 setOperationAction(Op: ISD::SREM, VT: MVT::v1i128, Action: Legal);
1039 setOperationAction(Op: ISD::UDIV, VT: MVT::v1i128, Action: Legal);
1040 setOperationAction(Op: ISD::SDIV, VT: MVT::v1i128, Action: Legal);
1041 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Legal);
1042 }
1043
1044 setOperationAction(Op: ISD::MUL, VT: MVT::v8i16, Action: Legal);
1045 setOperationAction(Op: ISD::MUL, VT: MVT::v16i8, Action: Custom);
1046
1047 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Custom);
1048 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Custom);
1049 // LE is P8+/64-bit so direct moves are supported and these operations
1050 // are legal. The custom transformation requires 64-bit since we need a
1051 // pair of stores that will cover a 128-bit load for P10.
1052 if (EnableP10StoreForward && isPPC64 && !Subtarget.isLittleEndian()) {
1053 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Custom);
1054 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Custom);
1055 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Custom);
1056 }
1057
1058 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v16i8, Action: Custom);
1059 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v8i16, Action: Custom);
1060 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i32, Action: Custom);
1061 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4f32, Action: Custom);
1062
1063 // Altivec does not contain unordered floating-point compare instructions
1064 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v4f32, Action: Expand);
1065 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v4f32, Action: Expand);
1066 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v4f32, Action: Expand);
1067 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v4f32, Action: Expand);
1068
1069 if (Subtarget.hasVSX()) {
1070 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
1071 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1072 if (Subtarget.hasP8Vector()) {
1073 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
1074 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Legal);
1075 }
1076 if (Subtarget.hasDirectMove() && isPPC64) {
1077 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Legal);
1078 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Legal);
1079 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Legal);
1080 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Legal);
1081 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1082 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1083 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1084 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1085 }
1086 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1087
1088 // The nearbyint variants are not allowed to raise the inexact exception
1089 // so we can only code-gen them with fpexcept.ignore.
1090 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f64, Action: Custom);
1091 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f32, Action: Custom);
1092 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Custom);
1093 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Custom);
1094
1095 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
1096 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
1097 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
1098 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
1099 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
1100 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
1101 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
1102
1103 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
1104 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
1105 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
1106 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
1107
1108 setOperationAction(Op: ISD::MUL, VT: MVT::v2f64, Action: Legal);
1109 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
1110
1111 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
1112 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
1113
1114 // Share the Altivec comparison restrictions.
1115 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v2f64, Action: Expand);
1116 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v2f64, Action: Expand);
1117 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v2f64, Action: Expand);
1118 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v2f64, Action: Expand);
1119
1120 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Legal);
1121 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Legal);
1122
1123 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2f64, Action: Custom);
1124
1125 if (Subtarget.hasP8Vector())
1126 addRegisterClass(VT: MVT::f32, RC: &PPC::VSSRCRegClass);
1127
1128 addRegisterClass(VT: MVT::f64, RC: &PPC::VSFRCRegClass);
1129
1130 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VSRCRegClass);
1131 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VSRCRegClass);
1132 addRegisterClass(VT: MVT::v2f64, RC: &PPC::VSRCRegClass);
1133
1134 if (Subtarget.hasP8Altivec()) {
1135 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Legal);
1136 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Legal);
1137 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Legal);
1138
1139 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1140 // SRL, but not for SRA because of the instructions available:
1141 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth
1142 // doing
1143 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Expand);
1144 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Expand);
1145 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1146
1147 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Legal);
1148 }
1149 else {
1150 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Expand);
1151 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Expand);
1152 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Expand);
1153
1154 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Custom);
1155
1156 // VSX v2i64 only supports non-arithmetic operations.
1157 setOperationAction(Op: ISD::ADD, VT: MVT::v2i64, Action: Expand);
1158 setOperationAction(Op: ISD::SUB, VT: MVT::v2i64, Action: Expand);
1159 }
1160
1161 if (Subtarget.isISA3_1())
1162 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Legal);
1163 else
1164 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Expand);
1165
1166 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
1167 AddPromotedToType (Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1168 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
1169 AddPromotedToType (Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1170
1171 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2i64, Action: Custom);
1172
1173 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1174 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1175 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1176 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1177 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1178 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1179 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1180 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1181
1182 // Custom handling for partial vectors of integers converted to
1183 // floating point. We already have optimal handling for v2i32 through
1184 // the DAG combine, so those aren't necessary.
1185 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1186 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1187 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1188 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1189 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1190 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1191 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1192 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1193 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1194 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1195 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1196 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1197 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1198 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1199 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1200 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1201
1202 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
1203 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
1204 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
1205 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
1206 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v4f32, Action: Legal);
1207 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f64, Action: Legal);
1208
1209 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2i64, Action: Custom);
1210 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2f64, Action: Custom);
1211
1212 // Handle constrained floating-point operations of vector.
1213 // The predictor is `hasVSX` because altivec instruction has
1214 // no exception but VSX vector instruction has.
1215 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
1216 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
1217 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
1218 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
1219 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
1220 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
1221 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v4f32, Action: Legal);
1222 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v4f32, Action: Legal);
1223 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
1224 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
1225 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
1226 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
1227 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
1228
1229 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
1230 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
1231 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
1232 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
1233 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
1234 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
1235 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v2f64, Action: Legal);
1236 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v2f64, Action: Legal);
1237 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
1238 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
1239 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
1240 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
1241 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
1242
1243 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VSRCRegClass);
1244 addRegisterClass(VT: MVT::f128, RC: &PPC::VRRCRegClass);
1245
1246 for (MVT FPT : MVT::fp_valuetypes())
1247 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: FPT, Action: Expand);
1248
1249 // Expand the SELECT to SELECT_CC
1250 setOperationAction(Op: ISD::SELECT, VT: MVT::f128, Action: Expand);
1251
1252 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
1253 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
1254
1255 // No implementation for these ops for PowerPC.
1256 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f128, Action: Expand);
1257 setOperationAction(Op: ISD::FSIN, VT: MVT::f128, Action: Expand);
1258 setOperationAction(Op: ISD::FCOS, VT: MVT::f128, Action: Expand);
1259 setOperationAction(Op: ISD::FPOW, VT: MVT::f128, Action: Expand);
1260 setOperationAction(Op: ISD::FPOWI, VT: MVT::f128, Action: Expand);
1261 setOperationAction(Op: ISD::FREM, VT: MVT::f128, Action: LibCall);
1262 }
1263
1264 if (Subtarget.hasP8Altivec()) {
1265 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VRRCRegClass);
1266 addRegisterClass(VT: MVT::v1i128, RC: &PPC::VRRCRegClass);
1267 }
1268
1269 if (Subtarget.hasP9Vector()) {
1270 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Custom);
1271 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
1272
1273 // Test data class instructions store results in CR bits.
1274 if (Subtarget.useCRBits()) {
1275 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Custom);
1276 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Custom);
1277 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f128, Action: Custom);
1278 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::ppcf128, Action: Custom);
1279 }
1280
1281 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1282 // SRL, but not for SRA because of the instructions available:
1283 // VS{RL} and VS{RL}O.
1284 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Legal);
1285 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Legal);
1286 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1287
1288 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: Legal);
1289 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: Legal);
1290 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Legal);
1291 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Legal);
1292 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Legal);
1293
1294 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Legal);
1295 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f128, Action: Expand);
1296 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f128, Action: Expand);
1297 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f128, Action: Expand);
1298 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f128, Action: Expand);
1299 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f128, Action: Expand);
1300 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f128, Action: Expand);
1301
1302 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f128, Action: Legal);
1303 setOperationAction(Op: ISD::FRINT, VT: MVT::f128, Action: Legal);
1304 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f128, Action: Legal);
1305 setOperationAction(Op: ISD::FCEIL, VT: MVT::f128, Action: Legal);
1306 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f128, Action: Legal);
1307 setOperationAction(Op: ISD::FROUND, VT: MVT::f128, Action: Legal);
1308
1309 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f64, Action: Legal);
1310 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f32, Action: Legal);
1311 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
1312
1313 // Handle constrained floating-point operations of fp128
1314 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f128, Action: Legal);
1315 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f128, Action: Legal);
1316 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f128, Action: Legal);
1317 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f128, Action: Legal);
1318 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Legal);
1319 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f128, Action: Legal);
1320 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Legal);
1321 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f64, Action: Legal);
1322 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
1323 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f128, Action: Legal);
1324 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f128, Action: Legal);
1325 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f128, Action: Legal);
1326 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f128, Action: Legal);
1327 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f128, Action: Legal);
1328 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f128, Action: Legal);
1329 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
1330 setOperationAction(Op: ISD::BSWAP, VT: MVT::v8i16, Action: Legal);
1331 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i32, Action: Legal);
1332 setOperationAction(Op: ISD::BSWAP, VT: MVT::v2i64, Action: Legal);
1333 setOperationAction(Op: ISD::BSWAP, VT: MVT::v1i128, Action: Legal);
1334 } else if (Subtarget.hasVSX()) {
1335 setOperationAction(Op: ISD::LOAD, VT: MVT::f128, Action: Promote);
1336 setOperationAction(Op: ISD::STORE, VT: MVT::f128, Action: Promote);
1337
1338 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1339 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1340
1341 // Set FADD/FSUB as libcall to avoid the legalizer to expand the
1342 // fp_to_uint and int_to_fp.
1343 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: LibCall);
1344 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: LibCall);
1345
1346 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Expand);
1347 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Expand);
1348 setOperationAction(Op: ISD::FNEG, VT: MVT::f128, Action: Expand);
1349 setOperationAction(Op: ISD::FABS, VT: MVT::f128, Action: Expand);
1350 setOperationAction(Op: ISD::FSQRT, VT: MVT::f128, Action: Expand);
1351 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
1352 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
1353
1354 // Expand the fp_extend if the target type is fp128.
1355 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Expand);
1356 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Expand);
1357
1358 // Expand the fp_round if the source type is fp128.
1359 for (MVT VT : {MVT::f32, MVT::f64}) {
1360 setOperationAction(Op: ISD::FP_ROUND, VT, Action: Custom);
1361 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Custom);
1362 }
1363
1364 setOperationAction(Op: ISD::SETCC, VT: MVT::f128, Action: Custom);
1365 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Custom);
1366 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Custom);
1367 setOperationAction(Op: ISD::BR_CC, VT: MVT::f128, Action: Expand);
1368
1369 // Lower following f128 select_cc pattern:
1370 // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE
1371 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1372
1373 // We need to handle f128 SELECT_CC with integer result type.
1374 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i32, Action: Custom);
1375 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
1376 }
1377
1378 if (Subtarget.hasP9Altivec()) {
1379 if (Subtarget.isISA3_1()) {
1380 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1381 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1382 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1383 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1384 } else {
1385 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Custom);
1386 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Custom);
1387 }
1388 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i8, Action: Legal);
1389 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i16, Action: Legal);
1390 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i32, Action: Legal);
1391 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1392 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1393 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1394 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i64, Action: Legal);
1395
1396 setOperationAction(Op: ISD::ABDU, VT: MVT::v16i8, Action: Legal);
1397 setOperationAction(Op: ISD::ABDU, VT: MVT::v8i16, Action: Legal);
1398 setOperationAction(Op: ISD::ABDU, VT: MVT::v4i32, Action: Legal);
1399 setOperationAction(Op: ISD::ABDS, VT: MVT::v4i32, Action: Legal);
1400 }
1401
1402 if (Subtarget.hasP10Vector()) {
1403 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1404 }
1405
1406 setOperationAction(Op: ISD::PARTIAL_REDUCE_UMLA, VT: MVT::v16i32, Action: Custom);
1407 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1408 Action: Legal);
1409 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1410 Action: Legal);
1411 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1412 Action: Legal);
1413 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SUMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1414 Action: Legal);
1415 }
1416
1417 if (Subtarget.pairedVectorMemops()) {
1418 addRegisterClass(VT: MVT::v256i1, RC: &PPC::VSRpRCRegClass);
1419 setOperationAction(Op: ISD::LOAD, VT: MVT::v256i1, Action: Custom);
1420 setOperationAction(Op: ISD::STORE, VT: MVT::v256i1, Action: Custom);
1421 }
1422 if (Subtarget.hasMMA()) {
1423 if (Subtarget.isISAFuture()) {
1424 addRegisterClass(VT: MVT::v512i1, RC: &PPC::WACCRCRegClass);
1425 addRegisterClass(VT: MVT::v1024i1, RC: &PPC::DMRRCRegClass);
1426 addRegisterClass(VT: MVT::v2048i1, RC: &PPC::DMRpRCRegClass);
1427 setOperationAction(Op: ISD::LOAD, VT: MVT::v1024i1, Action: Custom);
1428 setOperationAction(Op: ISD::STORE, VT: MVT::v1024i1, Action: Custom);
1429 setOperationAction(Op: ISD::LOAD, VT: MVT::v2048i1, Action: Custom);
1430 setOperationAction(Op: ISD::STORE, VT: MVT::v2048i1, Action: Custom);
1431 } else {
1432 addRegisterClass(VT: MVT::v512i1, RC: &PPC::UACCRCRegClass);
1433 }
1434 setOperationAction(Op: ISD::LOAD, VT: MVT::v512i1, Action: Custom);
1435 setOperationAction(Op: ISD::STORE, VT: MVT::v512i1, Action: Custom);
1436 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v512i1, Action: Custom);
1437 }
1438
1439 if (Subtarget.has64BitSupport())
1440 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Legal);
1441
1442 if (Subtarget.isISA3_1())
1443 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Legal);
1444
1445 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: isPPC64 ? Legal : Custom);
1446
1447 if (!isPPC64) {
1448 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i64, Action: Expand);
1449 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i64, Action: Expand);
1450 }
1451
1452 if (shouldInlineQuadwordAtomics()) {
1453 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
1454 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
1455 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i128, Action: Custom);
1456 }
1457
1458 setBooleanContents(ZeroOrOneBooleanContent);
1459
1460 if (Subtarget.hasAltivec()) {
1461 // Altivec instructions set fields to all zeros or all ones.
1462 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
1463 }
1464
1465 if (shouldInlineQuadwordAtomics())
1466 setMaxAtomicSizeInBitsSupported(128);
1467 else if (isPPC64)
1468 setMaxAtomicSizeInBitsSupported(64);
1469 else
1470 setMaxAtomicSizeInBitsSupported(32);
1471
1472 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1);
1473
1474 // We have target-specific dag combine patterns for the following nodes:
1475 setTargetDAGCombine({ISD::AND, ISD::ADD, ISD::XOR, ISD::SHL, ISD::SRA,
1476 ISD::SRL, ISD::MUL, ISD::FMA, ISD::SINT_TO_FP,
1477 ISD::BUILD_VECTOR});
1478 if (Subtarget.hasFPCVT())
1479 setTargetDAGCombine(ISD::UINT_TO_FP);
1480 setTargetDAGCombine({ISD::LOAD, ISD::STORE, ISD::BR_CC});
1481 if (Subtarget.useCRBits())
1482 setTargetDAGCombine(ISD::BRCOND);
1483 setTargetDAGCombine({ISD::BSWAP, ISD::INTRINSIC_WO_CHAIN,
1484 ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID});
1485
1486 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND, ISD::ANY_EXTEND});
1487
1488 setTargetDAGCombine({ISD::TRUNCATE, ISD::VECTOR_SHUFFLE});
1489
1490 if (Subtarget.useCRBits()) {
1491 setTargetDAGCombine({ISD::SETCC, ISD::SELECT_CC});
1492 }
1493
1494 if (Subtarget.hasP8Vector())
1495 setTargetDAGCombine(ISD::BITCAST);
1496
1497 // With 32 condition bits, we don't need to sink (and duplicate) compares
1498 // aggressively in CodeGenPrep.
1499 if (Subtarget.useCRBits()) {
1500 setJumpIsExpensive();
1501 }
1502
1503 // TODO: The default entry number is set to 64. This stops most jump table
1504 // generation on PPC. But it is good for current PPC HWs because the indirect
1505 // branch instruction mtctr to the jump table may lead to bad branch predict.
1506 // Re-evaluate this value on future HWs that can do better with mtctr.
1507 setMinimumJumpTableEntries(PPCMinimumJumpTableEntries);
1508
1509 // The default minimum of largest number in a BitTest cluster is 3.
1510 setMinimumBitTestCmps(PPCMinimumBitTestCmps);
1511
1512 setMinFunctionAlignment(Align(4));
1513 setMinCmpXchgSizeInBits(Subtarget.hasPartwordAtomics() ? 8 : 32);
1514
1515 auto CPUDirective = Subtarget.getCPUDirective();
1516 switch (CPUDirective) {
1517 default: break;
1518 case PPC::DIR_970:
1519 case PPC::DIR_A2:
1520 case PPC::DIR_E500:
1521 case PPC::DIR_E500mc:
1522 case PPC::DIR_E5500:
1523 case PPC::DIR_PWR4:
1524 case PPC::DIR_PWR5:
1525 case PPC::DIR_PWR5X:
1526 case PPC::DIR_PWR6:
1527 case PPC::DIR_PWR6X:
1528 case PPC::DIR_PWR7:
1529 case PPC::DIR_PWR8:
1530 case PPC::DIR_PWR9:
1531 case PPC::DIR_PWR10:
1532 case PPC::DIR_PWR11:
1533 case PPC::DIR_PWR_FUTURE:
1534 setPrefLoopAlignment(Align(16));
1535 setPrefFunctionAlignment(Align(16));
1536 break;
1537 }
1538
1539 if (Subtarget.enableMachineScheduler())
1540 setSchedulingPreference(Sched::Source);
1541 else
1542 setSchedulingPreference(Sched::Hybrid);
1543
1544 computeRegisterProperties(TRI: STI.getRegisterInfo());
1545
1546 // The Freescale cores do better with aggressive inlining of memcpy and
1547 // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
1548 if (CPUDirective == PPC::DIR_E500mc || CPUDirective == PPC::DIR_E5500) {
1549 MaxStoresPerMemset = 32;
1550 MaxStoresPerMemsetOptSize = 16;
1551 MaxStoresPerMemcpy = 32;
1552 MaxStoresPerMemcpyOptSize = 8;
1553 MaxStoresPerMemmove = 32;
1554 MaxStoresPerMemmoveOptSize = 8;
1555 } else if (CPUDirective == PPC::DIR_A2) {
1556 // The A2 also benefits from (very) aggressive inlining of memcpy and
1557 // friends. The overhead of a the function call, even when warm, can be
1558 // over one hundred cycles.
1559 MaxStoresPerMemset = 128;
1560 MaxStoresPerMemcpy = 128;
1561 MaxStoresPerMemmove = 128;
1562 MaxLoadsPerMemcmp = 128;
1563 } else {
1564 MaxLoadsPerMemcmp = 8;
1565 MaxLoadsPerMemcmpOptSize = 4;
1566 }
1567
1568 // Enable generation of STXVP instructions by default for mcpu=future.
1569 if (CPUDirective == PPC::DIR_PWR_FUTURE &&
1570 EnableAutoPairedVecSt.getNumOccurrences() == 0)
1571 EnableAutoPairedVecSt = true;
1572
1573 IsStrictFPEnabled = true;
1574
1575 // Let the subtarget (CPU) decide if a predictable select is more expensive
1576 // than the corresponding branch. This information is used in CGP to decide
1577 // when to convert selects into branches.
1578 PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive();
1579
1580 GatherAllAliasesMaxDepth = PPCGatherAllAliasesMaxDepth;
1581}
1582
1583// *********************************** NOTE ************************************
1584// For selecting load and store instructions, the addressing modes are defined
1585// as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD
1586// patterns to match the load the store instructions.
1587//
1588// The TD definitions for the addressing modes correspond to their respective
1589// Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely
1590// on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the
1591// address mode flags of a particular node. Afterwards, the computed address
1592// flags are passed into getAddrModeForFlags() in order to retrieve the optimal
1593// addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement
1594// accordingly, based on the preferred addressing mode.
1595//
1596// Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode.
1597// MemOpFlags contains all the possible flags that can be used to compute the
1598// optimal addressing mode for load and store instructions.
1599// AddrMode contains all the possible load and store addressing modes available
1600// on Power (such as DForm, DSForm, DQForm, XForm, etc.)
1601//
1602// When adding new load and store instructions, it is possible that new address
1603// flags may need to be added into MemOpFlags, and a new addressing mode will
1604// need to be added to AddrMode. An entry of the new addressing mode (consisting
1605// of the minimal and main distinguishing address flags for the new load/store
1606// instructions) will need to be added into initializeAddrModeMap() below.
1607// Finally, when adding new addressing modes, the getAddrModeForFlags() will
1608// need to be updated to account for selecting the optimal addressing mode.
1609// *****************************************************************************
1610/// Initialize the map that relates the different addressing modes of the load
1611/// and store instructions to a set of flags. This ensures the load/store
1612/// instruction is correctly matched during instruction selection.
1613void PPCTargetLowering::initializeAddrModeMap() {
1614 AddrModesMap[PPC::AM_DForm] = {
1615 // LWZ, STW
1616 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt,
1617 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt,
1618 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1619 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1620 // LBZ, LHZ, STB, STH
1621 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1622 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1623 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1624 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1625 // LHA
1626 PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1627 PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1628 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1629 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1630 // LFS, LFD, STFS, STFD
1631 PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1632 PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1633 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1634 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1635 };
1636 AddrModesMap[PPC::AM_DSForm] = {
1637 // LWA
1638 PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt,
1639 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1640 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1641 // LD, STD
1642 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt,
1643 PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt,
1644 PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt,
1645 // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64
1646 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1647 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1648 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1649 };
1650 AddrModesMap[PPC::AM_DQForm] = {
1651 // LXV, STXV
1652 PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1653 PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1654 PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1655 };
1656 AddrModesMap[PPC::AM_PrefixDForm] = {PPC::MOF_RPlusSImm34 |
1657 PPC::MOF_SubtargetP10};
1658 // TODO: Add mapping for quadword load/store.
1659}
1660
1661/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
1662/// the desired ByVal argument alignment.
1663static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
1664 if (MaxAlign == MaxMaxAlign)
1665 return;
1666 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
1667 if (MaxMaxAlign >= 32 &&
1668 VTy->getPrimitiveSizeInBits().getFixedValue() >= 256)
1669 MaxAlign = Align(32);
1670 else if (VTy->getPrimitiveSizeInBits().getFixedValue() >= 128 &&
1671 MaxAlign < 16)
1672 MaxAlign = Align(16);
1673 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
1674 Align EltAlign;
1675 getMaxByValAlign(Ty: ATy->getElementType(), MaxAlign&: EltAlign, MaxMaxAlign);
1676 if (EltAlign > MaxAlign)
1677 MaxAlign = EltAlign;
1678 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
1679 for (auto *EltTy : STy->elements()) {
1680 Align EltAlign;
1681 getMaxByValAlign(Ty: EltTy, MaxAlign&: EltAlign, MaxMaxAlign);
1682 if (EltAlign > MaxAlign)
1683 MaxAlign = EltAlign;
1684 if (MaxAlign == MaxMaxAlign)
1685 break;
1686 }
1687 }
1688}
1689
1690/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1691/// function arguments in the caller parameter area.
1692Align PPCTargetLowering::getByValTypeAlignment(Type *Ty,
1693 const DataLayout &DL) const {
1694 // 16byte and wider vectors are passed on 16byte boundary.
1695 // The rest is 8 on PPC64 and 4 on PPC32 boundary.
1696 Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
1697 if (Subtarget.hasAltivec())
1698 getMaxByValAlign(Ty, MaxAlign&: Alignment, MaxMaxAlign: Align(16));
1699 return Alignment;
1700}
1701
1702bool PPCTargetLowering::useSoftFloat() const {
1703 return Subtarget.useSoftFloat();
1704}
1705
1706bool PPCTargetLowering::hasSPE() const {
1707 return Subtarget.hasSPE();
1708}
1709
1710bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
1711 return VT.isScalarInteger();
1712}
1713
1714bool PPCTargetLowering::shallExtractConstSplatVectorElementToStore(
1715 Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const {
1716 if (!Subtarget.isPPC64() || !Subtarget.hasVSX())
1717 return false;
1718
1719 if (auto *VTy = dyn_cast<VectorType>(Val: VectorTy)) {
1720 if (VTy->getScalarType()->isIntegerTy()) {
1721 // ElemSizeInBits 8/16 can fit in immediate field, not needed here.
1722 if (ElemSizeInBits == 32) {
1723 Index = Subtarget.isLittleEndian() ? 2 : 1;
1724 return true;
1725 }
1726 if (ElemSizeInBits == 64) {
1727 Index = Subtarget.isLittleEndian() ? 1 : 0;
1728 return true;
1729 }
1730 }
1731 }
1732 return false;
1733}
1734
1735EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1736 EVT VT) const {
1737 if (!VT.isVector())
1738 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
1739
1740 return VT.changeVectorElementTypeToInteger();
1741}
1742
1743bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const {
1744 assert(VT.isFloatingPoint() && "Non-floating-point FMA?");
1745 return true;
1746}
1747
1748//===----------------------------------------------------------------------===//
1749// Node matching predicates, for use by the tblgen matching code.
1750//===----------------------------------------------------------------------===//
1751
1752/// isFloatingPointZero - Return true if this is 0.0 or -0.0.
1753static bool isFloatingPointZero(SDValue Op) {
1754 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
1755 return CFP->getValueAPF().isZero();
1756 else if (ISD::isEXTLoad(N: Op.getNode()) || ISD::isNON_EXTLoad(N: Op.getNode())) {
1757 // Maybe this has already been legalized into the constant pool?
1758 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val: Op.getOperand(i: 1)))
1759 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CP->getConstVal()))
1760 return CFP->getValueAPF().isZero();
1761 }
1762 return false;
1763}
1764
1765/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return
1766/// true if Op is undef or if it matches the specified value.
1767static bool isConstantOrUndef(int Op, int Val) {
1768 return Op < 0 || Op == Val;
1769}
1770
1771/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
1772/// VPKUHUM instruction.
1773/// The ShuffleKind distinguishes between big-endian operations with
1774/// two different inputs (0), either-endian operations with two identical
1775/// inputs (1), and little-endian operations with two different inputs (2).
1776/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1777bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1778 SelectionDAG &DAG) {
1779 bool IsLE = DAG.getDataLayout().isLittleEndian();
1780 if (ShuffleKind == 0) {
1781 if (IsLE)
1782 return false;
1783 for (unsigned i = 0; i != 16; ++i)
1784 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+1))
1785 return false;
1786 } else if (ShuffleKind == 2) {
1787 if (!IsLE)
1788 return false;
1789 for (unsigned i = 0; i != 16; ++i)
1790 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2))
1791 return false;
1792 } else if (ShuffleKind == 1) {
1793 unsigned j = IsLE ? 0 : 1;
1794 for (unsigned i = 0; i != 8; ++i)
1795 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+j) ||
1796 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j))
1797 return false;
1798 }
1799 return true;
1800}
1801
1802/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
1803/// VPKUWUM instruction.
1804/// The ShuffleKind distinguishes between big-endian operations with
1805/// two different inputs (0), either-endian operations with two identical
1806/// inputs (1), and little-endian operations with two different inputs (2).
1807/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1808bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1809 SelectionDAG &DAG) {
1810 bool IsLE = DAG.getDataLayout().isLittleEndian();
1811 if (ShuffleKind == 0) {
1812 if (IsLE)
1813 return false;
1814 for (unsigned i = 0; i != 16; i += 2)
1815 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+2) ||
1816 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+3))
1817 return false;
1818 } else if (ShuffleKind == 2) {
1819 if (!IsLE)
1820 return false;
1821 for (unsigned i = 0; i != 16; i += 2)
1822 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1823 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1))
1824 return false;
1825 } else if (ShuffleKind == 1) {
1826 unsigned j = IsLE ? 0 : 2;
1827 for (unsigned i = 0; i != 8; i += 2)
1828 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1829 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1830 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1831 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1))
1832 return false;
1833 }
1834 return true;
1835}
1836
1837/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
1838/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the
1839/// current subtarget.
1840///
1841/// The ShuffleKind distinguishes between big-endian operations with
1842/// two different inputs (0), either-endian operations with two identical
1843/// inputs (1), and little-endian operations with two different inputs (2).
1844/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1845bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1846 SelectionDAG &DAG) {
1847 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
1848 if (!Subtarget.hasP8Vector())
1849 return false;
1850
1851 bool IsLE = DAG.getDataLayout().isLittleEndian();
1852 if (ShuffleKind == 0) {
1853 if (IsLE)
1854 return false;
1855 for (unsigned i = 0; i != 16; i += 4)
1856 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+4) ||
1857 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+5) ||
1858 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+6) ||
1859 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+7))
1860 return false;
1861 } else if (ShuffleKind == 2) {
1862 if (!IsLE)
1863 return false;
1864 for (unsigned i = 0; i != 16; i += 4)
1865 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1866 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1) ||
1867 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+2) ||
1868 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+3))
1869 return false;
1870 } else if (ShuffleKind == 1) {
1871 unsigned j = IsLE ? 0 : 4;
1872 for (unsigned i = 0; i != 8; i += 4)
1873 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1874 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1875 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+j+2) ||
1876 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+j+3) ||
1877 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1878 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1) ||
1879 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+10), Val: i*2+j+2) ||
1880 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+11), Val: i*2+j+3))
1881 return false;
1882 }
1883 return true;
1884}
1885
1886/// isVMerge - Common function, used to match vmrg* shuffles.
1887///
1888static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
1889 unsigned LHSStart, unsigned RHSStart) {
1890 if (N->getValueType(ResNo: 0) != MVT::v16i8)
1891 return false;
1892 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&
1893 "Unsupported merge size!");
1894
1895 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units
1896 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit
1897 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+j),
1898 Val: LHSStart+j+i*UnitSize) ||
1899 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+UnitSize+j),
1900 Val: RHSStart+j+i*UnitSize))
1901 return false;
1902 }
1903 return true;
1904}
1905
1906/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
1907/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes).
1908/// The ShuffleKind distinguishes between big-endian merges with two
1909/// different inputs (0), either-endian merges with two identical inputs (1),
1910/// and little-endian merges with two different inputs (2). For the latter,
1911/// the input operands are swapped (see PPCInstrAltivec.td).
1912bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1913 unsigned ShuffleKind, SelectionDAG &DAG) {
1914 if (DAG.getDataLayout().isLittleEndian()) {
1915 if (ShuffleKind == 1) // unary
1916 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1917 else if (ShuffleKind == 2) // swapped
1918 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1919 else
1920 return false;
1921 } else {
1922 if (ShuffleKind == 1) // unary
1923 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1924 else if (ShuffleKind == 0) // normal
1925 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1926 else
1927 return false;
1928 }
1929}
1930
1931/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
1932/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes).
1933/// The ShuffleKind distinguishes between big-endian merges with two
1934/// different inputs (0), either-endian merges with two identical inputs (1),
1935/// and little-endian merges with two different inputs (2). For the latter,
1936/// the input operands are swapped (see PPCInstrAltivec.td).
1937bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1938 unsigned ShuffleKind, SelectionDAG &DAG) {
1939 if (DAG.getDataLayout().isLittleEndian()) {
1940 if (ShuffleKind == 1) // unary
1941 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1942 else if (ShuffleKind == 2) // swapped
1943 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1944 else
1945 return false;
1946 } else {
1947 if (ShuffleKind == 1) // unary
1948 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1949 else if (ShuffleKind == 0) // normal
1950 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1951 else
1952 return false;
1953 }
1954}
1955
1956/**
1957 * Common function used to match vmrgew and vmrgow shuffles
1958 *
1959 * The indexOffset determines whether to look for even or odd words in
1960 * the shuffle mask. This is based on the of the endianness of the target
1961 * machine.
1962 * - Little Endian:
1963 * - Use offset of 0 to check for odd elements
1964 * - Use offset of 4 to check for even elements
1965 * - Big Endian:
1966 * - Use offset of 0 to check for even elements
1967 * - Use offset of 4 to check for odd elements
1968 * A detailed description of the vector element ordering for little endian and
1969 * big endian can be found at
1970 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html
1971 * Targeting your applications - what little endian and big endian IBM XL C/C++
1972 * compiler differences mean to you
1973 *
1974 * The mask to the shuffle vector instruction specifies the indices of the
1975 * elements from the two input vectors to place in the result. The elements are
1976 * numbered in array-access order, starting with the first vector. These vectors
1977 * are always of type v16i8, thus each vector will contain 16 elements of size
1978 * 8. More info on the shuffle vector can be found in the
1979 * http://llvm.org/docs/LangRef.html#shufflevector-instruction
1980 * Language Reference.
1981 *
1982 * The RHSStartValue indicates whether the same input vectors are used (unary)
1983 * or two different input vectors are used, based on the following:
1984 * - If the instruction uses the same vector for both inputs, the range of the
1985 * indices will be 0 to 15. In this case, the RHSStart value passed should
1986 * be 0.
1987 * - If the instruction has two different vectors then the range of the
1988 * indices will be 0 to 31. In this case, the RHSStart value passed should
1989 * be 16 (indices 0-15 specify elements in the first vector while indices 16
1990 * to 31 specify elements in the second vector).
1991 *
1992 * \param[in] N The shuffle vector SD Node to analyze
1993 * \param[in] IndexOffset Specifies whether to look for even or odd elements
1994 * \param[in] RHSStartValue Specifies the starting index for the righthand input
1995 * vector to the shuffle_vector instruction
1996 * \return true iff this shuffle vector represents an even or odd word merge
1997 */
1998static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset,
1999 unsigned RHSStartValue) {
2000 if (N->getValueType(ResNo: 0) != MVT::v16i8)
2001 return false;
2002
2003 for (unsigned i = 0; i < 2; ++i)
2004 for (unsigned j = 0; j < 4; ++j)
2005 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j),
2006 Val: i*RHSStartValue+j+IndexOffset) ||
2007 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j+8),
2008 Val: i*RHSStartValue+j+IndexOffset+8))
2009 return false;
2010 return true;
2011}
2012
2013/**
2014 * Determine if the specified shuffle mask is suitable for the vmrgew or
2015 * vmrgow instructions.
2016 *
2017 * \param[in] N The shuffle vector SD Node to analyze
2018 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false)
2019 * \param[in] ShuffleKind Identify the type of merge:
2020 * - 0 = big-endian merge with two different inputs;
2021 * - 1 = either-endian merge with two identical inputs;
2022 * - 2 = little-endian merge with two different inputs (inputs are swapped for
2023 * little-endian merges).
2024 * \param[in] DAG The current SelectionDAG
2025 * \return true iff this shuffle mask
2026 */
2027bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
2028 unsigned ShuffleKind, SelectionDAG &DAG) {
2029 if (DAG.getDataLayout().isLittleEndian()) {
2030 unsigned indexOffset = CheckEven ? 4 : 0;
2031 if (ShuffleKind == 1) // Unary
2032 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2033 else if (ShuffleKind == 2) // swapped
2034 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2035 else
2036 return false;
2037 }
2038 else {
2039 unsigned indexOffset = CheckEven ? 0 : 4;
2040 if (ShuffleKind == 1) // Unary
2041 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2042 else if (ShuffleKind == 0) // Normal
2043 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2044 else
2045 return false;
2046 }
2047 return false;
2048}
2049
2050/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
2051/// amount, otherwise return -1.
2052/// The ShuffleKind distinguishes between big-endian operations with two
2053/// different inputs (0), either-endian operations with two identical inputs
2054/// (1), and little-endian operations with two different inputs (2). For the
2055/// latter, the input operands are swapped (see PPCInstrAltivec.td).
2056int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
2057 SelectionDAG &DAG) {
2058 if (N->getValueType(ResNo: 0) != MVT::v16i8)
2059 return -1;
2060
2061 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2062
2063 // Find the first non-undef value in the shuffle mask.
2064 unsigned i;
2065 for (i = 0; i != 16 && SVOp->getMaskElt(Idx: i) < 0; ++i)
2066 /*search*/;
2067
2068 if (i == 16) return -1; // all undef.
2069
2070 // Otherwise, check to see if the rest of the elements are consecutively
2071 // numbered from this value.
2072 unsigned ShiftAmt = SVOp->getMaskElt(Idx: i);
2073 if (ShiftAmt < i) return -1;
2074
2075 ShiftAmt -= i;
2076 bool isLE = DAG.getDataLayout().isLittleEndian();
2077
2078 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
2079 // Check the rest of the elements to see if they are consecutive.
2080 for (++i; i != 16; ++i)
2081 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: ShiftAmt+i))
2082 return -1;
2083 } else if (ShuffleKind == 1) {
2084 // Check the rest of the elements to see if they are consecutive.
2085 for (++i; i != 16; ++i)
2086 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: (ShiftAmt+i) & 15))
2087 return -1;
2088 } else
2089 return -1;
2090
2091 if (isLE)
2092 ShiftAmt = 16 - ShiftAmt;
2093
2094 return ShiftAmt;
2095}
2096
2097/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
2098/// specifies a splat of a single element that is suitable for input to
2099/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.).
2100bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
2101 EVT VT = N->getValueType(ResNo: 0);
2102 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2103 return EltSize == 8 && N->getMaskElt(Idx: 0) == N->getMaskElt(Idx: 1);
2104
2105 assert(VT == MVT::v16i8 && isPowerOf2_32(EltSize) &&
2106 EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes");
2107
2108 // The consecutive indices need to specify an element, not part of two
2109 // different elements. So abandon ship early if this isn't the case.
2110 if (N->getMaskElt(Idx: 0) % EltSize != 0)
2111 return false;
2112
2113 // This is a splat operation if each element of the permute is the same, and
2114 // if the value doesn't reference the second vector.
2115 unsigned ElementBase = N->getMaskElt(Idx: 0);
2116
2117 // FIXME: Handle UNDEF elements too!
2118 if (ElementBase >= 16)
2119 return false;
2120
2121 // Check that the indices are consecutive, in the case of a multi-byte element
2122 // splatted with a v16i8 mask.
2123 for (unsigned i = 1; i != EltSize; ++i)
2124 if (N->getMaskElt(Idx: i) < 0 || N->getMaskElt(Idx: i) != (int)(i+ElementBase))
2125 return false;
2126
2127 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
2128 // An UNDEF element is a sequence of UNDEF bytes.
2129 if (N->getMaskElt(Idx: i) < 0) {
2130 for (unsigned j = 1; j != EltSize; ++j)
2131 if (N->getMaskElt(Idx: i + j) >= 0)
2132 return false;
2133 } else
2134 for (unsigned j = 0; j != EltSize; ++j)
2135 if (N->getMaskElt(Idx: i + j) != N->getMaskElt(Idx: j))
2136 return false;
2137 }
2138 return true;
2139}
2140
2141/// Check that the mask is shuffling N byte elements. Within each N byte
2142/// element of the mask, the indices could be either in increasing or
2143/// decreasing order as long as they are consecutive.
2144/// \param[in] N the shuffle vector SD Node to analyze
2145/// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/
2146/// Word/DoubleWord/QuadWord).
2147/// \param[in] StepLen the delta indices number among the N byte element, if
2148/// the mask is in increasing/decreasing order then it is 1/-1.
2149/// \return true iff the mask is shuffling N byte elements.
2150static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width,
2151 int StepLen) {
2152 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) &&
2153 "Unexpected element width.");
2154 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width.");
2155
2156 unsigned NumOfElem = 16 / Width;
2157 unsigned MaskVal[16]; // Width is never greater than 16
2158 for (unsigned i = 0; i < NumOfElem; ++i) {
2159 MaskVal[0] = N->getMaskElt(Idx: i * Width);
2160 if ((StepLen == 1) && (MaskVal[0] % Width)) {
2161 return false;
2162 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) {
2163 return false;
2164 }
2165
2166 for (unsigned int j = 1; j < Width; ++j) {
2167 MaskVal[j] = N->getMaskElt(Idx: i * Width + j);
2168 if (MaskVal[j] != MaskVal[j-1] + StepLen) {
2169 return false;
2170 }
2171 }
2172 }
2173
2174 return true;
2175}
2176
2177bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2178 unsigned &InsertAtByte, bool &Swap, bool IsLE) {
2179 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2180 return false;
2181
2182 // Now we look at mask elements 0,4,8,12
2183 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2184 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2185 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2186 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2187 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 };
2188 unsigned BigEndianShifts[] = { 3, 0, 1, 2 };
2189
2190 // Below, let H and L be arbitrary elements of the shuffle mask
2191 // where H is in the range [4,7] and L is in the range [0,3].
2192 // H, 1, 2, 3 or L, 5, 6, 7
2193 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) ||
2194 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) {
2195 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3];
2196 InsertAtByte = IsLE ? 12 : 0;
2197 Swap = M0 < 4;
2198 return true;
2199 }
2200 // 0, H, 2, 3 or 4, L, 6, 7
2201 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) ||
2202 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) {
2203 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3];
2204 InsertAtByte = IsLE ? 8 : 4;
2205 Swap = M1 < 4;
2206 return true;
2207 }
2208 // 0, 1, H, 3 or 4, 5, L, 7
2209 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) ||
2210 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) {
2211 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3];
2212 InsertAtByte = IsLE ? 4 : 8;
2213 Swap = M2 < 4;
2214 return true;
2215 }
2216 // 0, 1, 2, H or 4, 5, 6, L
2217 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) ||
2218 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) {
2219 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3];
2220 InsertAtByte = IsLE ? 0 : 12;
2221 Swap = M3 < 4;
2222 return true;
2223 }
2224
2225 // If both vector operands for the shuffle are the same vector, the mask will
2226 // contain only elements from the first one and the second one will be undef.
2227 if (N->getOperand(Num: 1).isUndef()) {
2228 ShiftElts = 0;
2229 Swap = true;
2230 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1;
2231 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) {
2232 InsertAtByte = IsLE ? 12 : 0;
2233 return true;
2234 }
2235 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) {
2236 InsertAtByte = IsLE ? 8 : 4;
2237 return true;
2238 }
2239 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) {
2240 InsertAtByte = IsLE ? 4 : 8;
2241 return true;
2242 }
2243 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) {
2244 InsertAtByte = IsLE ? 0 : 12;
2245 return true;
2246 }
2247 }
2248
2249 return false;
2250}
2251
2252bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2253 bool &Swap, bool IsLE) {
2254 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2255 // Ensure each byte index of the word is consecutive.
2256 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2257 return false;
2258
2259 // Now we look at mask elements 0,4,8,12, which are the beginning of words.
2260 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2261 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2262 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2263 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2264
2265 // If both vector operands for the shuffle are the same vector, the mask will
2266 // contain only elements from the first one and the second one will be undef.
2267 if (N->getOperand(Num: 1).isUndef()) {
2268 assert(M0 < 4 && "Indexing into an undef vector?");
2269 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4)
2270 return false;
2271
2272 ShiftElts = IsLE ? (4 - M0) % 4 : M0;
2273 Swap = false;
2274 return true;
2275 }
2276
2277 // Ensure each word index of the ShuffleVector Mask is consecutive.
2278 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8)
2279 return false;
2280
2281 if (IsLE) {
2282 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) {
2283 // Input vectors don't need to be swapped if the leading element
2284 // of the result is one of the 3 left elements of the second vector
2285 // (or if there is no shift to be done at all).
2286 Swap = false;
2287 ShiftElts = (8 - M0) % 8;
2288 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) {
2289 // Input vectors need to be swapped if the leading element
2290 // of the result is one of the 3 left elements of the first vector
2291 // (or if we're shifting by 4 - thereby simply swapping the vectors).
2292 Swap = true;
2293 ShiftElts = (4 - M0) % 4;
2294 }
2295
2296 return true;
2297 } else { // BE
2298 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) {
2299 // Input vectors don't need to be swapped if the leading element
2300 // of the result is one of the 4 elements of the first vector.
2301 Swap = false;
2302 ShiftElts = M0;
2303 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) {
2304 // Input vectors need to be swapped if the leading element
2305 // of the result is one of the 4 elements of the right vector.
2306 Swap = true;
2307 ShiftElts = M0 - 4;
2308 }
2309
2310 return true;
2311 }
2312}
2313
2314bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) {
2315 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2316
2317 if (!isNByteElemShuffleMask(N, Width, StepLen: -1))
2318 return false;
2319
2320 for (int i = 0; i < 16; i += Width)
2321 if (N->getMaskElt(Idx: i) != i + Width - 1)
2322 return false;
2323
2324 return true;
2325}
2326
2327bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) {
2328 return isXXBRShuffleMaskHelper(N, Width: 2);
2329}
2330
2331bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) {
2332 return isXXBRShuffleMaskHelper(N, Width: 4);
2333}
2334
2335bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) {
2336 return isXXBRShuffleMaskHelper(N, Width: 8);
2337}
2338
2339bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) {
2340 return isXXBRShuffleMaskHelper(N, Width: 16);
2341}
2342
2343/// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap
2344/// if the inputs to the instruction should be swapped and set \p DM to the
2345/// value for the immediate.
2346/// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI
2347/// AND element 0 of the result comes from the first input (LE) or second input
2348/// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered.
2349/// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle
2350/// mask.
2351bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM,
2352 bool &Swap, bool IsLE) {
2353 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2354
2355 // Ensure each byte index of the double word is consecutive.
2356 if (!isNByteElemShuffleMask(N, Width: 8, StepLen: 1))
2357 return false;
2358
2359 unsigned M0 = N->getMaskElt(Idx: 0) / 8;
2360 unsigned M1 = N->getMaskElt(Idx: 8) / 8;
2361 assert(((M0 | M1) < 4) && "A mask element out of bounds?");
2362
2363 // If both vector operands for the shuffle are the same vector, the mask will
2364 // contain only elements from the first one and the second one will be undef.
2365 if (N->getOperand(Num: 1).isUndef()) {
2366 if ((M0 | M1) < 2) {
2367 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1);
2368 Swap = false;
2369 return true;
2370 } else
2371 return false;
2372 }
2373
2374 if (IsLE) {
2375 if (M0 > 1 && M1 < 2) {
2376 Swap = false;
2377 } else if (M0 < 2 && M1 > 1) {
2378 M0 = (M0 + 2) % 4;
2379 M1 = (M1 + 2) % 4;
2380 Swap = true;
2381 } else
2382 return false;
2383
2384 // Note: if control flow comes here that means Swap is already set above
2385 DM = (((~M1) & 1) << 1) + ((~M0) & 1);
2386 return true;
2387 } else { // BE
2388 if (M0 < 2 && M1 > 1) {
2389 Swap = false;
2390 } else if (M0 > 1 && M1 < 2) {
2391 M0 = (M0 + 2) % 4;
2392 M1 = (M1 + 2) % 4;
2393 Swap = true;
2394 } else
2395 return false;
2396
2397 // Note: if control flow comes here that means Swap is already set above
2398 DM = (M0 << 1) + (M1 & 1);
2399 return true;
2400 }
2401}
2402
2403
2404/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
2405/// appropriate for PPC mnemonics (which have a big endian bias - namely
2406/// elements are counted from the left of the vector register).
2407unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
2408 SelectionDAG &DAG) {
2409 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2410 assert(isSplatShuffleMask(SVOp, EltSize));
2411 EVT VT = SVOp->getValueType(ResNo: 0);
2412
2413 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2414 return DAG.getDataLayout().isLittleEndian() ? 1 - SVOp->getMaskElt(Idx: 0)
2415 : SVOp->getMaskElt(Idx: 0);
2416
2417 if (DAG.getDataLayout().isLittleEndian())
2418 return (16 / EltSize) - 1 - (SVOp->getMaskElt(Idx: 0) / EltSize);
2419 else
2420 return SVOp->getMaskElt(Idx: 0) / EltSize;
2421}
2422
2423/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
2424/// by using a vspltis[bhw] instruction of the specified element size, return
2425/// the constant being splatted. The ByteSize field indicates the number of
2426/// bytes of each element [124] -> [bhw].
2427SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
2428 SDValue OpVal;
2429
2430 // If ByteSize of the splat is bigger than the element size of the
2431 // build_vector, then we have a case where we are checking for a splat where
2432 // multiple elements of the buildvector are folded together into a single
2433 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8).
2434 unsigned EltSize = 16/N->getNumOperands();
2435 if (EltSize < ByteSize) {
2436 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval.
2437 SDValue UniquedVals[4];
2438 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?");
2439
2440 // See if all of the elements in the buildvector agree across.
2441 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2442 if (N->getOperand(Num: i).isUndef()) continue;
2443 // If the element isn't a constant, bail fully out.
2444 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: i))) return SDValue();
2445
2446 if (!UniquedVals[i&(Multiple-1)].getNode())
2447 UniquedVals[i&(Multiple-1)] = N->getOperand(Num: i);
2448 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(Num: i))
2449 return SDValue(); // no match.
2450 }
2451
2452 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains
2453 // either constant or undef values that are identical for each chunk. See
2454 // if these chunks can form into a larger vspltis*.
2455
2456 // Check to see if all of the leading entries are either 0 or -1. If
2457 // neither, then this won't fit into the immediate field.
2458 bool LeadingZero = true;
2459 bool LeadingOnes = true;
2460 for (unsigned i = 0; i != Multiple-1; ++i) {
2461 if (!UniquedVals[i].getNode()) continue; // Must have been undefs.
2462
2463 LeadingZero &= isNullConstant(V: UniquedVals[i]);
2464 LeadingOnes &= isAllOnesConstant(V: UniquedVals[i]);
2465 }
2466 // Finally, check the least significant entry.
2467 if (LeadingZero) {
2468 if (!UniquedVals[Multiple-1].getNode())
2469 return DAG.getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32); // 0,0,0,undef
2470 int Val = UniquedVals[Multiple - 1]->getAsZExtVal();
2471 if (Val < 16) // 0,0,0,4 -> vspltisw(4)
2472 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2473 }
2474 if (LeadingOnes) {
2475 if (!UniquedVals[Multiple-1].getNode())
2476 return DAG.getTargetConstant(Val: ~0U, DL: SDLoc(N), VT: MVT::i32); // -1,-1,-1,undef
2477 int Val =cast<ConstantSDNode>(Val&: UniquedVals[Multiple-1])->getSExtValue();
2478 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2)
2479 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2480 }
2481
2482 return SDValue();
2483 }
2484
2485 // Check to see if this buildvec has a single non-undef value in its elements.
2486 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2487 if (N->getOperand(Num: i).isUndef()) continue;
2488 if (!OpVal.getNode())
2489 OpVal = N->getOperand(Num: i);
2490 else if (OpVal != N->getOperand(Num: i))
2491 return SDValue();
2492 }
2493
2494 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def.
2495
2496 unsigned ValSizeInBytes = EltSize;
2497 uint64_t Value = 0;
2498 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: OpVal)) {
2499 Value = CN->getZExtValue();
2500 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal)) {
2501 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!");
2502 Value = llvm::bit_cast<uint32_t>(from: CN->getValueAPF().convertToFloat());
2503 }
2504
2505 // If the splat value is larger than the element value, then we can never do
2506 // this splat. The only case that we could fit the replicated bits into our
2507 // immediate field for would be zero, and we prefer to use vxor for it.
2508 if (ValSizeInBytes < ByteSize) return SDValue();
2509
2510 // If the element value is larger than the splat value, check if it consists
2511 // of a repeated bit pattern of size ByteSize.
2512 if (!APInt(ValSizeInBytes * 8, Value).isSplat(SplatSizeInBits: ByteSize * 8))
2513 return SDValue();
2514
2515 // Properly sign extend the value.
2516 int MaskVal = SignExtend32(X: Value, B: ByteSize * 8);
2517
2518 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
2519 if (MaskVal == 0) return SDValue();
2520
2521 // Finally, if this value fits in a 5 bit sext field, return it
2522 if (SignExtend32<5>(X: MaskVal) == MaskVal)
2523 return DAG.getSignedTargetConstant(Val: MaskVal, DL: SDLoc(N), VT: MVT::i32);
2524 return SDValue();
2525}
2526
2527//===----------------------------------------------------------------------===//
2528// Addressing Mode Selection
2529//===----------------------------------------------------------------------===//
2530
2531/// isIntS16Immediate - This method tests to see if the node is either a 32-bit
2532/// or 64-bit immediate, and if the value can be accurately represented as a
2533/// sign extension from a 16-bit value. If so, this returns true and the
2534/// immediate.
2535bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) {
2536 if (!isa<ConstantSDNode>(Val: N))
2537 return false;
2538
2539 Imm = (int16_t)N->getAsZExtVal();
2540 if (N->getValueType(ResNo: 0) == MVT::i32)
2541 return Imm == (int32_t)N->getAsZExtVal();
2542 else
2543 return Imm == (int64_t)N->getAsZExtVal();
2544}
2545bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) {
2546 return isIntS16Immediate(N: Op.getNode(), Imm);
2547}
2548
2549/// Used when computing address flags for selecting loads and stores.
2550/// If we have an OR, check if the LHS and RHS are provably disjoint.
2551/// An OR of two provably disjoint values is equivalent to an ADD.
2552/// Most PPC load/store instructions compute the effective address as a sum,
2553/// so doing this conversion is useful.
2554static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) {
2555 if (N.getOpcode() != ISD::OR)
2556 return false;
2557 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2558 if (!LHSKnown.Zero.getBoolValue())
2559 return false;
2560 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2561 return (~(LHSKnown.Zero | RHSKnown.Zero) == 0);
2562}
2563
2564/// SelectAddressEVXRegReg - Given the specified address, check to see if it can
2565/// be represented as an indexed [r+r] operation.
2566bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
2567 SDValue &Index,
2568 SelectionDAG &DAG) const {
2569 for (SDNode *U : N->users()) {
2570 if (MemSDNode *Memop = dyn_cast<MemSDNode>(Val: U)) {
2571 if (Memop->getMemoryVT() == MVT::f64) {
2572 Base = N.getOperand(i: 0);
2573 Index = N.getOperand(i: 1);
2574 return true;
2575 }
2576 }
2577 }
2578 return false;
2579}
2580
2581/// isIntS34Immediate - This method tests if value of node given can be
2582/// accurately represented as a sign extension from a 34-bit value. If so,
2583/// this returns true and the immediate.
2584bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) {
2585 if (!isa<ConstantSDNode>(Val: N))
2586 return false;
2587
2588 Imm = cast<ConstantSDNode>(Val: N)->getSExtValue();
2589 return isInt<34>(x: Imm);
2590}
2591bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) {
2592 return isIntS34Immediate(N: Op.getNode(), Imm);
2593}
2594
2595/// SelectAddressRegReg - Given the specified addressed, check to see if it
2596/// can be represented as an indexed [r+r] operation. Returns false if it
2597/// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is
2598/// non-zero and N can be represented by a base register plus a signed 16-bit
2599/// displacement, make a more precise judgement by checking (displacement % \p
2600/// EncodingAlignment).
2601bool PPCTargetLowering::SelectAddressRegReg(
2602 SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG,
2603 MaybeAlign EncodingAlignment) const {
2604 // If we have a PC Relative target flag don't select as [reg+reg]. It will be
2605 // a [pc+imm].
2606 if (SelectAddressPCRel(N, Base))
2607 return false;
2608
2609 int16_t Imm = 0;
2610 if (N.getOpcode() == ISD::ADD) {
2611 // Is there any SPE load/store (f64), which can't handle 16bit offset?
2612 // SPE load/store can only handle 8-bit offsets.
2613 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG))
2614 return true;
2615 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2616 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2617 return false; // r+i
2618 if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo)
2619 return false; // r+i
2620
2621 Base = N.getOperand(i: 0);
2622 Index = N.getOperand(i: 1);
2623 return true;
2624 } else if (N.getOpcode() == ISD::OR) {
2625 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2626 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2627 return false; // r+i can fold it if we can.
2628
2629 // If this is an or of disjoint bitfields, we can codegen this as an add
2630 // (for better address arithmetic) if the LHS and RHS of the OR are provably
2631 // disjoint.
2632 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2633
2634 if (LHSKnown.Zero.getBoolValue()) {
2635 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2636 // If all of the bits are known zero on the LHS or RHS, the add won't
2637 // carry.
2638 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) {
2639 Base = N.getOperand(i: 0);
2640 Index = N.getOperand(i: 1);
2641 return true;
2642 }
2643 }
2644 }
2645
2646 return false;
2647}
2648
2649// If we happen to be doing an i64 load or store into a stack slot that has
2650// less than a 4-byte alignment, then the frame-index elimination may need to
2651// use an indexed load or store instruction (because the offset may not be a
2652// multiple of 4). The extra register needed to hold the offset comes from the
2653// register scavenger, and it is possible that the scavenger will need to use
2654// an emergency spill slot. As a result, we need to make sure that a spill slot
2655// is allocated when doing an i64 load/store into a less-than-4-byte-aligned
2656// stack slot.
2657static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
2658 // FIXME: This does not handle the LWA case.
2659 if (VT != MVT::i64)
2660 return;
2661
2662 // NOTE: We'll exclude negative FIs here, which come from argument
2663 // lowering, because there are no known test cases triggering this problem
2664 // using packed structures (or similar). We can remove this exclusion if
2665 // we find such a test case. The reason why this is so test-case driven is
2666 // because this entire 'fixup' is only to prevent crashes (from the
2667 // register scavenger) on not-really-valid inputs. For example, if we have:
2668 // %a = alloca i1
2669 // %b = bitcast i1* %a to i64*
2670 // store i64* a, i64 b
2671 // then the store should really be marked as 'align 1', but is not. If it
2672 // were marked as 'align 1' then the indexed form would have been
2673 // instruction-selected initially, and the problem this 'fixup' is preventing
2674 // won't happen regardless.
2675 if (FrameIdx < 0)
2676 return;
2677
2678 MachineFunction &MF = DAG.getMachineFunction();
2679 MachineFrameInfo &MFI = MF.getFrameInfo();
2680
2681 if (MFI.getObjectAlign(ObjectIdx: FrameIdx) >= Align(4))
2682 return;
2683
2684 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2685 FuncInfo->setHasNonRISpills();
2686}
2687
2688/// Returns true if the address N can be represented by a base register plus
2689/// a signed 16-bit displacement [r+imm], and if it is not better
2690/// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept
2691/// displacements that are multiples of that value.
2692bool PPCTargetLowering::SelectAddressRegImm(
2693 SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG,
2694 MaybeAlign EncodingAlignment) const {
2695 // FIXME dl should come from parent load or store, not from address
2696 SDLoc dl(N);
2697
2698 // If we have a PC Relative target flag don't select as [reg+imm]. It will be
2699 // a [pc+imm].
2700 if (SelectAddressPCRel(N, Base))
2701 return false;
2702
2703 // If this can be more profitably realized as r+r, fail.
2704 if (SelectAddressRegReg(N, Base&: Disp, Index&: Base, DAG, EncodingAlignment))
2705 return false;
2706
2707 if (N.getOpcode() == ISD::ADD) {
2708 int16_t imm = 0;
2709 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2710 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2711 Disp = DAG.getSignedTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2712 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2713 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2714 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2715 } else {
2716 Base = N.getOperand(i: 0);
2717 }
2718 return true; // [r+i]
2719 } else if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo) {
2720 // Match LOAD (ADD (X, Lo(G))).
2721 assert(!N.getOperand(1).getConstantOperandVal(1) &&
2722 "Cannot handle constant offsets yet!");
2723 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
2724 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
2725 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
2726 Disp.getOpcode() == ISD::TargetConstantPool ||
2727 Disp.getOpcode() == ISD::TargetJumpTable);
2728 Base = N.getOperand(i: 0);
2729 return true; // [&g+r]
2730 }
2731 } else if (N.getOpcode() == ISD::OR) {
2732 int16_t imm = 0;
2733 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2734 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2735 // If this is an or of disjoint bitfields, we can codegen this as an add
2736 // (for better address arithmetic) if the LHS and RHS of the OR are
2737 // provably disjoint.
2738 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2739
2740 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
2741 // If all of the bits are known zero on the LHS or RHS, the add won't
2742 // carry.
2743 if (FrameIndexSDNode *FI =
2744 dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2745 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2746 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2747 } else {
2748 Base = N.getOperand(i: 0);
2749 }
2750 Disp = DAG.getTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2751 return true;
2752 }
2753 }
2754 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
2755 // Loading from a constant address.
2756
2757 // If this address fits entirely in a 16-bit sext immediate field, codegen
2758 // this as "d, 0"
2759 int16_t Imm;
2760 if (isIntS16Immediate(N: CN, Imm) &&
2761 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm))) {
2762 Disp = DAG.getTargetConstant(Val: Imm, DL: dl, VT: CN->getValueType(ResNo: 0));
2763 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2764 VT: CN->getValueType(ResNo: 0));
2765 return true;
2766 }
2767
2768 // Handle 32-bit sext immediates with LIS + addr mode.
2769 if ((CN->getValueType(ResNo: 0) == MVT::i32 ||
2770 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
2771 (!EncodingAlignment ||
2772 isAligned(Lhs: *EncodingAlignment, SizeInBytes: CN->getZExtValue()))) {
2773 int Addr = (int)CN->getZExtValue();
2774
2775 // Otherwise, break this down into an LIS + disp.
2776 Disp = DAG.getTargetConstant(Val: (short)Addr, DL: dl, VT: MVT::i32);
2777
2778 Base = DAG.getTargetConstant(Val: (Addr - (signed short)Addr) >> 16, DL: dl,
2779 VT: MVT::i32);
2780 unsigned Opc = CN->getValueType(ResNo: 0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
2781 Base = SDValue(DAG.getMachineNode(Opcode: Opc, dl, VT: CN->getValueType(ResNo: 0), Op1: Base), 0);
2782 return true;
2783 }
2784 }
2785
2786 Disp = DAG.getTargetConstant(Val: 0, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()));
2787 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
2788 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2789 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2790 } else
2791 Base = N;
2792 return true; // [r+0]
2793}
2794
2795/// Similar to the 16-bit case but for instructions that take a 34-bit
2796/// displacement field (prefixed loads/stores).
2797bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
2798 SDValue &Base,
2799 SelectionDAG &DAG) const {
2800 // Only on 64-bit targets.
2801 if (N.getValueType() != MVT::i64)
2802 return false;
2803
2804 SDLoc dl(N);
2805 int64_t Imm = 0;
2806
2807 if (N.getOpcode() == ISD::ADD) {
2808 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2809 return false;
2810 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2811 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2812 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2813 else
2814 Base = N.getOperand(i: 0);
2815 return true;
2816 }
2817
2818 if (N.getOpcode() == ISD::OR) {
2819 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2820 return false;
2821 // If this is an or of disjoint bitfields, we can codegen this as an add
2822 // (for better address arithmetic) if the LHS and RHS of the OR are
2823 // provably disjoint.
2824 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2825 if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL)
2826 return false;
2827 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2828 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2829 else
2830 Base = N.getOperand(i: 0);
2831 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2832 return true;
2833 }
2834
2835 if (isIntS34Immediate(Op: N, Imm)) { // If the address is a 34-bit const.
2836 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2837 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
2838 return true;
2839 }
2840
2841 return false;
2842}
2843
2844/// SelectAddressRegRegOnly - Given the specified addressed, force it to be
2845/// represented as an indexed [r+r] operation.
2846bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
2847 SDValue &Index,
2848 SelectionDAG &DAG) const {
2849 // Check to see if we can easily represent this as an [r+r] address. This
2850 // will fail if it thinks that the address is more profitably represented as
2851 // reg+imm, e.g. where imm = 0.
2852 if (SelectAddressRegReg(N, Base, Index, DAG))
2853 return true;
2854
2855 // If the address is the result of an add, we will utilize the fact that the
2856 // address calculation includes an implicit add. However, we can reduce
2857 // register pressure if we do not materialize a constant just for use as the
2858 // index register. We only get rid of the add if it is not an add of a
2859 // value and a 16-bit signed constant and both have a single use.
2860 int16_t imm = 0;
2861 if (N.getOpcode() == ISD::ADD &&
2862 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) ||
2863 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
2864 Base = N.getOperand(i: 0);
2865 Index = N.getOperand(i: 1);
2866 return true;
2867 }
2868
2869 // Otherwise, do it the hard way, using R0 as the base register.
2870 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2871 VT: N.getValueType());
2872 Index = N;
2873 return true;
2874}
2875
2876template <typename Ty> static bool isValidPCRelNode(SDValue N) {
2877 Ty *PCRelCand = dyn_cast<Ty>(N);
2878 return PCRelCand && (PPCInstrInfo::hasPCRelFlag(TF: PCRelCand->getTargetFlags()));
2879}
2880
2881/// Returns true if this address is a PC Relative address.
2882/// PC Relative addresses are marked with the flag PPCII::MO_PCREL_FLAG
2883/// or if the node opcode is PPCISD::MAT_PCREL_ADDR.
2884bool PPCTargetLowering::SelectAddressPCRel(SDValue N, SDValue &Base) const {
2885 // This is a materialize PC Relative node. Always select this as PC Relative.
2886 Base = N;
2887 if (N.getOpcode() == PPCISD::MAT_PCREL_ADDR)
2888 return true;
2889 if (isValidPCRelNode<ConstantPoolSDNode>(N) ||
2890 isValidPCRelNode<GlobalAddressSDNode>(N) ||
2891 isValidPCRelNode<JumpTableSDNode>(N) ||
2892 isValidPCRelNode<BlockAddressSDNode>(N))
2893 return true;
2894 return false;
2895}
2896
2897/// Returns true if we should use a direct load into vector instruction
2898/// (such as lxsd or lfd), instead of a load into gpr + direct move sequence.
2899static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) {
2900
2901 // If there are any other uses other than scalar to vector, then we should
2902 // keep it as a scalar load -> direct move pattern to prevent multiple
2903 // loads.
2904 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N);
2905 if (!LD)
2906 return false;
2907
2908 EVT MemVT = LD->getMemoryVT();
2909 if (!MemVT.isSimple())
2910 return false;
2911 switch(MemVT.getSimpleVT().SimpleTy) {
2912 case MVT::i64:
2913 break;
2914 case MVT::i32:
2915 if (!ST.hasP8Vector())
2916 return false;
2917 break;
2918 case MVT::i16:
2919 case MVT::i8:
2920 if (!ST.hasP9Vector())
2921 return false;
2922 break;
2923 default:
2924 return false;
2925 }
2926
2927 SDValue LoadedVal(N, 0);
2928 if (!LoadedVal.hasOneUse())
2929 return false;
2930
2931 for (SDUse &Use : LD->uses())
2932 if (Use.getResNo() == 0 &&
2933 Use.getUser()->getOpcode() != ISD::SCALAR_TO_VECTOR &&
2934 Use.getUser()->getOpcode() != PPCISD::SCALAR_TO_VECTOR_PERMUTED)
2935 return false;
2936
2937 return true;
2938}
2939
2940/// getPreIndexedAddressParts - returns true by value, base pointer and
2941/// offset pointer and addressing mode by reference if the node's address
2942/// can be legally represented as pre-indexed load / store address.
2943bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
2944 SDValue &Offset,
2945 ISD::MemIndexedMode &AM,
2946 SelectionDAG &DAG) const {
2947 if (!EnablePPCPreinc)
2948 return false;
2949
2950 bool isLoad = true;
2951 SDValue Ptr;
2952 EVT VT;
2953 Align Alignment;
2954 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
2955 Ptr = LD->getBasePtr();
2956 VT = LD->getMemoryVT();
2957 Alignment = LD->getAlign();
2958 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
2959 Ptr = ST->getBasePtr();
2960 VT = ST->getMemoryVT();
2961 Alignment = ST->getAlign();
2962 isLoad = false;
2963 } else
2964 return false;
2965
2966 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector
2967 // instructions because we can fold these into a more efficient instruction
2968 // instead, (such as LXSD).
2969 if (isLoad && usePartialVectorLoads(N, ST: Subtarget)) {
2970 return false;
2971 }
2972
2973 // PowerPC doesn't have preinc load/store instructions for vectors
2974 if (VT.isVector())
2975 return false;
2976
2977 if (SelectAddressRegReg(N: Ptr, Base, Index&: Offset, DAG)) {
2978 // Common code will reject creating a pre-inc form if the base pointer
2979 // is a frame index, or if N is a store and the base pointer is either
2980 // the same as or a predecessor of the value being stored. Check for
2981 // those situations here, and try with swapped Base/Offset instead.
2982 bool Swap = false;
2983
2984 if (isa<FrameIndexSDNode>(Val: Base) || isa<RegisterSDNode>(Val: Base))
2985 Swap = true;
2986 else if (!isLoad) {
2987 SDValue Val = cast<StoreSDNode>(Val: N)->getValue();
2988 if (Val == Base || Base.getNode()->isPredecessorOf(N: Val.getNode()))
2989 Swap = true;
2990 }
2991
2992 if (Swap)
2993 std::swap(a&: Base, b&: Offset);
2994
2995 AM = ISD::PRE_INC;
2996 return true;
2997 }
2998
2999 // LDU/STU can only handle immediates that are a multiple of 4.
3000 if (VT != MVT::i64) {
3001 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: std::nullopt))
3002 return false;
3003 } else {
3004 // LDU/STU need an address with at least 4-byte alignment.
3005 if (Alignment < Align(4))
3006 return false;
3007
3008 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: Align(4)))
3009 return false;
3010 }
3011
3012 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
3013 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of
3014 // sext i32 to i64 when addr mode is r+i.
3015 if (LD->getValueType(ResNo: 0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 &&
3016 LD->getExtensionType() == ISD::SEXTLOAD &&
3017 isa<ConstantSDNode>(Val: Offset))
3018 return false;
3019 }
3020
3021 AM = ISD::PRE_INC;
3022 return true;
3023}
3024
3025//===----------------------------------------------------------------------===//
3026// LowerOperation implementation
3027//===----------------------------------------------------------------------===//
3028
3029/// Return true if we should reference labels using a PICBase, set the HiOpFlags
3030/// and LoOpFlags to the target MO flags.
3031static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget,
3032 unsigned &HiOpFlags, unsigned &LoOpFlags,
3033 const GlobalValue *GV = nullptr) {
3034 HiOpFlags = PPCII::MO_HA;
3035 LoOpFlags = PPCII::MO_LO;
3036
3037 // Don't use the pic base if not in PIC relocation model.
3038 if (IsPIC) {
3039 HiOpFlags = PPCII::MO_PIC_HA_FLAG;
3040 LoOpFlags = PPCII::MO_PIC_LO_FLAG;
3041 }
3042}
3043
3044static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
3045 SelectionDAG &DAG) {
3046 SDLoc DL(HiPart);
3047 EVT PtrVT = HiPart.getValueType();
3048 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
3049
3050 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL, VT: PtrVT, N1: HiPart, N2: Zero);
3051 SDValue Lo = DAG.getNode(Opcode: PPCISD::Lo, DL, VT: PtrVT, N1: LoPart, N2: Zero);
3052
3053 // With PIC, the first instruction is actually "GR+hi(&G)".
3054 if (isPIC)
3055 Hi = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
3056 N1: DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL, VT: PtrVT), N2: Hi);
3057
3058 // Generate non-pic code that has direct accesses to the constant pool.
3059 // The address of the global is just (hi(&g)+lo(&g)).
3060 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Hi, N2: Lo);
3061}
3062
3063static void setUsesTOCBasePtr(MachineFunction &MF) {
3064 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3065 FuncInfo->setUsesTOCBasePtr();
3066}
3067
3068static void setUsesTOCBasePtr(SelectionDAG &DAG) {
3069 setUsesTOCBasePtr(DAG.getMachineFunction());
3070}
3071
3072SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
3073 SDValue GA) const {
3074 EVT VT = Subtarget.getScalarIntVT();
3075 SDValue Reg = Subtarget.isPPC64() ? DAG.getRegister(Reg: PPC::X2, VT)
3076 : Subtarget.isAIXABI()
3077 ? DAG.getRegister(Reg: PPC::R2, VT)
3078 : DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT);
3079 SDValue Ops[] = { GA, Reg };
3080 return DAG.getMemIntrinsicNode(
3081 Opcode: PPCISD::TOC_ENTRY, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT: VT,
3082 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: std::nullopt,
3083 Flags: MachineMemOperand::MOLoad);
3084}
3085
3086SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
3087 SelectionDAG &DAG) const {
3088 EVT PtrVT = Op.getValueType();
3089 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val&: Op);
3090 const Constant *C = CP->getConstVal();
3091
3092 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3093 // The actual address of the GlobalValue is stored in the TOC.
3094 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3095 if (Subtarget.isUsingPCRelativeCalls()) {
3096 SDLoc DL(CP);
3097 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3098 SDValue ConstPool = DAG.getTargetConstantPool(
3099 C, VT: Ty, Align: CP->getAlign(), Offset: CP->getOffset(), TargetFlags: PPCII::MO_PCREL_FLAG);
3100 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: ConstPool);
3101 }
3102 setUsesTOCBasePtr(DAG);
3103 SDValue GA = DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0);
3104 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3105 }
3106
3107 unsigned MOHiFlag, MOLoFlag;
3108 bool IsPIC = isPositionIndependent();
3109 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3110
3111 if (IsPIC && Subtarget.isSVR4ABI()) {
3112 SDValue GA =
3113 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: PPCII::MO_PIC_FLAG);
3114 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3115 }
3116
3117 SDValue CPIHi =
3118 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOHiFlag);
3119 SDValue CPILo =
3120 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOLoFlag);
3121 return LowerLabelRef(HiPart: CPIHi, LoPart: CPILo, isPIC: IsPIC, DAG);
3122}
3123
3124// For 64-bit PowerPC, prefer the more compact relative encodings.
3125// This trades 32 bits per jump table entry for one or two instructions
3126// on the jump site.
3127unsigned PPCTargetLowering::getJumpTableEncoding() const {
3128 if (isJumpTableRelative())
3129 return MachineJumpTableInfo::EK_LabelDifference32;
3130
3131 return TargetLowering::getJumpTableEncoding();
3132}
3133
3134bool PPCTargetLowering::isJumpTableRelative() const {
3135 if (UseAbsoluteJumpTables)
3136 return false;
3137 if (Subtarget.isPPC64() || Subtarget.isAIXABI())
3138 return true;
3139 return TargetLowering::isJumpTableRelative();
3140}
3141
3142SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3143 SelectionDAG &DAG) const {
3144 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3145 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3146
3147 switch (getTargetMachine().getCodeModel()) {
3148 case CodeModel::Small:
3149 case CodeModel::Medium:
3150 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3151 default:
3152 return DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: SDLoc(),
3153 VT: getPointerTy(DL: DAG.getDataLayout()));
3154 }
3155}
3156
3157const MCExpr *
3158PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
3159 unsigned JTI,
3160 MCContext &Ctx) const {
3161 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3162 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3163
3164 switch (getTargetMachine().getCodeModel()) {
3165 case CodeModel::Small:
3166 case CodeModel::Medium:
3167 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3168 default:
3169 return MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx);
3170 }
3171}
3172
3173SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
3174 EVT PtrVT = Op.getValueType();
3175 JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Op);
3176
3177 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3178 if (Subtarget.isUsingPCRelativeCalls()) {
3179 SDLoc DL(JT);
3180 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3181 SDValue GA =
3182 DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: Ty, TargetFlags: PPCII::MO_PCREL_FLAG);
3183 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3184 return MatAddr;
3185 }
3186
3187 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3188 // The actual address of the GlobalValue is stored in the TOC.
3189 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3190 setUsesTOCBasePtr(DAG);
3191 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
3192 return getTOCEntry(DAG, dl: SDLoc(JT), GA);
3193 }
3194
3195 unsigned MOHiFlag, MOLoFlag;
3196 bool IsPIC = isPositionIndependent();
3197 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3198
3199 if (IsPIC && Subtarget.isSVR4ABI()) {
3200 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT,
3201 TargetFlags: PPCII::MO_PIC_FLAG);
3202 return getTOCEntry(DAG, dl: SDLoc(GA), GA);
3203 }
3204
3205 SDValue JTIHi = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOHiFlag);
3206 SDValue JTILo = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOLoFlag);
3207 return LowerLabelRef(HiPart: JTIHi, LoPart: JTILo, isPIC: IsPIC, DAG);
3208}
3209
3210SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
3211 SelectionDAG &DAG) const {
3212 EVT PtrVT = Op.getValueType();
3213 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Val&: Op);
3214 const BlockAddress *BA = BASDN->getBlockAddress();
3215
3216 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3217 if (Subtarget.isUsingPCRelativeCalls()) {
3218 SDLoc DL(BASDN);
3219 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3220 SDValue GA = DAG.getTargetBlockAddress(BA, VT: Ty, Offset: BASDN->getOffset(),
3221 TargetFlags: PPCII::MO_PCREL_FLAG);
3222 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3223 return MatAddr;
3224 }
3225
3226 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3227 // The actual BlockAddress is stored in the TOC.
3228 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3229 setUsesTOCBasePtr(DAG);
3230 SDValue GA = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset());
3231 return getTOCEntry(DAG, dl: SDLoc(BASDN), GA);
3232 }
3233
3234 // 32-bit position-independent ELF stores the BlockAddress in the .got.
3235 if (Subtarget.is32BitELFABI() && isPositionIndependent())
3236 return getTOCEntry(
3237 DAG, dl: SDLoc(BASDN),
3238 GA: DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset()));
3239
3240 unsigned MOHiFlag, MOLoFlag;
3241 bool IsPIC = isPositionIndependent();
3242 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3243 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOHiFlag);
3244 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOLoFlag);
3245 return LowerLabelRef(HiPart: TgtBAHi, LoPart: TgtBALo, isPIC: IsPIC, DAG);
3246}
3247
3248SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
3249 SelectionDAG &DAG) const {
3250 if (Subtarget.isAIXABI())
3251 return LowerGlobalTLSAddressAIX(Op, DAG);
3252
3253 return LowerGlobalTLSAddressLinux(Op, DAG);
3254}
3255
3256/// updateForAIXShLibTLSModelOpt - Helper to initialize TLS model opt settings,
3257/// and then apply the update.
3258static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
3259 SelectionDAG &DAG,
3260 const TargetMachine &TM) {
3261 // Initialize TLS model opt setting lazily:
3262 // (1) Use initial-exec for single TLS var references within current function.
3263 // (2) Use local-dynamic for multiple TLS var references within current
3264 // function.
3265 PPCFunctionInfo *FuncInfo =
3266 DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
3267 if (!FuncInfo->isAIXFuncTLSModelOptInitDone()) {
3268 SmallPtrSet<const GlobalValue *, 8> TLSGV;
3269 // Iterate over all instructions within current function, collect all TLS
3270 // global variables (global variables taken as the first parameter to
3271 // Intrinsic::threadlocal_address).
3272 const Function &Func = DAG.getMachineFunction().getFunction();
3273 for (const BasicBlock &BB : Func)
3274 for (const Instruction &I : BB)
3275 if (I.getOpcode() == Instruction::Call)
3276 if (const CallInst *CI = dyn_cast<const CallInst>(Val: &I))
3277 if (Function *CF = CI->getCalledFunction())
3278 if (CF->isDeclaration() &&
3279 CF->getIntrinsicID() == Intrinsic::threadlocal_address)
3280 if (const GlobalValue *GV =
3281 dyn_cast<GlobalValue>(Val: I.getOperand(i: 0))) {
3282 TLSModel::Model GVModel = TM.getTLSModel(GV);
3283 if (GVModel == TLSModel::LocalDynamic)
3284 TLSGV.insert(Ptr: GV);
3285 }
3286
3287 unsigned TLSGVCnt = TLSGV.size();
3288 LLVM_DEBUG(dbgs() << format("LocalDynamic TLSGV count:%d\n", TLSGVCnt));
3289 if (TLSGVCnt <= PPCAIXTLSModelOptUseIEForLDLimit)
3290 FuncInfo->setAIXFuncUseTLSIEForLD();
3291 FuncInfo->setAIXFuncTLSModelOptInitDone();
3292 }
3293
3294 if (FuncInfo->isAIXFuncUseTLSIEForLD()) {
3295 LLVM_DEBUG(
3296 dbgs() << DAG.getMachineFunction().getName()
3297 << " function is using the TLS-IE model for TLS-LD access.\n");
3298 Model = TLSModel::InitialExec;
3299 }
3300}
3301
3302SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
3303 SelectionDAG &DAG) const {
3304 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3305
3306 if (DAG.getTarget().useEmulatedTLS())
3307 report_fatal_error(reason: "Emulated TLS is not yet supported on AIX");
3308
3309 SDLoc dl(GA);
3310 const GlobalValue *GV = GA->getGlobal();
3311 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3312 bool Is64Bit = Subtarget.isPPC64();
3313 TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
3314
3315 // Apply update to the TLS model.
3316 if (Subtarget.hasAIXShLibTLSModelOpt())
3317 updateForAIXShLibTLSModelOpt(Model, DAG, TM: getTargetMachine());
3318
3319 // TLS variables are accessed through TOC entries.
3320 // To support this, set the DAG to use the TOC base pointer.
3321 setUsesTOCBasePtr(DAG);
3322
3323 bool IsTLSLocalExecModel = Model == TLSModel::LocalExec;
3324
3325 if (IsTLSLocalExecModel || Model == TLSModel::InitialExec) {
3326 bool HasAIXSmallLocalExecTLS = Subtarget.hasAIXSmallLocalExecTLS();
3327 bool HasAIXSmallTLSGlobalAttr = false;
3328 SDValue VariableOffsetTGA =
3329 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TPREL_FLAG);
3330 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3331 SDValue TLSReg;
3332
3333 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(Val: GV))
3334 if (GVar->hasAttribute(Kind: "aix-small-tls"))
3335 HasAIXSmallTLSGlobalAttr = true;
3336
3337 if (Is64Bit) {
3338 // For local-exec and initial-exec on AIX (64-bit), the sequence generated
3339 // involves a load of the variable offset (from the TOC), followed by an
3340 // add of the loaded variable offset to R13 (the thread pointer).
3341 // This code sequence looks like:
3342 // ld reg1,var[TC](2)
3343 // add reg2, reg1, r13 // r13 contains the thread pointer
3344 TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3345
3346 // With the -maix-small-local-exec-tls option, or with the "aix-small-tls"
3347 // global variable attribute, produce a faster access sequence for
3348 // local-exec TLS variables where the offset from the TLS base is encoded
3349 // as an immediate operand.
3350 //
3351 // We only utilize the faster local-exec access sequence when the TLS
3352 // variable has a size within the policy limit. We treat types that are
3353 // not sized or are empty as being over the policy size limit.
3354 if ((HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr) &&
3355 IsTLSLocalExecModel) {
3356 Type *GVType = GV->getValueType();
3357 if (GVType->isSized() && !GVType->isEmptyTy() &&
3358 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3359 AIXSmallTlsPolicySizeLimit)
3360 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA, N2: TLSReg);
3361 }
3362 } else {
3363 // For local-exec and initial-exec on AIX (32-bit), the sequence generated
3364 // involves loading the variable offset from the TOC, generating a call to
3365 // .__get_tpointer to get the thread pointer (which will be in R3), and
3366 // adding the two together:
3367 // lwz reg1,var[TC](2)
3368 // bla .__get_tpointer
3369 // add reg2, reg1, r3
3370 TLSReg = DAG.getNode(Opcode: PPCISD::GET_TPOINTER, DL: dl, VT: PtrVT);
3371
3372 // We do not implement the 32-bit version of the faster access sequence
3373 // for local-exec that is controlled by the -maix-small-local-exec-tls
3374 // option, or the "aix-small-tls" global variable attribute.
3375 if (HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr)
3376 report_fatal_error(reason: "The small-local-exec TLS access sequence is "
3377 "currently only supported on AIX (64-bit mode).");
3378 }
3379 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: VariableOffset);
3380 }
3381
3382 if (Model == TLSModel::LocalDynamic) {
3383 bool HasAIXSmallLocalDynamicTLS = Subtarget.hasAIXSmallLocalDynamicTLS();
3384
3385 // We do not implement the 32-bit version of the faster access sequence
3386 // for local-dynamic that is controlled by -maix-small-local-dynamic-tls.
3387 if (!Is64Bit && HasAIXSmallLocalDynamicTLS)
3388 report_fatal_error(reason: "The small-local-dynamic TLS access sequence is "
3389 "currently only supported on AIX (64-bit mode).");
3390
3391 // For local-dynamic on AIX, we need to generate one TOC entry for each
3392 // variable offset, and a single module-handle TOC entry for the entire
3393 // file.
3394
3395 SDValue VariableOffsetTGA =
3396 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLD_FLAG);
3397 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3398
3399 Module *M = DAG.getMachineFunction().getFunction().getParent();
3400 GlobalVariable *TLSGV =
3401 dyn_cast_or_null<GlobalVariable>(Val: M->getOrInsertGlobal(
3402 Name: StringRef("_$TLSML"), Ty: PointerType::getUnqual(C&: *DAG.getContext())));
3403 TLSGV->setThreadLocalMode(GlobalVariable::LocalDynamicTLSModel);
3404 assert(TLSGV && "Not able to create GV for _$TLSML.");
3405 SDValue ModuleHandleTGA =
3406 DAG.getTargetGlobalAddress(GV: TLSGV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLDM_FLAG);
3407 SDValue ModuleHandleTOC = getTOCEntry(DAG, dl, GA: ModuleHandleTGA);
3408 SDValue ModuleHandle =
3409 DAG.getNode(Opcode: PPCISD::TLSLD_AIX, DL: dl, VT: PtrVT, Operand: ModuleHandleTOC);
3410
3411 // With the -maix-small-local-dynamic-tls option, produce a faster access
3412 // sequence for local-dynamic TLS variables where the offset from the
3413 // module-handle is encoded as an immediate operand.
3414 //
3415 // We only utilize the faster local-dynamic access sequence when the TLS
3416 // variable has a size within the policy limit. We treat types that are
3417 // not sized or are empty as being over the policy size limit.
3418 if (HasAIXSmallLocalDynamicTLS) {
3419 Type *GVType = GV->getValueType();
3420 if (GVType->isSized() && !GVType->isEmptyTy() &&
3421 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3422 AIXSmallTlsPolicySizeLimit)
3423 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA,
3424 N2: ModuleHandle);
3425 }
3426
3427 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ModuleHandle, N2: VariableOffset);
3428 }
3429
3430 // If Local- or Initial-exec or Local-dynamic is not possible or specified,
3431 // all GlobalTLSAddress nodes are lowered using the general-dynamic model. We
3432 // need to generate two TOC entries, one for the variable offset, one for the
3433 // region handle. The global address for the TOC entry of the region handle is
3434 // created with the MO_TLSGDM_FLAG flag and the global address for the TOC
3435 // entry of the variable offset is created with MO_TLSGD_FLAG.
3436 SDValue VariableOffsetTGA =
3437 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGD_FLAG);
3438 SDValue RegionHandleTGA =
3439 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGDM_FLAG);
3440 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3441 SDValue RegionHandle = getTOCEntry(DAG, dl, GA: RegionHandleTGA);
3442 return DAG.getNode(Opcode: PPCISD::TLSGD_AIX, DL: dl, VT: PtrVT, N1: VariableOffset,
3443 N2: RegionHandle);
3444}
3445
3446SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op,
3447 SelectionDAG &DAG) const {
3448 // FIXME: TLS addresses currently use medium model code sequences,
3449 // which is the most useful form. Eventually support for small and
3450 // large models could be added if users need it, at the cost of
3451 // additional complexity.
3452 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3453 if (DAG.getTarget().useEmulatedTLS())
3454 return LowerToTLSEmulatedModel(GA, DAG);
3455
3456 SDLoc dl(GA);
3457 const GlobalValue *GV = GA->getGlobal();
3458 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3459 bool is64bit = Subtarget.isPPC64();
3460 const Module *M = DAG.getMachineFunction().getFunction().getParent();
3461 PICLevel::Level picLevel = M->getPICLevel();
3462
3463 const TargetMachine &TM = getTargetMachine();
3464 TLSModel::Model Model = TM.getTLSModel(GV);
3465
3466 if (Model == TLSModel::LocalExec) {
3467 if (Subtarget.isUsingPCRelativeCalls()) {
3468 SDValue TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3469 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3470 TargetFlags: PPCII::MO_TPREL_PCREL_FLAG);
3471 SDValue MatAddr =
3472 DAG.getNode(Opcode: PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3473 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: MatAddr);
3474 }
3475
3476 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3477 TargetFlags: PPCII::MO_TPREL_HA);
3478 SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3479 TargetFlags: PPCII::MO_TPREL_LO);
3480 SDValue TLSReg = is64bit ? DAG.getRegister(Reg: PPC::X13, VT: MVT::i64)
3481 : DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
3482
3483 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL: dl, VT: PtrVT, N1: TGAHi, N2: TLSReg);
3484 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: TGALo, N2: Hi);
3485 }
3486
3487 if (Model == TLSModel::InitialExec) {
3488 bool IsPCRel = Subtarget.isUsingPCRelativeCalls();
3489 SDValue TGA = DAG.getTargetGlobalAddress(
3490 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0);
3491 SDValue TGATLS = DAG.getTargetGlobalAddress(
3492 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_TLS_PCREL_FLAG : PPCII::MO_TLS);
3493 SDValue TPOffset;
3494 if (IsPCRel) {
3495 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3496 TPOffset = DAG.getLoad(VT: MVT::i64, dl, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3497 PtrInfo: MachinePointerInfo());
3498 } else {
3499 SDValue GOTPtr;
3500 if (is64bit) {
3501 setUsesTOCBasePtr(DAG);
3502 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3503 GOTPtr =
3504 DAG.getNode(Opcode: PPCISD::ADDIS_GOT_TPREL_HA, DL: dl, VT: PtrVT, N1: GOTReg, N2: TGA);
3505 } else {
3506 if (!TM.isPositionIndependent())
3507 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_GOT, DL: dl, VT: PtrVT);
3508 else if (picLevel == PICLevel::SmallPIC)
3509 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3510 else
3511 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3512 }
3513 TPOffset = DAG.getNode(Opcode: PPCISD::LD_GOT_TPREL_L, DL: dl, VT: PtrVT, N1: TGA, N2: GOTPtr);
3514 }
3515 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TPOffset, N2: TGATLS);
3516 }
3517
3518 if (Model == TLSModel::GeneralDynamic) {
3519 if (Subtarget.isUsingPCRelativeCalls()) {
3520 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3521 TargetFlags: PPCII::MO_GOT_TLSGD_PCREL_FLAG);
3522 return DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3523 }
3524
3525 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3526 SDValue GOTPtr;
3527 if (is64bit) {
3528 setUsesTOCBasePtr(DAG);
3529 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3530 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSGD_HA, DL: dl, VT: PtrVT,
3531 N1: GOTReg, N2: TGA);
3532 } else {
3533 if (picLevel == PICLevel::SmallPIC)
3534 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3535 else
3536 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3537 }
3538 return DAG.getNode(Opcode: PPCISD::ADDI_TLSGD_L_ADDR, DL: dl, VT: PtrVT,
3539 N1: GOTPtr, N2: TGA, N3: TGA);
3540 }
3541
3542 if (Model == TLSModel::LocalDynamic) {
3543 if (Subtarget.isUsingPCRelativeCalls()) {
3544 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3545 TargetFlags: PPCII::MO_GOT_TLSLD_PCREL_FLAG);
3546 SDValue MatPCRel =
3547 DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3548 return DAG.getNode(Opcode: PPCISD::PADDI_DTPREL, DL: dl, VT: PtrVT, N1: MatPCRel, N2: TGA);
3549 }
3550
3551 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3552 SDValue GOTPtr;
3553 if (is64bit) {
3554 setUsesTOCBasePtr(DAG);
3555 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3556 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSLD_HA, DL: dl, VT: PtrVT,
3557 N1: GOTReg, N2: TGA);
3558 } else {
3559 if (picLevel == PICLevel::SmallPIC)
3560 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3561 else
3562 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3563 }
3564 SDValue TLSAddr = DAG.getNode(Opcode: PPCISD::ADDI_TLSLD_L_ADDR, DL: dl,
3565 VT: PtrVT, N1: GOTPtr, N2: TGA, N3: TGA);
3566 SDValue DtvOffsetHi = DAG.getNode(Opcode: PPCISD::ADDIS_DTPREL_HA, DL: dl,
3567 VT: PtrVT, N1: TLSAddr, N2: TGA);
3568 return DAG.getNode(Opcode: PPCISD::ADDI_DTPREL_L, DL: dl, VT: PtrVT, N1: DtvOffsetHi, N2: TGA);
3569 }
3570
3571 llvm_unreachable("Unknown TLS model!");
3572}
3573
3574SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
3575 SelectionDAG &DAG) const {
3576 EVT PtrVT = Op.getValueType();
3577 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Val&: Op);
3578 SDLoc DL(GSDN);
3579 const GlobalValue *GV = GSDN->getGlobal();
3580
3581 // 64-bit SVR4 ABI & AIX ABI code is always position-independent.
3582 // The actual address of the GlobalValue is stored in the TOC.
3583 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3584 if (Subtarget.isUsingPCRelativeCalls()) {
3585 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3586 if (isAccessedAsGotIndirect(N: Op)) {
3587 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3588 TargetFlags: PPCII::MO_GOT_PCREL_FLAG);
3589 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3590 SDValue Load = DAG.getLoad(VT: MVT::i64, dl: DL, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3591 PtrInfo: MachinePointerInfo());
3592 return Load;
3593 } else {
3594 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3595 TargetFlags: PPCII::MO_PCREL_FLAG);
3596 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3597 }
3598 }
3599 setUsesTOCBasePtr(DAG);
3600 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset());
3601 return getTOCEntry(DAG, dl: DL, GA);
3602 }
3603
3604 unsigned MOHiFlag, MOLoFlag;
3605 bool IsPIC = isPositionIndependent();
3606 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag, GV);
3607
3608 if (IsPIC && Subtarget.isSVR4ABI()) {
3609 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT,
3610 offset: GSDN->getOffset(),
3611 TargetFlags: PPCII::MO_PIC_FLAG);
3612 return getTOCEntry(DAG, dl: DL, GA);
3613 }
3614
3615 SDValue GAHi =
3616 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOHiFlag);
3617 SDValue GALo =
3618 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOLoFlag);
3619
3620 return LowerLabelRef(HiPart: GAHi, LoPart: GALo, isPIC: IsPIC, DAG);
3621}
3622
3623SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3624 bool IsStrict = Op->isStrictFPOpcode();
3625 const SDNodeFlags Flags = Op.getNode()->getFlags();
3626 ISD::CondCode CC =
3627 cast<CondCodeSDNode>(Val: Op.getOperand(i: IsStrict ? 3 : 2))->get();
3628 SDValue LHS = Op.getOperand(i: IsStrict ? 1 : 0);
3629 SDValue RHS = Op.getOperand(i: IsStrict ? 2 : 1);
3630 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
3631 EVT LHSVT = LHS.getValueType();
3632 SDLoc dl(Op);
3633
3634 // Soften the setcc with libcall if it is fp128 or it is SPE and fp32/fp64.
3635 if (LHSVT == MVT::f128 ||
3636 (Subtarget.hasSPE() && (LHSVT == MVT::f32 || LHSVT == MVT::f64) &&
3637 (!Flags.hasNoNaNs() || !Flags.hasNoInfs()))) {
3638 assert(!Subtarget.hasP9Vector() &&
3639 "SETCC for f128 is already legal under Power9!");
3640 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS, Chain,
3641 IsSignaling: Op->getOpcode() == ISD::STRICT_FSETCCS);
3642 if (RHS.getNode())
3643 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS,
3644 N3: DAG.getCondCode(Cond: CC));
3645 if (IsStrict)
3646 return DAG.getMergeValues(Ops: {LHS, Chain}, dl);
3647 return LHS;
3648 } else if (LHSVT == MVT::f32 || LHSVT == MVT::f64) {
3649 return Op;
3650 }
3651
3652 assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!");
3653
3654 if (Op.getValueType() == MVT::v2i64) {
3655 // When the operands themselves are v2i64 values, we need to do something
3656 // special because VSX has no underlying comparison operations for these.
3657 if (LHS.getValueType() == MVT::v2i64) {
3658 // Equality can be handled by casting to the legal type for Altivec
3659 // comparisons, everything else needs to be expanded.
3660 if (CC != ISD::SETEQ && CC != ISD::SETNE)
3661 return SDValue();
3662 SDValue SetCC32 = DAG.getSetCC(
3663 DL: dl, VT: MVT::v4i32, LHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: LHS),
3664 RHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: RHS), Cond: CC);
3665 int ShuffV[] = {1, 0, 3, 2};
3666 SDValue Shuff =
3667 DAG.getVectorShuffle(VT: MVT::v4i32, dl, N1: SetCC32, N2: SetCC32, Mask: ShuffV);
3668 return DAG.getBitcast(VT: MVT::v2i64,
3669 V: DAG.getNode(Opcode: CC == ISD::SETEQ ? ISD::AND : ISD::OR,
3670 DL: dl, VT: MVT::v4i32, N1: Shuff, N2: SetCC32));
3671 }
3672
3673 // We handle most of these in the usual way.
3674 return Op;
3675 }
3676
3677 // If we're comparing for equality to zero, expose the fact that this is
3678 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can
3679 // fold the new nodes.
3680 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG))
3681 return V;
3682
3683 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3684 // Leave comparisons against 0 and -1 alone for now, since they're usually
3685 // optimized. FIXME: revisit this when we can custom lower all setcc
3686 // optimizations.
3687 if (C->isAllOnes() || C->isZero())
3688 return SDValue();
3689 }
3690
3691 // If we have an integer seteq/setne, turn it into a compare against zero
3692 // by xor'ing the rhs with the lhs, which is faster than setting a
3693 // condition register, reading it back out, and masking the correct bit. The
3694 // normal approach here uses sub to do this instead of xor. Using xor exposes
3695 // the result to other bit-twiddling opportunities.
3696 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
3697 EVT VT = Op.getValueType();
3698 SDValue Sub = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSVT, N1: LHS, N2: RHS);
3699 return DAG.getSetCC(DL: dl, VT, LHS: Sub, RHS: DAG.getConstant(Val: 0, DL: dl, VT: LHSVT), Cond: CC);
3700 }
3701 return SDValue();
3702}
3703
3704SDValue PPCTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3705 const SDNodeFlags Flags = Op->getFlags();
3706 SDValue Chain = Op.getOperand(i: 0);
3707 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
3708 SDValue LHS = Op.getOperand(i: 2);
3709 SDValue RHS = Op.getOperand(i: 3);
3710 SDValue Dest = Op.getOperand(i: 4);
3711 EVT LHSVT = LHS.getValueType();
3712 SDLoc dl(Op);
3713
3714 assert(Subtarget.hasSPE() && "LowerBR_CC used only for targets with SPE");
3715
3716 if ((LHSVT == MVT::f32 || LHSVT == MVT::f64) && Flags.hasNoNaNs() &&
3717 Flags.hasNoInfs())
3718 return Op;
3719
3720 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
3721
3722 // If softenSetCCOperands returned a scalar, we need to compare the result
3723 // against zero to select between true and false values.
3724 if (!RHS) {
3725 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHSVT);
3726 CC = ISD::SETNE;
3727 }
3728
3729 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Op.getValueType(), N1: Chain,
3730 N2: DAG.getCondCode(Cond: CC), N3: LHS, N4: RHS, N5: Dest);
3731}
3732
3733SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3734 SDNode *Node = Op.getNode();
3735 EVT VT = Node->getValueType(ResNo: 0);
3736 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3737 SDValue InChain = Node->getOperand(Num: 0);
3738 SDValue VAListPtr = Node->getOperand(Num: 1);
3739 const Value *SV = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
3740 SDLoc dl(Node);
3741
3742 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only");
3743
3744 // gpr_index
3745 SDValue GprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3746 Ptr: VAListPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3747 InChain = GprIndex.getValue(R: 1);
3748
3749 if (VT == MVT::i64) {
3750 // Check if GprIndex is even
3751 SDValue GprAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: GprIndex,
3752 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3753 SDValue CC64 = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: GprAnd,
3754 RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), Cond: ISD::SETNE);
3755 SDValue GprIndexPlusOne = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: GprIndex,
3756 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3757 // Align GprIndex to be even if it isn't
3758 GprIndex = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC64, N2: GprIndexPlusOne,
3759 N3: GprIndex);
3760 }
3761
3762 // fpr index is 1 byte after gpr
3763 SDValue FprPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3764 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3765
3766 // fpr
3767 SDValue FprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3768 Ptr: FprPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3769 InChain = FprIndex.getValue(R: 1);
3770
3771 SDValue RegSaveAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3772 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
3773
3774 SDValue OverflowAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3775 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
3776
3777 // areas
3778 SDValue OverflowArea =
3779 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: OverflowAreaPtr, PtrInfo: MachinePointerInfo());
3780 InChain = OverflowArea.getValue(R: 1);
3781
3782 SDValue RegSaveArea =
3783 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: RegSaveAreaPtr, PtrInfo: MachinePointerInfo());
3784 InChain = RegSaveArea.getValue(R: 1);
3785
3786 // select overflow_area if index > 8
3787 SDValue CC = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: VT.isInteger() ? GprIndex : FprIndex,
3788 RHS: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32), Cond: ISD::SETLT);
3789
3790 // adjustment constant gpr_index * 4/8
3791 SDValue RegConstant = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32,
3792 N1: VT.isInteger() ? GprIndex : FprIndex,
3793 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8, DL: dl,
3794 VT: MVT::i32));
3795
3796 // OurReg = RegSaveArea + RegConstant
3797 SDValue OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: RegSaveArea,
3798 N2: RegConstant);
3799
3800 // Floating types are 32 bytes into RegSaveArea
3801 if (VT.isFloatingPoint())
3802 OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OurReg,
3803 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
3804
3805 // increase {f,g}pr_index by 1 (or 2 if VT is i64)
3806 SDValue IndexPlus1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32,
3807 N1: VT.isInteger() ? GprIndex : FprIndex,
3808 N2: DAG.getConstant(Val: VT == MVT::i64 ? 2 : 1, DL: dl,
3809 VT: MVT::i32));
3810
3811 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: IndexPlus1,
3812 Ptr: VT.isInteger() ? VAListPtr : FprPtr,
3813 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
3814
3815 // determine if we should load from reg_save_area or overflow_area
3816 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: PtrVT, N1: CC, N2: OurReg, N3: OverflowArea);
3817
3818 // increase overflow_area by 4/8 if gpr/fpr > 8
3819 SDValue OverflowAreaPlusN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OverflowArea,
3820 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8,
3821 DL: dl, VT: MVT::i32));
3822
3823 OverflowArea = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC, N2: OverflowArea,
3824 N3: OverflowAreaPlusN);
3825
3826 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: OverflowArea, Ptr: OverflowAreaPtr,
3827 PtrInfo: MachinePointerInfo(), SVT: MVT::i32);
3828
3829 return DAG.getLoad(VT, dl, Chain: InChain, Ptr: Result, PtrInfo: MachinePointerInfo());
3830}
3831
3832SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
3833 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only");
3834
3835 // We have to copy the entire va_list struct:
3836 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
3837 return DAG.getMemcpy(Chain: Op.getOperand(i: 0), dl: Op, Dst: Op.getOperand(i: 1), Src: Op.getOperand(i: 2),
3838 Size: DAG.getConstant(Val: 12, DL: SDLoc(Op), VT: MVT::i32), DstAlign: Align(8),
3839 SrcAlign: Align(8), isVol: false, AlwaysInline: true, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
3840 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
3841}
3842
3843SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
3844 SelectionDAG &DAG) const {
3845 return Op.getOperand(i: 0);
3846}
3847
3848SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
3849 MachineFunction &MF = DAG.getMachineFunction();
3850 PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>();
3851
3852 assert((Op.getOpcode() == ISD::INLINEASM ||
3853 Op.getOpcode() == ISD::INLINEASM_BR) &&
3854 "Expecting Inline ASM node.");
3855
3856 // If an LR store is already known to be required then there is not point in
3857 // checking this ASM as well.
3858 if (MFI.isLRStoreRequired())
3859 return Op;
3860
3861 // Inline ASM nodes have an optional last operand that is an incoming Flag of
3862 // type MVT::Glue. We want to ignore this last operand if that is the case.
3863 unsigned NumOps = Op.getNumOperands();
3864 if (Op.getOperand(i: NumOps - 1).getValueType() == MVT::Glue)
3865 --NumOps;
3866
3867 // Check all operands that may contain the LR.
3868 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
3869 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
3870 unsigned NumVals = Flags.getNumOperandRegisters();
3871 ++i; // Skip the ID value.
3872
3873 switch (Flags.getKind()) {
3874 default:
3875 llvm_unreachable("Bad flags!");
3876 case InlineAsm::Kind::RegUse:
3877 case InlineAsm::Kind::Imm:
3878 case InlineAsm::Kind::Mem:
3879 i += NumVals;
3880 break;
3881 case InlineAsm::Kind::Clobber:
3882 case InlineAsm::Kind::RegDef:
3883 case InlineAsm::Kind::RegDefEarlyClobber: {
3884 for (; NumVals; --NumVals, ++i) {
3885 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
3886 if (Reg != PPC::LR && Reg != PPC::LR8)
3887 continue;
3888 MFI.setLRStoreRequired();
3889 return Op;
3890 }
3891 break;
3892 }
3893 }
3894 }
3895
3896 return Op;
3897}
3898
3899SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
3900 SelectionDAG &DAG) const {
3901 SDValue Chain = Op.getOperand(i: 0);
3902 SDValue Trmp = Op.getOperand(i: 1); // trampoline
3903 SDValue FPtr = Op.getOperand(i: 2); // nested function
3904 SDValue Nest = Op.getOperand(i: 3); // 'nest' parameter value
3905 SDLoc dl(Op);
3906
3907 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3908
3909 if (Subtarget.isAIXABI()) {
3910 // On AIX we create a trampoline descriptor by combining the
3911 // entry point and TOC from the global descriptor (FPtr) with the
3912 // nest argument as the environment pointer.
3913 uint64_t PointerSize = Subtarget.isPPC64() ? 8 : 4;
3914 MaybeAlign PointerAlign(PointerSize);
3915 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
3916 ? (MachineMemOperand::MODereferenceable |
3917 MachineMemOperand::MOInvariant)
3918 : MachineMemOperand::MONone;
3919
3920 uint64_t TOCPointerOffset = 1 * PointerSize;
3921 uint64_t EnvPointerOffset = 2 * PointerSize;
3922 SDValue SDTOCPtrOffset = DAG.getConstant(Val: TOCPointerOffset, DL: dl, VT: PtrVT);
3923 SDValue SDEnvPtrOffset = DAG.getConstant(Val: EnvPointerOffset, DL: dl, VT: PtrVT);
3924
3925 const Value *TrampolineAddr =
3926 cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
3927 const Function *Func =
3928 cast<Function>(Val: cast<SrcValueSDNode>(Val: Op.getOperand(i: 5))->getValue());
3929
3930 SDValue OutChains[3];
3931
3932 // Copy the entry point address from the global descriptor to the
3933 // trampoline buffer.
3934 SDValue LoadEntryPoint =
3935 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: FPtr, PtrInfo: MachinePointerInfo(Func, 0),
3936 Alignment: PointerAlign, MMOFlags);
3937 SDValue EPLoadChain = LoadEntryPoint.getValue(R: 1);
3938 OutChains[0] = DAG.getStore(Chain: EPLoadChain, dl, Val: LoadEntryPoint, Ptr: Trmp,
3939 PtrInfo: MachinePointerInfo(TrampolineAddr, 0));
3940
3941 // Copy the TOC pointer from the global descriptor to the trampoline
3942 // buffer.
3943 SDValue TOCFromDescriptorPtr =
3944 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FPtr, N2: SDTOCPtrOffset);
3945 SDValue TOCReg = DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: TOCFromDescriptorPtr,
3946 PtrInfo: MachinePointerInfo(Func, TOCPointerOffset),
3947 Alignment: PointerAlign, MMOFlags);
3948 SDValue TrampolineTOCPointer =
3949 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDTOCPtrOffset);
3950 SDValue TOCLoadChain = TOCReg.getValue(R: 1);
3951 OutChains[1] =
3952 DAG.getStore(Chain: TOCLoadChain, dl, Val: TOCReg, Ptr: TrampolineTOCPointer,
3953 PtrInfo: MachinePointerInfo(TrampolineAddr, TOCPointerOffset));
3954
3955 // Store the nest argument into the environment pointer in the trampoline
3956 // buffer.
3957 SDValue EnvPointer = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDEnvPtrOffset);
3958 OutChains[2] =
3959 DAG.getStore(Chain, dl, Val: Nest, Ptr: EnvPointer,
3960 PtrInfo: MachinePointerInfo(TrampolineAddr, EnvPointerOffset));
3961
3962 SDValue TokenFactor =
3963 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
3964 return TokenFactor;
3965 }
3966
3967 bool isPPC64 = (PtrVT == MVT::i64);
3968 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(C&: *DAG.getContext());
3969
3970 TargetLowering::ArgListTy Args;
3971 Args.emplace_back(args&: Trmp, args&: IntPtrTy);
3972 // TrampSize == (isPPC64 ? 48 : 40);
3973 Args.emplace_back(
3974 args: DAG.getConstant(Val: isPPC64 ? 48 : 40, DL: dl, VT: Subtarget.getScalarIntVT()),
3975 args&: IntPtrTy);
3976 Args.emplace_back(args&: FPtr, args&: IntPtrTy);
3977 Args.emplace_back(args&: Nest, args&: IntPtrTy);
3978
3979 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
3980 TargetLowering::CallLoweringInfo CLI(DAG);
3981 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3982 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
3983 Target: DAG.getExternalSymbol(Sym: "__trampoline_setup", VT: PtrVT), ArgsList: std::move(Args));
3984
3985 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3986 return CallResult.second;
3987}
3988
3989SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3990 MachineFunction &MF = DAG.getMachineFunction();
3991 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3992 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
3993
3994 SDLoc dl(Op);
3995
3996 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) {
3997 // vastart just stores the address of the VarArgsFrameIndex slot into the
3998 // memory location argument.
3999 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4000 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4001 return DAG.getStore(Chain: Op.getOperand(i: 0), dl, Val: FR, Ptr: Op.getOperand(i: 1),
4002 PtrInfo: MachinePointerInfo(SV));
4003 }
4004
4005 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct.
4006 // We suppose the given va_list is already allocated.
4007 //
4008 // typedef struct {
4009 // char gpr; /* index into the array of 8 GPRs
4010 // * stored in the register save area
4011 // * gpr=0 corresponds to r3,
4012 // * gpr=1 to r4, etc.
4013 // */
4014 // char fpr; /* index into the array of 8 FPRs
4015 // * stored in the register save area
4016 // * fpr=0 corresponds to f1,
4017 // * fpr=1 to f2, etc.
4018 // */
4019 // char *overflow_arg_area;
4020 // /* location on stack that holds
4021 // * the next overflow argument
4022 // */
4023 // char *reg_save_area;
4024 // /* where r3:r10 and f1:f8 (if saved)
4025 // * are stored
4026 // */
4027 // } va_list[1];
4028
4029 SDValue ArgGPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumGPR(), DL: dl, VT: MVT::i32);
4030 SDValue ArgFPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumFPR(), DL: dl, VT: MVT::i32);
4031 SDValue StackOffsetFI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsStackOffset(),
4032 VT: PtrVT);
4033 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4034 VT: PtrVT);
4035
4036 uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
4037 SDValue ConstFrameOffset = DAG.getConstant(Val: FrameOffset, DL: dl, VT: PtrVT);
4038
4039 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
4040 SDValue ConstStackOffset = DAG.getConstant(Val: StackOffset, DL: dl, VT: PtrVT);
4041
4042 uint64_t FPROffset = 1;
4043 SDValue ConstFPROffset = DAG.getConstant(Val: FPROffset, DL: dl, VT: PtrVT);
4044
4045 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4046
4047 // Store first byte : number of int regs
4048 SDValue firstStore =
4049 DAG.getTruncStore(Chain: Op.getOperand(i: 0), dl, Val: ArgGPR, Ptr: Op.getOperand(i: 1),
4050 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
4051 uint64_t nextOffset = FPROffset;
4052 SDValue nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Op.getOperand(i: 1),
4053 N2: ConstFPROffset);
4054
4055 // Store second byte : number of float regs
4056 SDValue secondStore =
4057 DAG.getTruncStore(Chain: firstStore, dl, Val: ArgFPR, Ptr: nextPtr,
4058 PtrInfo: MachinePointerInfo(SV, nextOffset), SVT: MVT::i8);
4059 nextOffset += StackOffset;
4060 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstStackOffset);
4061
4062 // Store second word : arguments given on stack
4063 SDValue thirdStore = DAG.getStore(Chain: secondStore, dl, Val: StackOffsetFI, Ptr: nextPtr,
4064 PtrInfo: MachinePointerInfo(SV, nextOffset));
4065 nextOffset += FrameOffset;
4066 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstFrameOffset);
4067
4068 // Store third word : arguments given in registers
4069 return DAG.getStore(Chain: thirdStore, dl, Val: FR, Ptr: nextPtr,
4070 PtrInfo: MachinePointerInfo(SV, nextOffset));
4071}
4072
4073/// FPR - The set of FP registers that should be allocated for arguments
4074/// on Darwin and AIX.
4075static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5,
4076 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10,
4077 PPC::F11, PPC::F12, PPC::F13};
4078
4079/// CalculateStackSlotSize - Calculates the size reserved for this argument on
4080/// the stack.
4081static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
4082 unsigned PtrByteSize) {
4083 unsigned ArgSize = ArgVT.getStoreSize();
4084 if (Flags.isByVal())
4085 ArgSize = Flags.getByValSize();
4086
4087 // Round up to multiples of the pointer size, except for array members,
4088 // which are always packed.
4089 if (!Flags.isInConsecutiveRegs())
4090 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4091
4092 return ArgSize;
4093}
4094
4095/// CalculateStackSlotAlignment - Calculates the alignment of this argument
4096/// on the stack.
4097static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
4098 ISD::ArgFlagsTy Flags,
4099 unsigned PtrByteSize) {
4100 Align Alignment(PtrByteSize);
4101
4102 // Altivec parameters are padded to a 16 byte boundary.
4103 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4104 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4105 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4106 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4107 Alignment = Align(16);
4108
4109 // ByVal parameters are aligned as requested.
4110 if (Flags.isByVal()) {
4111 auto BVAlign = Flags.getNonZeroByValAlign();
4112 if (BVAlign > PtrByteSize) {
4113 if (BVAlign.value() % PtrByteSize != 0)
4114 llvm_unreachable(
4115 "ByVal alignment is not a multiple of the pointer size");
4116
4117 Alignment = BVAlign;
4118 }
4119 }
4120
4121 // Array members are always packed to their original alignment.
4122 if (Flags.isInConsecutiveRegs()) {
4123 // If the array member was split into multiple registers, the first
4124 // needs to be aligned to the size of the full type. (Except for
4125 // ppcf128, which is only aligned as its f64 components.)
4126 if (Flags.isSplit() && OrigVT != MVT::ppcf128)
4127 Alignment = Align(OrigVT.getStoreSize());
4128 else
4129 Alignment = Align(ArgVT.getStoreSize());
4130 }
4131
4132 return Alignment;
4133}
4134
4135/// CalculateStackSlotUsed - Return whether this argument will use its
4136/// stack slot (instead of being passed in registers). ArgOffset,
4137/// AvailableFPRs, and AvailableVRs must hold the current argument
4138/// position, and will be updated to account for this argument.
4139static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
4140 unsigned PtrByteSize, unsigned LinkageSize,
4141 unsigned ParamAreaSize, unsigned &ArgOffset,
4142 unsigned &AvailableFPRs,
4143 unsigned &AvailableVRs) {
4144 bool UseMemory = false;
4145
4146 // Respect alignment of argument on the stack.
4147 Align Alignment =
4148 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
4149 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4150 // If there's no space left in the argument save area, we must
4151 // use memory (this check also catches zero-sized arguments).
4152 if (ArgOffset >= LinkageSize + ParamAreaSize)
4153 UseMemory = true;
4154
4155 // Allocate argument on the stack.
4156 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
4157 if (Flags.isInConsecutiveRegsLast())
4158 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4159 // If we overran the argument save area, we must use memory
4160 // (this check catches arguments passed partially in memory)
4161 if (ArgOffset > LinkageSize + ParamAreaSize)
4162 UseMemory = true;
4163
4164 // However, if the argument is actually passed in an FPR or a VR,
4165 // we don't use memory after all.
4166 if (!Flags.isByVal()) {
4167 if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
4168 if (AvailableFPRs > 0) {
4169 --AvailableFPRs;
4170 return false;
4171 }
4172 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4173 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4174 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4175 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4176 if (AvailableVRs > 0) {
4177 --AvailableVRs;
4178 return false;
4179 }
4180 }
4181
4182 return UseMemory;
4183}
4184
4185/// EnsureStackAlignment - Round stack frame size up from NumBytes to
4186/// ensure minimum alignment required for target.
4187static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
4188 unsigned NumBytes) {
4189 return alignTo(Size: NumBytes, A: Lowering->getStackAlign());
4190}
4191
4192SDValue PPCTargetLowering::LowerFormalArguments(
4193 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4194 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4195 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4196 if (Subtarget.isAIXABI())
4197 return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG,
4198 InVals);
4199 if (Subtarget.is64BitELFABI())
4200 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4201 InVals);
4202 assert(Subtarget.is32BitELFABI());
4203 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4204 InVals);
4205}
4206
4207SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
4208 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4209 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4210 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4211
4212 // 32-bit SVR4 ABI Stack Frame Layout:
4213 // +-----------------------------------+
4214 // +--> | Back chain |
4215 // | +-----------------------------------+
4216 // | | Floating-point register save area |
4217 // | +-----------------------------------+
4218 // | | General register save area |
4219 // | +-----------------------------------+
4220 // | | CR save word |
4221 // | +-----------------------------------+
4222 // | | VRSAVE save word |
4223 // | +-----------------------------------+
4224 // | | Alignment padding |
4225 // | +-----------------------------------+
4226 // | | Vector register save area |
4227 // | +-----------------------------------+
4228 // | | Local variable space |
4229 // | +-----------------------------------+
4230 // | | Parameter list area |
4231 // | +-----------------------------------+
4232 // | | LR save word |
4233 // | +-----------------------------------+
4234 // SP--> +--- | Back chain |
4235 // +-----------------------------------+
4236 //
4237 // Specifications:
4238 // System V Application Binary Interface PowerPC Processor Supplement
4239 // AltiVec Technology Programming Interface Manual
4240
4241 MachineFunction &MF = DAG.getMachineFunction();
4242 MachineFrameInfo &MFI = MF.getFrameInfo();
4243 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4244
4245 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4246 // Potential tail calls could cause overwriting of argument stack slots.
4247 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4248 (CallConv == CallingConv::Fast));
4249 const Align PtrAlign(4);
4250
4251 // Assign locations to all of the incoming arguments.
4252 SmallVector<CCValAssign, 16> ArgLocs;
4253 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4254 *DAG.getContext());
4255
4256 // Reserve space for the linkage area on the stack.
4257 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4258 CCInfo.AllocateStack(Size: LinkageSize, Alignment: PtrAlign);
4259 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4);
4260
4261 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4262 CCValAssign &VA = ArgLocs[i];
4263
4264 // Arguments stored in registers.
4265 if (VA.isRegLoc()) {
4266 const TargetRegisterClass *RC;
4267 EVT ValVT = VA.getValVT();
4268
4269 switch (ValVT.getSimpleVT().SimpleTy) {
4270 default:
4271 llvm_unreachable("ValVT not supported by formal arguments Lowering");
4272 case MVT::i1:
4273 case MVT::i32:
4274 RC = &PPC::GPRCRegClass;
4275 break;
4276 case MVT::f32:
4277 if (Subtarget.hasP8Vector())
4278 RC = &PPC::VSSRCRegClass;
4279 else if (Subtarget.hasSPE())
4280 RC = &PPC::GPRCRegClass;
4281 else
4282 RC = &PPC::F4RCRegClass;
4283 break;
4284 case MVT::f64:
4285 if (Subtarget.hasVSX())
4286 RC = &PPC::VSFRCRegClass;
4287 else if (Subtarget.hasSPE())
4288 // SPE passes doubles in GPR pairs.
4289 RC = &PPC::GPRCRegClass;
4290 else
4291 RC = &PPC::F8RCRegClass;
4292 break;
4293 case MVT::v16i8:
4294 case MVT::v8i16:
4295 case MVT::v4i32:
4296 case MVT::v4f32:
4297 case MVT::v2f64:
4298 case MVT::v2i64:
4299 case MVT::f128:
4300 RC = &PPC::VRRCRegClass;
4301 break;
4302 }
4303
4304 SDValue ArgValue;
4305 // Transform the arguments stored in physical registers into
4306 // virtual ones.
4307 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) {
4308 assert(i + 1 < e && "No second half of double precision argument");
4309 Register RegLo = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4310 Register RegHi = MF.addLiveIn(PReg: ArgLocs[++i].getLocReg(), RC);
4311 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, Reg: RegLo, VT: MVT::i32);
4312 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, Reg: RegHi, VT: MVT::i32);
4313 if (!Subtarget.isLittleEndian())
4314 std::swap (a&: ArgValueLo, b&: ArgValueHi);
4315 ArgValue = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: ArgValueLo,
4316 N2: ArgValueHi);
4317 } else {
4318 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4319 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
4320 VT: ValVT == MVT::i1 ? MVT::i32 : ValVT);
4321 if (ValVT == MVT::i1)
4322 ArgValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: ArgValue);
4323 }
4324
4325 InVals.push_back(Elt: ArgValue);
4326 } else {
4327 // Argument stored in memory.
4328 assert(VA.isMemLoc());
4329
4330 // Get the extended size of the argument type in stack
4331 unsigned ArgSize = VA.getLocVT().getStoreSize();
4332 // Get the actual size of the argument type
4333 unsigned ObjSize = VA.getValVT().getStoreSize();
4334 unsigned ArgOffset = VA.getLocMemOffset();
4335 // Stack objects in PPC32 are right justified.
4336 ArgOffset += ArgSize - ObjSize;
4337 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: isImmutable);
4338
4339 // Create load nodes to retrieve arguments from the stack.
4340 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4341 InVals.push_back(
4342 Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo()));
4343 }
4344 }
4345
4346 // Assign locations to all of the incoming aggregate by value arguments.
4347 // Aggregates passed by value are stored in the local variable space of the
4348 // caller's stack frame, right above the parameter list area.
4349 SmallVector<CCValAssign, 16> ByValArgLocs;
4350 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
4351 ByValArgLocs, *DAG.getContext());
4352
4353 // Reserve stack space for the allocations in CCInfo.
4354 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
4355
4356 CCByValInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4_ByVal);
4357
4358 // Area that is at least reserved in the caller of this function.
4359 unsigned MinReservedArea = CCByValInfo.getStackSize();
4360 MinReservedArea = std::max(a: MinReservedArea, b: LinkageSize);
4361
4362 // Set the size that is at least reserved in caller of this function. Tail
4363 // call optimized function's reserved stack space needs to be aligned so that
4364 // taking the difference between two stack areas will result in an aligned
4365 // stack.
4366 MinReservedArea =
4367 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4368 FuncInfo->setMinReservedArea(MinReservedArea);
4369
4370 SmallVector<SDValue, 8> MemOps;
4371
4372 // If the function takes variable number of arguments, make a frame index for
4373 // the start of the first vararg value... for expansion of llvm.va_start.
4374 if (isVarArg) {
4375 static const MCPhysReg GPArgRegs[] = {
4376 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
4377 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
4378 };
4379 const unsigned NumGPArgRegs = std::size(GPArgRegs);
4380
4381 static const MCPhysReg FPArgRegs[] = {
4382 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
4383 PPC::F8
4384 };
4385 unsigned NumFPArgRegs = std::size(FPArgRegs);
4386
4387 if (useSoftFloat() || hasSPE())
4388 NumFPArgRegs = 0;
4389
4390 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(Regs: GPArgRegs));
4391 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(Regs: FPArgRegs));
4392
4393 // Make room for NumGPArgRegs and NumFPArgRegs.
4394 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 +
4395 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8;
4396
4397 FuncInfo->setVarArgsStackOffset(MFI.CreateFixedObject(
4398 Size: PtrVT.getSizeInBits() / 8, SPOffset: CCInfo.getStackSize(), IsImmutable: true));
4399
4400 FuncInfo->setVarArgsFrameIndex(
4401 MFI.CreateStackObject(Size: Depth, Alignment: Align(8), isSpillSlot: false));
4402 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4403
4404 // The fixed integer arguments of a variadic function are stored to the
4405 // VarArgsFrameIndex on the stack so that they may be loaded by
4406 // dereferencing the result of va_next.
4407 for (MCPhysReg GPArgReg : GPArgRegs) {
4408 // Get an existing live-in vreg, or add a new one.
4409 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: GPArgReg);
4410 if (!VReg)
4411 VReg = MF.addLiveIn(PReg: GPArgReg, RC: &PPC::GPRCRegClass);
4412
4413 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4414 SDValue Store =
4415 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4416 MemOps.push_back(Elt: Store);
4417 // Increment the address by four for the next argument to store
4418 SDValue PtrOff = DAG.getConstant(Val: PtrVT.getSizeInBits()/8, DL: dl, VT: PtrVT);
4419 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4420 }
4421
4422 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6
4423 // is set.
4424 // The double arguments are stored to the VarArgsFrameIndex
4425 // on the stack.
4426 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) {
4427 // Get an existing live-in vreg, or add a new one.
4428 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: FPArgRegs[FPRIndex]);
4429 if (!VReg)
4430 VReg = MF.addLiveIn(PReg: FPArgRegs[FPRIndex], RC: &PPC::F8RCRegClass);
4431
4432 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::f64);
4433 SDValue Store =
4434 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4435 MemOps.push_back(Elt: Store);
4436 // Increment the address by eight for the next argument to store
4437 SDValue PtrOff = DAG.getConstant(Val: MVT(MVT::f64).getSizeInBits()/8, DL: dl,
4438 VT: PtrVT);
4439 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4440 }
4441 }
4442
4443 if (!MemOps.empty())
4444 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4445
4446 return Chain;
4447}
4448
4449// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4450// value to MVT::i64 and then truncate to the correct register size.
4451SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags,
4452 EVT ObjectVT, SelectionDAG &DAG,
4453 SDValue ArgVal,
4454 const SDLoc &dl) const {
4455 if (Flags.isSExt())
4456 ArgVal = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: MVT::i64, N1: ArgVal,
4457 N2: DAG.getValueType(ObjectVT));
4458 else if (Flags.isZExt())
4459 ArgVal = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i64, N1: ArgVal,
4460 N2: DAG.getValueType(ObjectVT));
4461
4462 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ObjectVT, Operand: ArgVal);
4463}
4464
4465SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
4466 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4467 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4468 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4469 // TODO: add description of PPC stack frame format, or at least some docs.
4470 //
4471 bool isELFv2ABI = Subtarget.isELFv2ABI();
4472 bool isLittleEndian = Subtarget.isLittleEndian();
4473 MachineFunction &MF = DAG.getMachineFunction();
4474 MachineFrameInfo &MFI = MF.getFrameInfo();
4475 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4476
4477 assert(!(CallConv == CallingConv::Fast && isVarArg) &&
4478 "fastcc not supported on varargs functions");
4479
4480 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4481 // Potential tail calls could cause overwriting of argument stack slots.
4482 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4483 (CallConv == CallingConv::Fast));
4484 unsigned PtrByteSize = 8;
4485 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4486
4487 static const MCPhysReg GPR[] = {
4488 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4489 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4490 };
4491 static const MCPhysReg VR[] = {
4492 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4493 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4494 };
4495
4496 const unsigned Num_GPR_Regs = std::size(GPR);
4497 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
4498 const unsigned Num_VR_Regs = std::size(VR);
4499
4500 // Do a first pass over the arguments to determine whether the ABI
4501 // guarantees that our caller has allocated the parameter save area
4502 // on its stack frame. In the ELFv1 ABI, this is always the case;
4503 // in the ELFv2 ABI, it is true if this is a vararg function or if
4504 // any parameter is located in a stack slot.
4505
4506 bool HasParameterArea = !isELFv2ABI || isVarArg;
4507 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize;
4508 unsigned NumBytes = LinkageSize;
4509 unsigned AvailableFPRs = Num_FPR_Regs;
4510 unsigned AvailableVRs = Num_VR_Regs;
4511 for (const ISD::InputArg &In : Ins) {
4512 if (In.Flags.isNest())
4513 continue;
4514
4515 if (CalculateStackSlotUsed(ArgVT: In.VT, OrigVT: In.ArgVT, Flags: In.Flags, PtrByteSize,
4516 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4517 AvailableFPRs, AvailableVRs))
4518 HasParameterArea = true;
4519 }
4520
4521 // Add DAG nodes to load the arguments or copy them out of registers. On
4522 // entry to a function on PPC, the arguments start after the linkage area,
4523 // although the first ones are often in registers.
4524
4525 unsigned ArgOffset = LinkageSize;
4526 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
4527 SmallVector<SDValue, 8> MemOps;
4528 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
4529 unsigned CurArgIdx = 0;
4530 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
4531 SDValue ArgVal;
4532 bool needsLoad = false;
4533 EVT ObjectVT = Ins[ArgNo].VT;
4534 EVT OrigVT = Ins[ArgNo].ArgVT;
4535 unsigned ObjSize = ObjectVT.getStoreSize();
4536 unsigned ArgSize = ObjSize;
4537 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
4538 if (Ins[ArgNo].isOrigArg()) {
4539 std::advance(i&: FuncArg, n: Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
4540 CurArgIdx = Ins[ArgNo].getOrigArgIndex();
4541 }
4542 // We re-align the argument offset for each argument, except when using the
4543 // fast calling convention, when we need to make sure we do that only when
4544 // we'll actually use a stack slot.
4545 unsigned CurArgOffset;
4546 Align Alignment;
4547 auto ComputeArgOffset = [&]() {
4548 /* Respect alignment of argument on the stack. */
4549 Alignment =
4550 CalculateStackSlotAlignment(ArgVT: ObjectVT, OrigVT, Flags, PtrByteSize);
4551 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4552 CurArgOffset = ArgOffset;
4553 };
4554
4555 if (CallConv != CallingConv::Fast) {
4556 ComputeArgOffset();
4557
4558 /* Compute GPR index associated with argument offset. */
4559 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4560 GPR_idx = std::min(a: GPR_idx, b: Num_GPR_Regs);
4561 }
4562
4563 // FIXME the codegen can be much improved in some cases.
4564 // We do not have to keep everything in memory.
4565 if (Flags.isByVal()) {
4566 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit");
4567
4568 if (CallConv == CallingConv::Fast)
4569 ComputeArgOffset();
4570
4571 // ObjSize is the true size, ArgSize rounded up to multiple of registers.
4572 ObjSize = Flags.getByValSize();
4573 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4574 // Empty aggregate parameters do not take up registers. Examples:
4575 // struct { } a;
4576 // union { } b;
4577 // int c[0];
4578 // etc. However, we have to provide a place-holder in InVals, so
4579 // pretend we have an 8-byte item at the current address for that
4580 // purpose.
4581 if (!ObjSize) {
4582 int FI = MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: ArgOffset, IsImmutable: true);
4583 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4584 InVals.push_back(Elt: FIN);
4585 continue;
4586 }
4587
4588 // Create a stack object covering all stack doublewords occupied
4589 // by the argument. If the argument is (fully or partially) on
4590 // the stack, or if the argument is fully in registers but the
4591 // caller has allocated the parameter save anyway, we can refer
4592 // directly to the caller's stack frame. Otherwise, create a
4593 // local copy in our own frame.
4594 int FI;
4595 if (HasParameterArea ||
4596 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize)
4597 FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: false, isAliased: true);
4598 else
4599 FI = MFI.CreateStackObject(Size: ArgSize, Alignment, isSpillSlot: false);
4600 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4601
4602 // Handle aggregates smaller than 8 bytes.
4603 if (ObjSize < PtrByteSize) {
4604 // The value of the object is its address, which differs from the
4605 // address of the enclosing doubleword on big-endian systems.
4606 SDValue Arg = FIN;
4607 if (!isLittleEndian) {
4608 SDValue ArgOff = DAG.getConstant(Val: PtrByteSize - ObjSize, DL: dl, VT: PtrVT);
4609 Arg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ArgOff.getValueType(), N1: Arg, N2: ArgOff);
4610 }
4611 InVals.push_back(Elt: Arg);
4612
4613 if (GPR_idx != Num_GPR_Regs) {
4614 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4615 FuncInfo->addLiveInAttr(VReg, Flags);
4616 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4617 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ObjSize * 8);
4618 SDValue Store =
4619 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Arg,
4620 PtrInfo: MachinePointerInfo(&*FuncArg), SVT: ObjType);
4621 MemOps.push_back(Elt: Store);
4622 }
4623 // Whether we copied from a register or not, advance the offset
4624 // into the parameter save area by a full doubleword.
4625 ArgOffset += PtrByteSize;
4626 continue;
4627 }
4628
4629 // The value of the object is its address, which is the address of
4630 // its first stack doubleword.
4631 InVals.push_back(Elt: FIN);
4632
4633 // Store whatever pieces of the object are in registers to memory.
4634 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
4635 if (GPR_idx == Num_GPR_Regs)
4636 break;
4637
4638 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4639 FuncInfo->addLiveInAttr(VReg, Flags);
4640 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4641 SDValue Addr = FIN;
4642 if (j) {
4643 SDValue Off = DAG.getConstant(Val: j, DL: dl, VT: PtrVT);
4644 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Off.getValueType(), N1: Addr, N2: Off);
4645 }
4646 unsigned StoreSizeInBits = std::min(a: PtrByteSize, b: (ObjSize - j)) * 8;
4647 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreSizeInBits);
4648 SDValue Store =
4649 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Addr,
4650 PtrInfo: MachinePointerInfo(&*FuncArg, j), SVT: ObjType);
4651 MemOps.push_back(Elt: Store);
4652 ++GPR_idx;
4653 }
4654 ArgOffset += ArgSize;
4655 continue;
4656 }
4657
4658 switch (ObjectVT.getSimpleVT().SimpleTy) {
4659 default: llvm_unreachable("Unhandled argument type!");
4660 case MVT::i1:
4661 case MVT::i32:
4662 case MVT::i64:
4663 if (Flags.isNest()) {
4664 // The 'nest' parameter, if any, is passed in R11.
4665 Register VReg = MF.addLiveIn(PReg: PPC::X11, RC: &PPC::G8RCRegClass);
4666 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4667
4668 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4669 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4670
4671 break;
4672 }
4673
4674 // These can be scalar arguments or elements of an integer array type
4675 // passed directly. Clang may use those instead of "byval" aggregate
4676 // types to avoid forcing arguments to memory unnecessarily.
4677 if (GPR_idx != Num_GPR_Regs) {
4678 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4679 FuncInfo->addLiveInAttr(VReg, Flags);
4680 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4681
4682 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4683 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4684 // value to MVT::i64 and then truncate to the correct register size.
4685 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4686 } else {
4687 if (CallConv == CallingConv::Fast)
4688 ComputeArgOffset();
4689
4690 needsLoad = true;
4691 ArgSize = PtrByteSize;
4692 }
4693 if (CallConv != CallingConv::Fast || needsLoad)
4694 ArgOffset += 8;
4695 break;
4696
4697 case MVT::f32:
4698 case MVT::f64:
4699 // These can be scalar arguments or elements of a float array type
4700 // passed directly. The latter are used to implement ELFv2 homogenous
4701 // float aggregates.
4702 if (FPR_idx != Num_FPR_Regs) {
4703 unsigned VReg;
4704
4705 if (ObjectVT == MVT::f32)
4706 VReg = MF.addLiveIn(PReg: FPR[FPR_idx],
4707 RC: Subtarget.hasP8Vector()
4708 ? &PPC::VSSRCRegClass
4709 : &PPC::F4RCRegClass);
4710 else
4711 VReg = MF.addLiveIn(PReg: FPR[FPR_idx], RC: Subtarget.hasVSX()
4712 ? &PPC::VSFRCRegClass
4713 : &PPC::F8RCRegClass);
4714
4715 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4716 ++FPR_idx;
4717 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) {
4718 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
4719 // once we support fp <-> gpr moves.
4720
4721 // This can only ever happen in the presence of f32 array types,
4722 // since otherwise we never run out of FPRs before running out
4723 // of GPRs.
4724 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4725 FuncInfo->addLiveInAttr(VReg, Flags);
4726 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4727
4728 if (ObjectVT == MVT::f32) {
4729 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
4730 ArgVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgVal,
4731 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
4732 ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: ArgVal);
4733 }
4734
4735 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ObjectVT, Operand: ArgVal);
4736 } else {
4737 if (CallConv == CallingConv::Fast)
4738 ComputeArgOffset();
4739
4740 needsLoad = true;
4741 }
4742
4743 // When passing an array of floats, the array occupies consecutive
4744 // space in the argument area; only round up to the next doubleword
4745 // at the end of the array. Otherwise, each float takes 8 bytes.
4746 if (CallConv != CallingConv::Fast || needsLoad) {
4747 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize;
4748 ArgOffset += ArgSize;
4749 if (Flags.isInConsecutiveRegsLast())
4750 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4751 }
4752 break;
4753 case MVT::v4f32:
4754 case MVT::v4i32:
4755 case MVT::v8i16:
4756 case MVT::v16i8:
4757 case MVT::v2f64:
4758 case MVT::v2i64:
4759 case MVT::v1i128:
4760 case MVT::f128:
4761 // These can be scalar arguments or elements of a vector array type
4762 // passed directly. The latter are used to implement ELFv2 homogenous
4763 // vector aggregates.
4764 if (VR_idx != Num_VR_Regs) {
4765 Register VReg = MF.addLiveIn(PReg: VR[VR_idx], RC: &PPC::VRRCRegClass);
4766 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4767 ++VR_idx;
4768 } else {
4769 if (CallConv == CallingConv::Fast)
4770 ComputeArgOffset();
4771 needsLoad = true;
4772 }
4773 if (CallConv != CallingConv::Fast || needsLoad)
4774 ArgOffset += 16;
4775 break;
4776 }
4777
4778 // We need to load the argument to a virtual register if we determined
4779 // above that we ran out of physical registers of the appropriate type.
4780 if (needsLoad) {
4781 if (ObjSize < ArgSize && !isLittleEndian)
4782 CurArgOffset += ArgSize - ObjSize;
4783 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: CurArgOffset, IsImmutable: isImmutable);
4784 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4785 ArgVal = DAG.getLoad(VT: ObjectVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
4786 }
4787
4788 InVals.push_back(Elt: ArgVal);
4789 }
4790
4791 // Area that is at least reserved in the caller of this function.
4792 unsigned MinReservedArea;
4793 if (HasParameterArea)
4794 MinReservedArea = std::max(a: ArgOffset, b: LinkageSize + 8 * PtrByteSize);
4795 else
4796 MinReservedArea = LinkageSize;
4797
4798 // Set the size that is at least reserved in caller of this function. Tail
4799 // call optimized functions' reserved stack space needs to be aligned so that
4800 // taking the difference between two stack areas will result in an aligned
4801 // stack.
4802 MinReservedArea =
4803 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4804 FuncInfo->setMinReservedArea(MinReservedArea);
4805
4806 // If the function takes variable number of arguments, make a frame index for
4807 // the start of the first vararg value... for expansion of llvm.va_start.
4808 // On ELFv2ABI spec, it writes:
4809 // C programs that are intended to be *portable* across different compilers
4810 // and architectures must use the header file <stdarg.h> to deal with variable
4811 // argument lists.
4812 if (isVarArg && MFI.hasVAStart()) {
4813 int Depth = ArgOffset;
4814
4815 FuncInfo->setVarArgsFrameIndex(
4816 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: Depth, IsImmutable: true));
4817 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4818
4819 // If this function is vararg, store any remaining integer argument regs
4820 // to their spots on the stack so that they may be loaded by dereferencing
4821 // the result of va_next.
4822 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4823 GPR_idx < Num_GPR_Regs; ++GPR_idx) {
4824 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4825 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4826 SDValue Store =
4827 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4828 MemOps.push_back(Elt: Store);
4829 // Increment the address by four for the next argument to store
4830 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
4831 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4832 }
4833 }
4834
4835 if (!MemOps.empty())
4836 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4837
4838 return Chain;
4839}
4840
4841/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be
4842/// adjusted to accommodate the arguments for the tailcall.
4843static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
4844 unsigned ParamSize) {
4845
4846 if (!isTailCall) return 0;
4847
4848 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
4849 unsigned CallerMinReservedArea = FI->getMinReservedArea();
4850 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize;
4851 // Remember only if the new adjustment is bigger.
4852 if (SPDiff < FI->getTailCallSPDelta())
4853 FI->setTailCallSPDelta(SPDiff);
4854
4855 return SPDiff;
4856}
4857
4858static bool isFunctionGlobalAddress(const GlobalValue *CalleeGV);
4859
4860static bool callsShareTOCBase(const Function *Caller,
4861 const GlobalValue *CalleeGV,
4862 const TargetMachine &TM) {
4863 // It does not make sense to call callsShareTOCBase() with a caller that
4864 // is PC Relative since PC Relative callers do not have a TOC.
4865#ifndef NDEBUG
4866 const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller);
4867 assert(!STICaller->isUsingPCRelativeCalls() &&
4868 "PC Relative callers do not have a TOC and cannot share a TOC Base");
4869#endif
4870
4871 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
4872 // don't have enough information to determine if the caller and callee share
4873 // the same TOC base, so we have to pessimistically assume they don't for
4874 // correctness.
4875 if (!CalleeGV)
4876 return false;
4877
4878 // If the callee is preemptable, then the static linker will use a plt-stub
4879 // which saves the toc to the stack, and needs a nop after the call
4880 // instruction to convert to a toc-restore.
4881 if (!TM.shouldAssumeDSOLocal(GV: CalleeGV))
4882 return false;
4883
4884 // Functions with PC Relative enabled may clobber the TOC in the same DSO.
4885 // We may need a TOC restore in the situation where the caller requires a
4886 // valid TOC but the callee is PC Relative and does not.
4887 const Function *F = dyn_cast<Function>(Val: CalleeGV);
4888 const GlobalAlias *Alias = dyn_cast<GlobalAlias>(Val: CalleeGV);
4889
4890 // If we have an Alias we can try to get the function from there.
4891 if (Alias) {
4892 const GlobalObject *GlobalObj = Alias->getAliaseeObject();
4893 F = dyn_cast<Function>(Val: GlobalObj);
4894 }
4895
4896 // If we still have no valid function pointer we do not have enough
4897 // information to determine if the callee uses PC Relative calls so we must
4898 // assume that it does.
4899 if (!F)
4900 return false;
4901
4902 // If the callee uses PC Relative we cannot guarantee that the callee won't
4903 // clobber the TOC of the caller and so we must assume that the two
4904 // functions do not share a TOC base.
4905 const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(F: *F);
4906 if (STICallee->isUsingPCRelativeCalls())
4907 return false;
4908
4909 // If the GV is not a strong definition then we need to assume it can be
4910 // replaced by another function at link time. The function that replaces
4911 // it may not share the same TOC as the caller since the callee may be
4912 // replaced by a PC Relative version of the same function.
4913 if (!CalleeGV->isStrongDefinitionForLinker())
4914 return false;
4915
4916 // The medium and large code models are expected to provide a sufficiently
4917 // large TOC to provide all data addressing needs of a module with a
4918 // single TOC.
4919 if (CodeModel::Medium == TM.getCodeModel() ||
4920 CodeModel::Large == TM.getCodeModel())
4921 return true;
4922
4923 // Any explicitly-specified sections and section prefixes must also match.
4924 // Also, if we're using -ffunction-sections, then each function is always in
4925 // a different section (the same is true for COMDAT functions).
4926 if (TM.getFunctionSections() || CalleeGV->hasComdat() ||
4927 Caller->hasComdat() || CalleeGV->getSection() != Caller->getSection())
4928 return false;
4929 if (const auto *F = dyn_cast<Function>(Val: CalleeGV)) {
4930 if (F->getSectionPrefix() != Caller->getSectionPrefix())
4931 return false;
4932 }
4933
4934 return true;
4935}
4936
4937static bool
4938needStackSlotPassParameters(const PPCSubtarget &Subtarget,
4939 const SmallVectorImpl<ISD::OutputArg> &Outs) {
4940 assert(Subtarget.is64BitELFABI());
4941
4942 const unsigned PtrByteSize = 8;
4943 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4944
4945 static const MCPhysReg GPR[] = {
4946 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4947 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4948 };
4949 static const MCPhysReg VR[] = {
4950 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4951 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4952 };
4953
4954 const unsigned NumGPRs = std::size(GPR);
4955 const unsigned NumFPRs = 13;
4956 const unsigned NumVRs = std::size(VR);
4957 const unsigned ParamAreaSize = NumGPRs * PtrByteSize;
4958
4959 unsigned NumBytes = LinkageSize;
4960 unsigned AvailableFPRs = NumFPRs;
4961 unsigned AvailableVRs = NumVRs;
4962
4963 for (const ISD::OutputArg& Param : Outs) {
4964 if (Param.Flags.isNest()) continue;
4965
4966 if (CalculateStackSlotUsed(ArgVT: Param.VT, OrigVT: Param.ArgVT, Flags: Param.Flags, PtrByteSize,
4967 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4968 AvailableFPRs, AvailableVRs))
4969 return true;
4970 }
4971 return false;
4972}
4973
4974static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) {
4975 if (CB.arg_size() != CallerFn->arg_size())
4976 return false;
4977
4978 auto CalleeArgIter = CB.arg_begin();
4979 auto CalleeArgEnd = CB.arg_end();
4980 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin();
4981
4982 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) {
4983 const Value* CalleeArg = *CalleeArgIter;
4984 const Value* CallerArg = &(*CallerArgIter);
4985 if (CalleeArg == CallerArg)
4986 continue;
4987
4988 // e.g. @caller([4 x i64] %a, [4 x i64] %b) {
4989 // tail call @callee([4 x i64] undef, [4 x i64] %b)
4990 // }
4991 // 1st argument of callee is undef and has the same type as caller.
4992 if (CalleeArg->getType() == CallerArg->getType() &&
4993 isa<UndefValue>(Val: CalleeArg))
4994 continue;
4995
4996 return false;
4997 }
4998
4999 return true;
5000}
5001
5002// Returns true if TCO is possible between the callers and callees
5003// calling conventions.
5004static bool
5005areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC,
5006 CallingConv::ID CalleeCC) {
5007 // Tail calls are possible with fastcc and ccc.
5008 auto isTailCallableCC = [] (CallingConv::ID CC){
5009 return CC == CallingConv::C || CC == CallingConv::Fast;
5010 };
5011 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC))
5012 return false;
5013
5014 // We can safely tail call both fastcc and ccc callees from a c calling
5015 // convention caller. If the caller is fastcc, we may have less stack space
5016 // than a non-fastcc caller with the same signature so disable tail-calls in
5017 // that case.
5018 return CallerCC == CallingConv::C || CallerCC == CalleeCC;
5019}
5020
5021bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
5022 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5023 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5024 const SmallVectorImpl<ISD::OutputArg> &Outs,
5025 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5026 bool isCalleeExternalSymbol) const {
5027 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
5028
5029 if (!EnableSCO && !TailCallOpt)
5030 return false;
5031
5032 // Variadic argument functions are not supported.
5033 if (isVarArg) return false;
5034
5035 // Check that the calling conventions are compatible for tco.
5036 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC, CalleeCC))
5037 return false;
5038
5039 // Caller contains any byval parameter is not supported.
5040 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5041 return false;
5042
5043 // Callee contains any byval parameter is not supported, too.
5044 // Note: This is a quick work around, because in some cases, e.g.
5045 // caller's stack size > callee's stack size, we are still able to apply
5046 // sibling call optimization. For example, gcc is able to do SCO for caller1
5047 // in the following example, but not for caller2.
5048 // struct test {
5049 // long int a;
5050 // char ary[56];
5051 // } gTest;
5052 // __attribute__((noinline)) int callee(struct test v, struct test *b) {
5053 // b->a = v.a;
5054 // return 0;
5055 // }
5056 // void caller1(struct test a, struct test c, struct test *b) {
5057 // callee(gTest, b); }
5058 // void caller2(struct test *b) { callee(gTest, b); }
5059 if (any_of(Range: Outs, P: [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); }))
5060 return false;
5061
5062 // If callee and caller use different calling conventions, we cannot pass
5063 // parameters on stack since offsets for the parameter area may be different.
5064 if (CallerCC != CalleeCC && needStackSlotPassParameters(Subtarget, Outs))
5065 return false;
5066
5067 // All variants of 64-bit ELF ABIs without PC-Relative addressing require that
5068 // the caller and callee share the same TOC for TCO/SCO. If the caller and
5069 // callee potentially have different TOC bases then we cannot tail call since
5070 // we need to restore the TOC pointer after the call.
5071 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977
5072 // We cannot guarantee this for indirect calls or calls to external functions.
5073 // When PC-Relative addressing is used, the concept of the TOC is no longer
5074 // applicable so this check is not required.
5075 // Check first for indirect calls.
5076 if (!Subtarget.isUsingPCRelativeCalls() &&
5077 !isFunctionGlobalAddress(CalleeGV) && !isCalleeExternalSymbol)
5078 return false;
5079
5080 // Check if we share the TOC base.
5081 if (!Subtarget.isUsingPCRelativeCalls() &&
5082 !callsShareTOCBase(Caller: CallerFunc, CalleeGV, TM: getTargetMachine()))
5083 return false;
5084
5085 // TCO allows altering callee ABI, so we don't have to check further.
5086 if (CalleeCC == CallingConv::Fast && TailCallOpt)
5087 return true;
5088
5089 if (!EnableSCO)
5090 return false;
5091
5092 // If callee use the same argument list that caller is using, then we can
5093 // apply SCO on this case. If it is not, then we need to check if callee needs
5094 // stack for passing arguments.
5095 // PC Relative tail calls may not have a CallBase.
5096 // If there is no CallBase we cannot verify if we have the same argument
5097 // list so assume that we don't have the same argument list.
5098 if (CB && !hasSameArgumentList(CallerFn: CallerFunc, CB: *CB) &&
5099 needStackSlotPassParameters(Subtarget, Outs))
5100 return false;
5101 else if (!CB && needStackSlotPassParameters(Subtarget, Outs))
5102 return false;
5103
5104 return true;
5105}
5106
5107/// IsEligibleForTailCallOptimization - Check whether the call is eligible
5108/// for tail call optimization. Targets which want to do tail call
5109/// optimization should implement this function.
5110bool PPCTargetLowering::IsEligibleForTailCallOptimization(
5111 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5112 CallingConv::ID CallerCC, bool isVarArg,
5113 const SmallVectorImpl<ISD::InputArg> &Ins) const {
5114 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5115 return false;
5116
5117 // Variable argument functions are not supported.
5118 if (isVarArg)
5119 return false;
5120
5121 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) {
5122 // Functions containing by val parameters are not supported.
5123 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5124 return false;
5125
5126 // Non-PIC/GOT tail calls are supported.
5127 if (getTargetMachine().getRelocationModel() != Reloc::PIC_)
5128 return true;
5129
5130 // At the moment we can only do local tail calls (in same module, hidden
5131 // or protected) if we are generating PIC.
5132 if (CalleeGV)
5133 return CalleeGV->hasHiddenVisibility() ||
5134 CalleeGV->hasProtectedVisibility();
5135 }
5136
5137 return false;
5138}
5139
5140/// isCallCompatibleAddress - Return the immediate to use if the specified
5141/// 32-bit value is representable in the immediate field of a BxA instruction.
5142static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
5143 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op);
5144 if (!C) return nullptr;
5145
5146 int Addr = C->getZExtValue();
5147 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
5148 SignExtend32<26>(X: Addr) != Addr)
5149 return nullptr; // Top 6 bits have to be sext of immediate.
5150
5151 return DAG
5152 .getSignedConstant(
5153 Val: (int)C->getZExtValue() >> 2, DL: SDLoc(Op),
5154 VT: DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout()))
5155 .getNode();
5156}
5157
5158namespace {
5159
5160struct TailCallArgumentInfo {
5161 SDValue Arg;
5162 SDValue FrameIdxOp;
5163 int FrameIdx = 0;
5164
5165 TailCallArgumentInfo() = default;
5166};
5167
5168} // end anonymous namespace
5169
5170/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot.
5171static void StoreTailCallArgumentsToStackSlot(
5172 SelectionDAG &DAG, SDValue Chain,
5173 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs,
5174 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) {
5175 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) {
5176 SDValue Arg = TailCallArgs[i].Arg;
5177 SDValue FIN = TailCallArgs[i].FrameIdxOp;
5178 int FI = TailCallArgs[i].FrameIdx;
5179 // Store relative to framepointer.
5180 MemOpChains.push_back(Elt: DAG.getStore(
5181 Chain, dl, Val: Arg, Ptr: FIN,
5182 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI)));
5183 }
5184}
5185
5186/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to
5187/// the appropriate stack slot for the tail call optimized function call.
5188static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
5189 SDValue OldRetAddr, SDValue OldFP,
5190 int SPDiff, const SDLoc &dl) {
5191 if (SPDiff) {
5192 // Calculate the new stack slot for the return address.
5193 MachineFunction &MF = DAG.getMachineFunction();
5194 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
5195 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
5196 int SlotSize = Subtarget.isPPC64() ? 8 : 4;
5197 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset();
5198 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(Size: SlotSize,
5199 SPOffset: NewRetAddrLoc, IsImmutable: true);
5200 SDValue NewRetAddrFrIdx =
5201 DAG.getFrameIndex(FI: NewRetAddr, VT: Subtarget.getScalarIntVT());
5202 Chain = DAG.getStore(Chain, dl, Val: OldRetAddr, Ptr: NewRetAddrFrIdx,
5203 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: NewRetAddr));
5204 }
5205 return Chain;
5206}
5207
5208/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate
5209/// the position of the argument.
5210static void CalculateTailCallArgDest(
5211 SelectionDAG &DAG, MachineFunction &MF, bool IsPPC64, SDValue Arg,
5212 int SPDiff, unsigned ArgOffset,
5213 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5214 int Offset = ArgOffset + SPDiff;
5215 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8;
5216 int FI = MF.getFrameInfo().CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
5217 EVT VT = IsPPC64 ? MVT::i64 : MVT::i32;
5218 SDValue FIN = DAG.getFrameIndex(FI, VT);
5219 TailCallArgumentInfo Info;
5220 Info.Arg = Arg;
5221 Info.FrameIdxOp = FIN;
5222 Info.FrameIdx = FI;
5223 TailCallArguments.push_back(Elt: Info);
5224}
5225
5226/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address
5227/// stack slot. Returns the chain as result and the loaded frame pointers in
5228/// LROpOut/FPOpout. Used when tail calling.
5229SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(
5230 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
5231 SDValue &FPOpOut, const SDLoc &dl) const {
5232 if (SPDiff) {
5233 // Load the LR and FP stack slot for later adjusting.
5234 LROpOut = getReturnAddrFrameIndex(DAG);
5235 LROpOut = DAG.getLoad(VT: Subtarget.getScalarIntVT(), dl, Chain, Ptr: LROpOut,
5236 PtrInfo: MachinePointerInfo());
5237 Chain = SDValue(LROpOut.getNode(), 1);
5238 }
5239 return Chain;
5240}
5241
5242/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
5243/// by "Src" to address "Dst" of size "Size". Alignment information is
5244/// specified by the specific parameter attribute. The copy will be passed as
5245/// a byval function parameter.
5246/// Sometimes what we are copying is the end of a larger object, the part that
5247/// does not fit in registers.
5248static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
5249 SDValue Chain, ISD::ArgFlagsTy Flags,
5250 SelectionDAG &DAG, const SDLoc &dl) {
5251 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
5252 Align Alignment = Flags.getNonZeroByValAlign();
5253 return DAG.getMemcpy(
5254 Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment, isVol: false, AlwaysInline: false,
5255 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
5256}
5257
5258/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
5259/// tail calls.
5260static void LowerMemOpCallTo(
5261 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
5262 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64,
5263 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
5264 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) {
5265 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5266 if (!isTailCall) {
5267 if (isVector) {
5268 SDValue StackPtr;
5269 if (isPPC64)
5270 StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
5271 else
5272 StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
5273 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr,
5274 N2: DAG.getConstant(Val: ArgOffset, DL: dl, VT: PtrVT));
5275 }
5276 MemOpChains.push_back(
5277 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
5278 // Calculate and remember argument location.
5279 } else
5280 CalculateTailCallArgDest(DAG, MF, IsPPC64: isPPC64, Arg, SPDiff, ArgOffset,
5281 TailCallArguments);
5282}
5283
5284static void
5285PrepareTailCall(SelectionDAG &DAG, SDValue &InGlue, SDValue &Chain,
5286 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp,
5287 SDValue FPOp,
5288 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5289 // Emit a sequence of copyto/copyfrom virtual registers for arguments that
5290 // might overwrite each other in case of tail call optimization.
5291 SmallVector<SDValue, 8> MemOpChains2;
5292 // Do not flag preceding copytoreg stuff together with the following stuff.
5293 InGlue = SDValue();
5294 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArgs: TailCallArguments,
5295 MemOpChains&: MemOpChains2, dl);
5296 if (!MemOpChains2.empty())
5297 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains2);
5298
5299 // Store the return address to the appropriate stack slot.
5300 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, OldRetAddr: LROp, OldFP: FPOp, SPDiff, dl);
5301
5302 // Emit callseq_end just before tailcall node.
5303 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL: dl);
5304 InGlue = Chain.getValue(R: 1);
5305}
5306
5307// Is this global address that of a function that can be called by name? (as
5308// opposed to something that must hold a descriptor for an indirect call).
5309static bool isFunctionGlobalAddress(const GlobalValue *GV) {
5310 if (GV) {
5311 if (GV->isThreadLocal())
5312 return false;
5313
5314 return GV->getValueType()->isFunctionTy();
5315 }
5316
5317 return false;
5318}
5319
5320SDValue PPCTargetLowering::LowerCallResult(
5321 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
5322 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5323 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
5324 SmallVector<CCValAssign, 16> RVLocs;
5325 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5326 *DAG.getContext());
5327
5328 CCRetInfo.AnalyzeCallResult(
5329 Ins, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
5330 ? RetCC_PPC_Cold
5331 : RetCC_PPC);
5332
5333 // Copy all of the result registers out of their specified physreg.
5334 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
5335 CCValAssign &VA = RVLocs[i];
5336 assert(VA.isRegLoc() && "Can only return in registers!");
5337
5338 SDValue Val;
5339
5340 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
5341 SDValue Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5342 Glue: InGlue);
5343 Chain = Lo.getValue(R: 1);
5344 InGlue = Lo.getValue(R: 2);
5345 VA = RVLocs[++i]; // skip ahead to next loc
5346 SDValue Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5347 Glue: InGlue);
5348 Chain = Hi.getValue(R: 1);
5349 InGlue = Hi.getValue(R: 2);
5350 if (!Subtarget.isLittleEndian())
5351 std::swap (a&: Lo, b&: Hi);
5352 Val = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
5353 } else {
5354 Val = DAG.getCopyFromReg(Chain, dl,
5355 Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
5356 Chain = Val.getValue(R: 1);
5357 InGlue = Val.getValue(R: 2);
5358 }
5359
5360 switch (VA.getLocInfo()) {
5361 default: llvm_unreachable("Unknown loc info!");
5362 case CCValAssign::Full: break;
5363 case CCValAssign::AExt:
5364 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5365 break;
5366 case CCValAssign::ZExt:
5367 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: VA.getLocVT(), N1: Val,
5368 N2: DAG.getValueType(VA.getValVT()));
5369 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5370 break;
5371 case CCValAssign::SExt:
5372 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: VA.getLocVT(), N1: Val,
5373 N2: DAG.getValueType(VA.getValVT()));
5374 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5375 break;
5376 }
5377
5378 InVals.push_back(Elt: Val);
5379 }
5380
5381 return Chain;
5382}
5383
5384static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG,
5385 const PPCSubtarget &Subtarget, bool isPatchPoint) {
5386 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5387 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5388
5389 // PatchPoint calls are not indirect.
5390 if (isPatchPoint)
5391 return false;
5392
5393 if (isFunctionGlobalAddress(GV) || isa<ExternalSymbolSDNode>(Val: Callee))
5394 return false;
5395
5396 // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not
5397 // becuase the immediate function pointer points to a descriptor instead of
5398 // a function entry point. The ELFv2 ABI cannot use a BLA because the function
5399 // pointer immediate points to the global entry point, while the BLA would
5400 // need to jump to the local entry point (see rL211174).
5401 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() &&
5402 isBLACompatibleAddress(Op: Callee, DAG))
5403 return false;
5404
5405 return true;
5406}
5407
5408// AIX and 64-bit ELF ABIs w/o PCRel require a TOC save/restore around calls.
5409static inline bool isTOCSaveRestoreRequired(const PPCSubtarget &Subtarget) {
5410 return Subtarget.isAIXABI() ||
5411 (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls());
5412}
5413
5414static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags,
5415 const Function &Caller, const SDValue &Callee,
5416 const PPCSubtarget &Subtarget,
5417 const TargetMachine &TM,
5418 bool IsStrictFPCall = false) {
5419 if (CFlags.IsTailCall)
5420 return PPCISD::TC_RETURN;
5421
5422 unsigned RetOpc = 0;
5423 // This is a call through a function pointer.
5424 if (CFlags.IsIndirect) {
5425 // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross
5426 // indirect calls. The save of the caller's TOC pointer to the stack will be
5427 // inserted into the DAG as part of call lowering. The restore of the TOC
5428 // pointer is modeled by using a pseudo instruction for the call opcode that
5429 // represents the 2 instruction sequence of an indirect branch and link,
5430 // immediately followed by a load of the TOC pointer from the stack save
5431 // slot into gpr2. For 64-bit ELFv2 ABI with PCRel, do not restore the TOC
5432 // as it is not saved or used.
5433 if (Subtarget.usePointerGlueHelper())
5434 RetOpc = PPCISD::BL_LOAD_TOC;
5435 else
5436 RetOpc = isTOCSaveRestoreRequired(Subtarget) ? PPCISD::BCTRL_LOAD_TOC
5437 : PPCISD::BCTRL;
5438 } else if (Subtarget.isUsingPCRelativeCalls()) {
5439 assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI.");
5440 RetOpc = PPCISD::CALL_NOTOC;
5441 } else if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) {
5442 // The ABIs that maintain a TOC pointer accross calls need to have a nop
5443 // immediately following the call instruction if the caller and callee may
5444 // have different TOC bases. At link time if the linker determines the calls
5445 // may not share a TOC base, the call is redirected to a trampoline inserted
5446 // by the linker. The trampoline will (among other things) save the callers
5447 // TOC pointer at an ABI designated offset in the linkage area and the
5448 // linker will rewrite the nop to be a load of the TOC pointer from the
5449 // linkage area into gpr2.
5450 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5451 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5452 RetOpc =
5453 callsShareTOCBase(Caller: &Caller, CalleeGV: GV, TM) ? PPCISD::CALL : PPCISD::CALL_NOP;
5454 } else
5455 RetOpc = PPCISD::CALL;
5456 if (IsStrictFPCall) {
5457 switch (RetOpc) {
5458 default:
5459 llvm_unreachable("Unknown call opcode");
5460 case PPCISD::BCTRL_LOAD_TOC:
5461 RetOpc = PPCISD::BCTRL_LOAD_TOC_RM;
5462 break;
5463 case PPCISD::BCTRL:
5464 RetOpc = PPCISD::BCTRL_RM;
5465 break;
5466 case PPCISD::BL_LOAD_TOC:
5467 RetOpc = PPCISD::BL_LOAD_TOC_RM;
5468 break;
5469 case PPCISD::CALL_NOTOC:
5470 RetOpc = PPCISD::CALL_NOTOC_RM;
5471 break;
5472 case PPCISD::CALL:
5473 RetOpc = PPCISD::CALL_RM;
5474 break;
5475 case PPCISD::CALL_NOP:
5476 RetOpc = PPCISD::CALL_NOP_RM;
5477 break;
5478 }
5479 }
5480 return RetOpc;
5481}
5482
5483static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
5484 const SDLoc &dl, const PPCSubtarget &Subtarget) {
5485 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI())
5486 if (SDNode *Dest = isBLACompatibleAddress(Op: Callee, DAG))
5487 return SDValue(Dest, 0);
5488
5489 // Returns true if the callee is local, and false otherwise.
5490 auto isLocalCallee = [&]() {
5491 const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5492 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5493
5494 return DAG.getTarget().shouldAssumeDSOLocal(GV) &&
5495 !isa_and_nonnull<GlobalIFunc>(Val: GV);
5496 };
5497
5498 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in
5499 // a static relocation model causes some versions of GNU LD (2.17.50, at
5500 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are
5501 // built with secure-PLT.
5502 bool UsePlt =
5503 Subtarget.is32BitELFABI() && !isLocalCallee() &&
5504 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_;
5505
5506 const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) {
5507 const TargetMachine &TM = Subtarget.getTargetMachine();
5508 const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering();
5509 auto *S =
5510 static_cast<MCSymbolXCOFF *>(TLOF->getFunctionEntryPointSymbol(Func: GV, TM));
5511
5512 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5513 return DAG.getMCSymbol(Sym: S, VT: PtrVT);
5514 };
5515
5516 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5517 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5518 if (isFunctionGlobalAddress(GV)) {
5519 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val: Callee)->getGlobal();
5520
5521 if (Subtarget.isAIXABI()) {
5522 return getAIXFuncEntryPointSymbolSDNode(GV);
5523 }
5524 return DAG.getTargetGlobalAddress(GV, DL: dl, VT: Callee.getValueType(), offset: 0,
5525 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5526 }
5527
5528 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val: Callee)) {
5529 const char *SymName = S->getSymbol();
5530 if (Subtarget.isAIXABI()) {
5531 // If there exists a user-declared function whose name is the same as the
5532 // ExternalSymbol's, then we pick up the user-declared version.
5533 const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
5534 if (const Function *F =
5535 dyn_cast_or_null<Function>(Val: Mod->getNamedValue(Name: SymName)))
5536 return getAIXFuncEntryPointSymbolSDNode(F);
5537
5538 // On AIX, direct function calls reference the symbol for the function's
5539 // entry point, which is named by prepending a "." before the function's
5540 // C-linkage name. A Qualname is returned here because an external
5541 // function entry point is a csect with XTY_ER property.
5542 const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) {
5543 auto &Context = DAG.getMachineFunction().getContext();
5544 MCSectionXCOFF *Sec = Context.getXCOFFSection(
5545 Section: (Twine(".") + Twine(SymName)).str(), K: SectionKind::getMetadata(),
5546 CsectProp: XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER));
5547 return Sec->getQualNameSymbol();
5548 };
5549
5550 SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data();
5551 }
5552 return DAG.getTargetExternalSymbol(Sym: SymName, VT: Callee.getValueType(),
5553 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5554 }
5555
5556 // No transformation needed.
5557 assert(Callee.getNode() && "What no callee?");
5558 return Callee;
5559}
5560
5561static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) {
5562 assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START &&
5563 "Expected a CALLSEQ_STARTSDNode.");
5564
5565 // The last operand is the chain, except when the node has glue. If the node
5566 // has glue, then the last operand is the glue, and the chain is the second
5567 // last operand.
5568 SDValue LastValue = CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 1);
5569 if (LastValue.getValueType() != MVT::Glue)
5570 return LastValue;
5571
5572 return CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 2);
5573}
5574
5575// Creates the node that moves a functions address into the count register
5576// to prepare for an indirect call instruction.
5577static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5578 SDValue &Glue, SDValue &Chain,
5579 const SDLoc &dl) {
5580 SDValue MTCTROps[] = {Chain, Callee, Glue};
5581 EVT ReturnTypes[] = {MVT::Other, MVT::Glue};
5582 Chain = DAG.getNode(Opcode: PPCISD::MTCTR, DL: dl, ResultTys: ReturnTypes,
5583 Ops: ArrayRef(MTCTROps, Glue.getNode() ? 3 : 2));
5584 // The glue is the second value produced.
5585 Glue = Chain.getValue(R: 1);
5586}
5587
5588static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5589 SDValue &Glue, SDValue &Chain,
5590 SDValue CallSeqStart,
5591 const CallBase *CB, const SDLoc &dl,
5592 bool hasNest,
5593 const PPCSubtarget &Subtarget) {
5594 // Function pointers in the 64-bit SVR4 ABI do not point to the function
5595 // entry point, but to the function descriptor (the function entry point
5596 // address is part of the function descriptor though).
5597 // The function descriptor is a three doubleword structure with the
5598 // following fields: function entry point, TOC base address and
5599 // environment pointer.
5600 // Thus for a call through a function pointer, the following actions need
5601 // to be performed:
5602 // 1. Save the TOC of the caller in the TOC save area of its stack
5603 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
5604 // 2. Load the address of the function entry point from the function
5605 // descriptor.
5606 // 3. Load the TOC of the callee from the function descriptor into r2.
5607 // 4. Load the environment pointer from the function descriptor into
5608 // r11.
5609 // 5. Branch to the function entry point address.
5610 // 6. On return of the callee, the TOC of the caller needs to be
5611 // restored (this is done in FinishCall()).
5612 //
5613 // The loads are scheduled at the beginning of the call sequence, and the
5614 // register copies are flagged together to ensure that no other
5615 // operations can be scheduled in between. E.g. without flagging the
5616 // copies together, a TOC access in the caller could be scheduled between
5617 // the assignment of the callee TOC and the branch to the callee, which leads
5618 // to incorrect code.
5619
5620 // Start by loading the function address from the descriptor.
5621 SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart);
5622 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
5623 ? (MachineMemOperand::MODereferenceable |
5624 MachineMemOperand::MOInvariant)
5625 : MachineMemOperand::MONone;
5626
5627 MachinePointerInfo MPI(CB ? CB->getCalledOperand() : nullptr);
5628
5629 // Registers used in building the DAG.
5630 const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister();
5631 const MCRegister TOCReg = Subtarget.getTOCPointerRegister();
5632
5633 // Offsets of descriptor members.
5634 const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset();
5635 const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset();
5636
5637 const MVT RegVT = Subtarget.getScalarIntVT();
5638 const Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
5639
5640 // One load for the functions entry point address.
5641 SDValue LoadFuncPtr = DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: Callee, PtrInfo: MPI,
5642 Alignment, MMOFlags);
5643
5644 // One for loading the TOC anchor for the module that contains the called
5645 // function.
5646 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCAnchorOffset, DL: dl);
5647 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: TOCOff);
5648 SDValue TOCPtr =
5649 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddTOC,
5650 PtrInfo: MPI.getWithOffset(O: TOCAnchorOffset), Alignment, MMOFlags);
5651
5652 // One for loading the environment pointer.
5653 SDValue PtrOff = DAG.getIntPtrConstant(Val: EnvPtrOffset, DL: dl);
5654 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: PtrOff);
5655 SDValue LoadEnvPtr =
5656 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddPtr,
5657 PtrInfo: MPI.getWithOffset(O: EnvPtrOffset), Alignment, MMOFlags);
5658
5659
5660 // Then copy the newly loaded TOC anchor to the TOC pointer.
5661 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, Reg: TOCReg, N: TOCPtr, Glue);
5662 Chain = TOCVal.getValue(R: 0);
5663 Glue = TOCVal.getValue(R: 1);
5664
5665 // If the function call has an explicit 'nest' parameter, it takes the
5666 // place of the environment pointer.
5667 assert((!hasNest || !Subtarget.isAIXABI()) &&
5668 "Nest parameter is not supported on AIX.");
5669 if (!hasNest) {
5670 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, Reg: EnvPtrReg, N: LoadEnvPtr, Glue);
5671 Chain = EnvVal.getValue(R: 0);
5672 Glue = EnvVal.getValue(R: 1);
5673 }
5674
5675 // The rest of the indirect call sequence is the same as the non-descriptor
5676 // DAG.
5677 prepareIndirectCall(DAG, Callee&: LoadFuncPtr, Glue, Chain, dl);
5678}
5679
5680static void prepareOutOfLineGlueCall(SelectionDAG &DAG, SDValue &Callee,
5681 SDValue &Glue, SDValue &Chain,
5682 SDValue CallSeqStart, const CallBase *CB,
5683 const SDLoc &dl, bool hasNest,
5684 const PPCSubtarget &Subtarget) {
5685 // On AIX there is a feature ("out of line glue code") which uses a special
5686 // trampoline function ._ptrgl to do the indirect call. If this option is
5687 // enabled we instead simply load the address of the descriptor into gpr11,
5688 // with the arguments in the 'normal' registers and branch to the ._ptrgl
5689 // stub.
5690 const MCRegister PtrGlueReg = Subtarget.getGlueCodeDescriptorRegister();
5691 SDValue MoveToPhysicalReg =
5692 DAG.getCopyToReg(Chain, dl, Reg: PtrGlueReg, N: Callee, Glue);
5693 Chain = MoveToPhysicalReg.getValue(R: 0);
5694 Glue = MoveToPhysicalReg.getValue(R: 1);
5695}
5696
5697static void
5698buildCallOperands(SmallVectorImpl<SDValue> &Ops,
5699 PPCTargetLowering::CallFlags CFlags, const SDLoc &dl,
5700 SelectionDAG &DAG,
5701 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass,
5702 SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff,
5703 const PPCSubtarget &Subtarget) {
5704 const bool IsPPC64 = Subtarget.isPPC64();
5705 // MVT for a general purpose register.
5706 const MVT RegVT = Subtarget.getScalarIntVT();
5707
5708 // First operand is always the chain.
5709 Ops.push_back(Elt: Chain);
5710
5711 // If it's a direct call pass the callee as the second operand.
5712 if (!CFlags.IsIndirect)
5713 Ops.push_back(Elt: Callee);
5714 else if (Subtarget.usePointerGlueHelper()) {
5715 Ops.push_back(Elt: Callee);
5716 // Add the register used to pass the descriptor address.
5717 Ops.push_back(
5718 Elt: DAG.getRegister(Reg: Subtarget.getGlueCodeDescriptorRegister(), VT: RegVT));
5719 } else {
5720 assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect.");
5721
5722 // For the TOC based ABIs, we have saved the TOC pointer to the linkage area
5723 // on the stack (this would have been done in `LowerCall_64SVR4` or
5724 // `LowerCall_AIX`). The call instruction is a pseudo instruction that
5725 // represents both the indirect branch and a load that restores the TOC
5726 // pointer from the linkage area. The operand for the TOC restore is an add
5727 // of the TOC save offset to the stack pointer. This must be the second
5728 // operand: after the chain input but before any other variadic arguments.
5729 // For 64-bit ELFv2 ABI with PCRel, do not restore the TOC as it is not
5730 // saved or used.
5731 if (isTOCSaveRestoreRequired(Subtarget)) {
5732 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
5733
5734 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: RegVT);
5735 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5736 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
5737 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: StackPtr, N2: TOCOff);
5738 Ops.push_back(Elt: AddTOC);
5739 }
5740
5741 // Add the register used for the environment pointer.
5742 if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest)
5743 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getEnvironmentPointerRegister(),
5744 VT: RegVT));
5745
5746
5747 // Add CTR register as callee so a bctr can be emitted later.
5748 if (CFlags.IsTailCall)
5749 Ops.push_back(Elt: DAG.getRegister(Reg: IsPPC64 ? PPC::CTR8 : PPC::CTR, VT: RegVT));
5750 }
5751
5752 // If this is a tail call add stack pointer delta.
5753 if (CFlags.IsTailCall)
5754 Ops.push_back(Elt: DAG.getConstant(Val: SPDiff, DL: dl, VT: MVT::i32));
5755
5756 // Add argument registers to the end of the list so that they are known live
5757 // into the call.
5758 for (const auto &[Reg, N] : RegsToPass)
5759 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
5760
5761 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
5762 // no way to mark dependencies as implicit here.
5763 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter.
5764 if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) &&
5765 !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls())
5766 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getTOCPointerRegister(), VT: RegVT));
5767
5768 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
5769 if (CFlags.IsVarArg && Subtarget.is32BitELFABI())
5770 Ops.push_back(Elt: DAG.getRegister(Reg: PPC::CR1EQ, VT: MVT::i32));
5771
5772 // Add a register mask operand representing the call-preserved registers.
5773 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
5774 const uint32_t *Mask =
5775 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CFlags.CallConv);
5776 assert(Mask && "Missing call preserved mask for calling convention");
5777 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
5778
5779 // If the glue is valid, it is the last operand.
5780 if (Glue.getNode())
5781 Ops.push_back(Elt: Glue);
5782}
5783
5784SDValue PPCTargetLowering::FinishCall(
5785 CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG,
5786 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue,
5787 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5788 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins,
5789 SmallVectorImpl<SDValue> &InVals, const CallBase *CB) const {
5790
5791 if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) ||
5792 Subtarget.isAIXABI())
5793 setUsesTOCBasePtr(DAG);
5794
5795 unsigned CallOpc =
5796 getCallOpcode(CFlags, Caller: DAG.getMachineFunction().getFunction(), Callee,
5797 Subtarget, TM: DAG.getTarget(), IsStrictFPCall: CB ? CB->isStrictFP() : false);
5798
5799 if (!CFlags.IsIndirect)
5800 Callee = transformCallee(Callee, DAG, dl, Subtarget);
5801 else if (Subtarget.usesFunctionDescriptors()) {
5802 if (Subtarget.usePointerGlueHelper()) {
5803 prepareOutOfLineGlueCall(DAG, Callee, Glue, Chain, CallSeqStart, CB, dl,
5804 hasNest: CFlags.HasNest, Subtarget);
5805 SDValue PtrGlueCallee =
5806 DAG.getExternalSymbol(Sym: "_ptrgl", VT: getPointerTy(DL: DAG.getDataLayout()));
5807 Callee = transformCallee(Callee: PtrGlueCallee, DAG, dl, Subtarget);
5808 } else {
5809 prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CB,
5810 dl, hasNest: CFlags.HasNest, Subtarget);
5811 }
5812 } else {
5813 prepareIndirectCall(DAG, Callee, Glue, Chain, dl);
5814 }
5815
5816 // Build the operand list for the call instruction.
5817 SmallVector<SDValue, 8> Ops;
5818 buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee,
5819 SPDiff, Subtarget);
5820
5821 // Emit tail call.
5822 if (CFlags.IsTailCall) {
5823 // Indirect tail call when using PC Relative calls do not have the same
5824 // constraints.
5825 assert(((Callee.getOpcode() == ISD::Register &&
5826 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) ||
5827 Callee.getOpcode() == ISD::TargetExternalSymbol ||
5828 Callee.getOpcode() == ISD::TargetGlobalAddress ||
5829 isa<ConstantSDNode>(Callee) ||
5830 (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) &&
5831 "Expecting a global address, external symbol, absolute value, "
5832 "register or an indirect tail call when PC Relative calls are "
5833 "used.");
5834 // PC Relative calls also use TC_RETURN as the way to mark tail calls.
5835 assert(CallOpc == PPCISD::TC_RETURN &&
5836 "Unexpected call opcode for a tail call.");
5837 DAG.getMachineFunction().getFrameInfo().setHasTailCall();
5838 SDValue Ret = DAG.getNode(Opcode: CallOpc, DL: dl, VT: MVT::Other, Ops);
5839 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CFlags.NoMerge);
5840 return Ret;
5841 }
5842
5843 std::array<EVT, 2> ReturnTypes = {._M_elems: {MVT::Other, MVT::Glue}};
5844 Chain = DAG.getNode(Opcode: CallOpc, DL: dl, ResultTys: ReturnTypes, Ops);
5845 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CFlags.NoMerge);
5846 Glue = Chain.getValue(R: 1);
5847
5848 // When performing tail call optimization the callee pops its arguments off
5849 // the stack. Account for this here so these bytes can be pushed back on in
5850 // PPCFrameLowering::eliminateCallFramePseudoInstr.
5851 int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast &&
5852 getTargetMachine().Options.GuaranteedTailCallOpt)
5853 ? NumBytes
5854 : 0;
5855
5856 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: BytesCalleePops, Glue, DL: dl);
5857 Glue = Chain.getValue(R: 1);
5858
5859 return LowerCallResult(Chain, InGlue: Glue, CallConv: CFlags.CallConv, isVarArg: CFlags.IsVarArg, Ins, dl,
5860 DAG, InVals);
5861}
5862
5863bool PPCTargetLowering::supportsTailCallFor(const CallBase *CB) const {
5864 CallingConv::ID CalleeCC = CB->getCallingConv();
5865 const Function *CallerFunc = CB->getCaller();
5866 CallingConv::ID CallerCC = CallerFunc->getCallingConv();
5867 const Function *CalleeFunc = CB->getCalledFunction();
5868 if (!CalleeFunc)
5869 return false;
5870 const GlobalValue *CalleeGV = dyn_cast<GlobalValue>(Val: CalleeFunc);
5871
5872 SmallVector<ISD::OutputArg, 2> Outs;
5873 SmallVector<ISD::InputArg, 2> Ins;
5874
5875 GetReturnInfo(CC: CalleeCC, ReturnType: CalleeFunc->getReturnType(),
5876 attr: CalleeFunc->getAttributes(), Outs, TLI: *this,
5877 DL: CalleeFunc->getDataLayout());
5878
5879 return isEligibleForTCO(CalleeGV, CalleeCC, CallerCC, CB,
5880 isVarArg: CalleeFunc->isVarArg(), Outs, Ins, CallerFunc,
5881 isCalleeExternalSymbol: false /*isCalleeExternalSymbol*/);
5882}
5883
5884bool PPCTargetLowering::isEligibleForTCO(
5885 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5886 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5887 const SmallVectorImpl<ISD::OutputArg> &Outs,
5888 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5889 bool isCalleeExternalSymbol) const {
5890 if (Subtarget.useLongCalls() && !(CB && CB->isMustTailCall()))
5891 return false;
5892
5893 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64())
5894 return IsEligibleForTailCallOptimization_64SVR4(
5895 CalleeGV, CalleeCC, CallerCC, CB, isVarArg, Outs, Ins, CallerFunc,
5896 isCalleeExternalSymbol);
5897 else
5898 return IsEligibleForTailCallOptimization(CalleeGV, CalleeCC, CallerCC,
5899 isVarArg, Ins);
5900}
5901
5902SDValue
5903PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
5904 SmallVectorImpl<SDValue> &InVals) const {
5905 SelectionDAG &DAG = CLI.DAG;
5906 SDLoc &dl = CLI.DL;
5907 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
5908 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
5909 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
5910 SDValue Chain = CLI.Chain;
5911 SDValue Callee = CLI.Callee;
5912 bool &isTailCall = CLI.IsTailCall;
5913 CallingConv::ID CallConv = CLI.CallConv;
5914 bool isVarArg = CLI.IsVarArg;
5915 bool isPatchPoint = CLI.IsPatchPoint;
5916 const CallBase *CB = CLI.CB;
5917
5918 if (isTailCall) {
5919 MachineFunction &MF = DAG.getMachineFunction();
5920 CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
5921 auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee);
5922 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5923 bool IsCalleeExternalSymbol = isa<ExternalSymbolSDNode>(Val: Callee);
5924
5925 isTailCall =
5926 isEligibleForTCO(CalleeGV: GV, CalleeCC: CallConv, CallerCC, CB, isVarArg, Outs, Ins,
5927 CallerFunc: &(MF.getFunction()), isCalleeExternalSymbol: IsCalleeExternalSymbol);
5928 if (isTailCall) {
5929 ++NumTailCalls;
5930 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5931 ++NumSiblingCalls;
5932
5933 // PC Relative calls no longer guarantee that the callee is a Global
5934 // Address Node. The callee could be an indirect tail call in which
5935 // case the SDValue for the callee could be a load (to load the address
5936 // of a function pointer) or it may be a register copy (to move the
5937 // address of the callee from a function parameter into a virtual
5938 // register). It may also be an ExternalSymbolSDNode (ex memcopy).
5939 assert((Subtarget.isUsingPCRelativeCalls() ||
5940 isa<GlobalAddressSDNode>(Callee)) &&
5941 "Callee should be an llvm::Function object.");
5942
5943 LLVM_DEBUG(dbgs() << "TCO caller: " << DAG.getMachineFunction().getName()
5944 << "\nTCO callee: ");
5945 LLVM_DEBUG(Callee.dump());
5946 }
5947 }
5948
5949 if (!isTailCall && CB && CB->isMustTailCall())
5950 report_fatal_error(reason: "failed to perform tail call elimination on a call "
5951 "site marked musttail");
5952
5953 // When long calls (i.e. indirect calls) are always used, calls are always
5954 // made via function pointer. If we have a function name, first translate it
5955 // into a pointer.
5956 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Val: Callee) &&
5957 !isTailCall)
5958 Callee = LowerGlobalAddress(Op: Callee, DAG);
5959
5960 CallFlags CFlags(
5961 CallConv, isTailCall, isVarArg, isPatchPoint,
5962 isIndirectCall(Callee, DAG, Subtarget, isPatchPoint),
5963 // hasNest
5964 Subtarget.is64BitELFABI() &&
5965 any_of(Range&: Outs, P: [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }),
5966 CLI.NoMerge);
5967
5968 if (Subtarget.isAIXABI())
5969 return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5970 InVals, CB);
5971
5972 assert(Subtarget.isSVR4ABI());
5973 if (Subtarget.isPPC64())
5974 return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5975 InVals, CB);
5976 return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5977 InVals, CB);
5978}
5979
5980SDValue PPCTargetLowering::LowerCall_32SVR4(
5981 SDValue Chain, SDValue Callee, CallFlags CFlags,
5982 const SmallVectorImpl<ISD::OutputArg> &Outs,
5983 const SmallVectorImpl<SDValue> &OutVals,
5984 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5985 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5986 const CallBase *CB) const {
5987 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
5988 // of the 32-bit SVR4 ABI stack frame layout.
5989
5990 const CallingConv::ID CallConv = CFlags.CallConv;
5991 const bool IsVarArg = CFlags.IsVarArg;
5992 const bool IsTailCall = CFlags.IsTailCall;
5993
5994 assert((CallConv == CallingConv::C ||
5995 CallConv == CallingConv::Cold ||
5996 CallConv == CallingConv::Fast) && "Unknown calling convention!");
5997
5998 const Align PtrAlign(4);
5999
6000 MachineFunction &MF = DAG.getMachineFunction();
6001
6002 // Mark this function as potentially containing a function that contains a
6003 // tail call. As a consequence the frame pointer will be used for dynamicalloc
6004 // and restoring the callers stack pointer in this functions epilog. This is
6005 // done because by tail calling the called function might overwrite the value
6006 // in this function's (MF) stack pointer stack slot 0(SP).
6007 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
6008 CallConv == CallingConv::Fast)
6009 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
6010
6011 // Count how many bytes are to be pushed on the stack, including the linkage
6012 // area, parameter list area and the part of the local variable space which
6013 // contains copies of aggregates which are passed by value.
6014
6015 // Assign locations to all of the outgoing arguments.
6016 SmallVector<CCValAssign, 16> ArgLocs;
6017 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
6018
6019 // Reserve space for the linkage area on the stack.
6020 CCInfo.AllocateStack(Size: Subtarget.getFrameLowering()->getLinkageSize(),
6021 Alignment: PtrAlign);
6022
6023 if (IsVarArg) {
6024 // Handle fixed and variable vector arguments differently.
6025 // Fixed vector arguments go into registers as long as registers are
6026 // available. Variable vector arguments always go into memory.
6027 unsigned NumArgs = Outs.size();
6028
6029 for (unsigned i = 0; i != NumArgs; ++i) {
6030 MVT ArgVT = Outs[i].VT;
6031 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
6032 bool Result;
6033
6034 if (!ArgFlags.isVarArg()) {
6035 Result = CC_PPC32_SVR4(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full, ArgFlags,
6036 OrigTy: Outs[i].OrigTy, State&: CCInfo);
6037 } else {
6038 Result = CC_PPC32_SVR4_VarArg(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full,
6039 ArgFlags, OrigTy: Outs[i].OrigTy, State&: CCInfo);
6040 }
6041
6042 if (Result) {
6043#ifndef NDEBUG
6044 errs() << "Call operand #" << i << " has unhandled type "
6045 << ArgVT << "\n";
6046#endif
6047 llvm_unreachable(nullptr);
6048 }
6049 }
6050 } else {
6051 // All arguments are treated the same.
6052 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4);
6053 }
6054
6055 // Assign locations to all of the outgoing aggregate by value arguments.
6056 SmallVector<CCValAssign, 16> ByValArgLocs;
6057 CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext());
6058
6059 // Reserve stack space for the allocations in CCInfo.
6060 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
6061
6062 CCByValInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4_ByVal);
6063
6064 // Size of the linkage area, parameter list area and the part of the local
6065 // space variable where copies of aggregates which are passed by value are
6066 // stored.
6067 unsigned NumBytes = CCByValInfo.getStackSize();
6068
6069 // Calculate by how many bytes the stack has to be adjusted in case of tail
6070 // call optimization.
6071 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: IsTailCall, ParamSize: NumBytes);
6072
6073 // Adjust the stack pointer for the new arguments...
6074 // These operations are automatically eliminated by the prolog/epilog pass
6075 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6076 SDValue CallSeqStart = Chain;
6077
6078 // Load the return address and frame pointer so it can be moved somewhere else
6079 // later.
6080 SDValue LROp, FPOp;
6081 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6082
6083 // Set up a copy of the stack pointer for use loading and storing any
6084 // arguments that may not fit in the registers available for argument
6085 // passing.
6086 SDValue StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
6087
6088 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6089 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6090 SmallVector<SDValue, 8> MemOpChains;
6091
6092 bool seenFloatArg = false;
6093 // Walk the register/memloc assignments, inserting copies/loads.
6094 // i - Tracks the index into the list of registers allocated for the call
6095 // RealArgIdx - Tracks the index into the list of actual function arguments
6096 // j - Tracks the index into the list of byval arguments
6097 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size();
6098 i != e;
6099 ++i, ++RealArgIdx) {
6100 CCValAssign &VA = ArgLocs[i];
6101 SDValue Arg = OutVals[RealArgIdx];
6102 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags;
6103
6104 if (Flags.isByVal()) {
6105 // Argument is an aggregate which is passed by value, thus we need to
6106 // create a copy of it in the local variable space of the current stack
6107 // frame (which is the stack frame of the caller) and pass the address of
6108 // this copy to the callee.
6109 assert((j < ByValArgLocs.size()) && "Index out of bounds!");
6110 CCValAssign &ByValVA = ByValArgLocs[j++];
6111 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
6112
6113 // Memory reserved in the local variable space of the callers stack frame.
6114 unsigned LocMemOffset = ByValVA.getLocMemOffset();
6115
6116 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6117 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6118 N1: StackPtr, N2: PtrOff);
6119
6120 // Create a copy of the argument in the local area of the current
6121 // stack frame.
6122 SDValue MemcpyCall =
6123 CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6124 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6125 Flags, DAG, dl);
6126
6127 // This must go outside the CALLSEQ_START..END.
6128 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: NumBytes, OutSize: 0,
6129 DL: SDLoc(MemcpyCall));
6130 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6131 To: NewCallSeqStart.getNode());
6132 Chain = CallSeqStart = NewCallSeqStart;
6133
6134 // Pass the address of the aggregate copy on the stack either in a
6135 // physical register or in the parameter list area of the current stack
6136 // frame to the callee.
6137 Arg = PtrOff;
6138 }
6139
6140 // When useCRBits() is true, there can be i1 arguments.
6141 // It is because getRegisterType(MVT::i1) => MVT::i1,
6142 // and for other integer types getRegisterType() => MVT::i32.
6143 // Extend i1 and ensure callee will get i32.
6144 if (Arg.getValueType() == MVT::i1)
6145 Arg = DAG.getNode(Opcode: Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
6146 DL: dl, VT: MVT::i32, Operand: Arg);
6147
6148 if (VA.isRegLoc()) {
6149 seenFloatArg |= VA.getLocVT().isFloatingPoint();
6150 // Put argument in a physical register.
6151 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) {
6152 bool IsLE = Subtarget.isLittleEndian();
6153 SDValue SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6154 N2: DAG.getIntPtrConstant(Val: IsLE ? 0 : 1, DL: dl));
6155 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y: SVal.getValue(R: 0)));
6156 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6157 N2: DAG.getIntPtrConstant(Val: IsLE ? 1 : 0, DL: dl));
6158 RegsToPass.push_back(Elt: std::make_pair(x: ArgLocs[++i].getLocReg(),
6159 y: SVal.getValue(R: 0)));
6160 } else
6161 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
6162 } else {
6163 // Put argument in the parameter list area of the current stack frame.
6164 assert(VA.isMemLoc());
6165 unsigned LocMemOffset = VA.getLocMemOffset();
6166
6167 if (!IsTailCall) {
6168 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6169 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6170 N1: StackPtr, N2: PtrOff);
6171
6172 MemOpChains.push_back(
6173 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
6174 } else {
6175 // Calculate and remember argument location.
6176 CalculateTailCallArgDest(DAG, MF, IsPPC64: false, Arg, SPDiff, ArgOffset: LocMemOffset,
6177 TailCallArguments);
6178 }
6179 }
6180 }
6181
6182 if (!MemOpChains.empty())
6183 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6184
6185 // Build a sequence of copy-to-reg nodes chained together with token chain
6186 // and flag operands which copy the outgoing args into the appropriate regs.
6187 SDValue InGlue;
6188 for (const auto &[Reg, N] : RegsToPass) {
6189 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6190 InGlue = Chain.getValue(R: 1);
6191 }
6192
6193 // Set CR bit 6 to true if this is a vararg call with floating args passed in
6194 // registers.
6195 if (IsVarArg) {
6196 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
6197 SDValue Ops[] = { Chain, InGlue };
6198
6199 Chain = DAG.getNode(Opcode: seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, DL: dl,
6200 VTList: VTs, Ops: ArrayRef(Ops, InGlue.getNode() ? 2 : 1));
6201
6202 InGlue = Chain.getValue(R: 1);
6203 }
6204
6205 if (IsTailCall)
6206 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6207 TailCallArguments);
6208
6209 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6210 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6211}
6212
6213// Copy an argument into memory, being careful to do this outside the
6214// call sequence for the call to which the argument belongs.
6215SDValue PPCTargetLowering::createMemcpyOutsideCallSeq(
6216 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
6217 SelectionDAG &DAG, const SDLoc &dl) const {
6218 SDValue MemcpyCall = CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6219 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6220 Flags, DAG, dl);
6221 // The MEMCPY must go outside the CALLSEQ_START..END.
6222 int64_t FrameSize = CallSeqStart.getConstantOperandVal(i: 1);
6223 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: FrameSize, OutSize: 0,
6224 DL: SDLoc(MemcpyCall));
6225 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6226 To: NewCallSeqStart.getNode());
6227 return NewCallSeqStart;
6228}
6229
6230SDValue PPCTargetLowering::LowerCall_64SVR4(
6231 SDValue Chain, SDValue Callee, CallFlags CFlags,
6232 const SmallVectorImpl<ISD::OutputArg> &Outs,
6233 const SmallVectorImpl<SDValue> &OutVals,
6234 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
6235 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
6236 const CallBase *CB) const {
6237 bool isELFv2ABI = Subtarget.isELFv2ABI();
6238 bool isLittleEndian = Subtarget.isLittleEndian();
6239 unsigned NumOps = Outs.size();
6240 bool IsSibCall = false;
6241 bool IsFastCall = CFlags.CallConv == CallingConv::Fast;
6242
6243 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
6244 unsigned PtrByteSize = 8;
6245
6246 MachineFunction &MF = DAG.getMachineFunction();
6247
6248 if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt)
6249 IsSibCall = true;
6250
6251 // Mark this function as potentially containing a function that contains a
6252 // tail call. As a consequence the frame pointer will be used for dynamicalloc
6253 // and restoring the callers stack pointer in this functions epilog. This is
6254 // done because by tail calling the called function might overwrite the value
6255 // in this function's (MF) stack pointer stack slot 0(SP).
6256 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6257 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
6258
6259 assert(!(IsFastCall && CFlags.IsVarArg) &&
6260 "fastcc not supported on varargs functions");
6261
6262 // Count how many bytes are to be pushed on the stack, including the linkage
6263 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes
6264 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage
6265 // area is 32 bytes reserved space for [SP][CR][LR][TOC].
6266 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
6267 unsigned NumBytes = LinkageSize;
6268 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
6269
6270 static const MCPhysReg GPR[] = {
6271 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6272 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
6273 };
6274 static const MCPhysReg VR[] = {
6275 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
6276 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
6277 };
6278
6279 const unsigned NumGPRs = std::size(GPR);
6280 const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
6281 const unsigned NumVRs = std::size(VR);
6282
6283 // On ELFv2, we can avoid allocating the parameter area if all the arguments
6284 // can be passed to the callee in registers.
6285 // For the fast calling convention, there is another check below.
6286 // Note: We should keep consistent with LowerFormalArguments_64SVR4()
6287 bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall;
6288 if (!HasParameterArea) {
6289 unsigned ParamAreaSize = NumGPRs * PtrByteSize;
6290 unsigned AvailableFPRs = NumFPRs;
6291 unsigned AvailableVRs = NumVRs;
6292 unsigned NumBytesTmp = NumBytes;
6293 for (unsigned i = 0; i != NumOps; ++i) {
6294 if (Outs[i].Flags.isNest()) continue;
6295 if (CalculateStackSlotUsed(ArgVT: Outs[i].VT, OrigVT: Outs[i].ArgVT, Flags: Outs[i].Flags,
6296 PtrByteSize, LinkageSize, ParamAreaSize,
6297 ArgOffset&: NumBytesTmp, AvailableFPRs, AvailableVRs))
6298 HasParameterArea = true;
6299 }
6300 }
6301
6302 // When using the fast calling convention, we don't provide backing for
6303 // arguments that will be in registers.
6304 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0;
6305
6306 // Avoid allocating parameter area for fastcc functions if all the arguments
6307 // can be passed in the registers.
6308 if (IsFastCall)
6309 HasParameterArea = false;
6310
6311 // Add up all the space actually used.
6312 for (unsigned i = 0; i != NumOps; ++i) {
6313 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6314 EVT ArgVT = Outs[i].VT;
6315 EVT OrigVT = Outs[i].ArgVT;
6316
6317 if (Flags.isNest())
6318 continue;
6319
6320 if (IsFastCall) {
6321 if (Flags.isByVal()) {
6322 NumGPRsUsed += (Flags.getByValSize()+7)/8;
6323 if (NumGPRsUsed > NumGPRs)
6324 HasParameterArea = true;
6325 } else {
6326 switch (ArgVT.getSimpleVT().SimpleTy) {
6327 default: llvm_unreachable("Unexpected ValueType for argument!");
6328 case MVT::i1:
6329 case MVT::i32:
6330 case MVT::i64:
6331 if (++NumGPRsUsed <= NumGPRs)
6332 continue;
6333 break;
6334 case MVT::v4i32:
6335 case MVT::v8i16:
6336 case MVT::v16i8:
6337 case MVT::v2f64:
6338 case MVT::v2i64:
6339 case MVT::v1i128:
6340 case MVT::f128:
6341 if (++NumVRsUsed <= NumVRs)
6342 continue;
6343 break;
6344 case MVT::v4f32:
6345 if (++NumVRsUsed <= NumVRs)
6346 continue;
6347 break;
6348 case MVT::f32:
6349 case MVT::f64:
6350 if (++NumFPRsUsed <= NumFPRs)
6351 continue;
6352 break;
6353 }
6354 HasParameterArea = true;
6355 }
6356 }
6357
6358 /* Respect alignment of argument on the stack. */
6359 auto Alignement =
6360 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6361 NumBytes = alignTo(Size: NumBytes, A: Alignement);
6362
6363 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
6364 if (Flags.isInConsecutiveRegsLast())
6365 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6366 }
6367
6368 unsigned NumBytesActuallyUsed = NumBytes;
6369
6370 // In the old ELFv1 ABI,
6371 // the prolog code of the callee may store up to 8 GPR argument registers to
6372 // the stack, allowing va_start to index over them in memory if its varargs.
6373 // Because we cannot tell if this is needed on the caller side, we have to
6374 // conservatively assume that it is needed. As such, make sure we have at
6375 // least enough stack space for the caller to store the 8 GPRs.
6376 // In the ELFv2 ABI, we allocate the parameter area iff a callee
6377 // really requires memory operands, e.g. a vararg function.
6378 if (HasParameterArea)
6379 NumBytes = std::max(a: NumBytes, b: LinkageSize + 8 * PtrByteSize);
6380 else
6381 NumBytes = LinkageSize;
6382
6383 // Tail call needs the stack to be aligned.
6384 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6385 NumBytes = EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes);
6386
6387 int SPDiff = 0;
6388
6389 // Calculate by how many bytes the stack has to be adjusted in case of tail
6390 // call optimization.
6391 if (!IsSibCall)
6392 SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: CFlags.IsTailCall, ParamSize: NumBytes);
6393
6394 // To protect arguments on the stack from being clobbered in a tail call,
6395 // force all the loads to happen before doing any other lowering.
6396 if (CFlags.IsTailCall)
6397 Chain = DAG.getStackArgumentTokenFactor(Chain);
6398
6399 // Adjust the stack pointer for the new arguments...
6400 // These operations are automatically eliminated by the prolog/epilog pass
6401 if (!IsSibCall)
6402 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6403 SDValue CallSeqStart = Chain;
6404
6405 // Load the return address and frame pointer so it can be move somewhere else
6406 // later.
6407 SDValue LROp, FPOp;
6408 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6409
6410 // Set up a copy of the stack pointer for use loading and storing any
6411 // arguments that may not fit in the registers available for argument
6412 // passing.
6413 SDValue StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
6414
6415 // Figure out which arguments are going to go in registers, and which in
6416 // memory. Also, if this is a vararg function, floating point operations
6417 // must be stored to our stack, and loaded into integer regs as well, if
6418 // any integer regs are available for argument passing.
6419 unsigned ArgOffset = LinkageSize;
6420
6421 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6422 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6423
6424 SmallVector<SDValue, 8> MemOpChains;
6425 for (unsigned i = 0; i != NumOps; ++i) {
6426 SDValue Arg = OutVals[i];
6427 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6428 EVT ArgVT = Outs[i].VT;
6429 EVT OrigVT = Outs[i].ArgVT;
6430
6431 // PtrOff will be used to store the current argument to the stack if a
6432 // register cannot be found for it.
6433 SDValue PtrOff;
6434
6435 // We re-align the argument offset for each argument, except when using the
6436 // fast calling convention, when we need to make sure we do that only when
6437 // we'll actually use a stack slot.
6438 auto ComputePtrOff = [&]() {
6439 /* Respect alignment of argument on the stack. */
6440 auto Alignment =
6441 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6442 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
6443
6444 PtrOff = DAG.getConstant(Val: ArgOffset, DL: dl, VT: StackPtr.getValueType());
6445
6446 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6447 };
6448
6449 if (!IsFastCall) {
6450 ComputePtrOff();
6451
6452 /* Compute GPR index associated with argument offset. */
6453 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
6454 GPR_idx = std::min(a: GPR_idx, b: NumGPRs);
6455 }
6456
6457 // Promote integers to 64-bit values.
6458 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) {
6459 // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
6460 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
6461 Arg = DAG.getNode(Opcode: ExtOp, DL: dl, VT: MVT::i64, Operand: Arg);
6462 }
6463
6464 // FIXME memcpy is used way more than necessary. Correctness first.
6465 // Note: "by value" is code for passing a structure by value, not
6466 // basic types.
6467 if (Flags.isByVal()) {
6468 // Note: Size includes alignment padding, so
6469 // struct x { short a; char b; }
6470 // will have Size = 4. With #pragma pack(1), it will have Size = 3.
6471 // These are the proper values we need for right-justifying the
6472 // aggregate in a parameter register.
6473 unsigned Size = Flags.getByValSize();
6474
6475 // An empty aggregate parameter takes up no storage and no
6476 // registers.
6477 if (Size == 0)
6478 continue;
6479
6480 if (IsFastCall)
6481 ComputePtrOff();
6482
6483 // All aggregates smaller than 8 bytes must be passed right-justified.
6484 if (Size==1 || Size==2 || Size==4) {
6485 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
6486 if (GPR_idx != NumGPRs) {
6487 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: Arg,
6488 PtrInfo: MachinePointerInfo(), MemVT: VT);
6489 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6490 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6491
6492 ArgOffset += PtrByteSize;
6493 continue;
6494 }
6495 }
6496
6497 if (GPR_idx == NumGPRs && Size < 8) {
6498 SDValue AddPtr = PtrOff;
6499 if (!isLittleEndian) {
6500 SDValue Const = DAG.getConstant(Val: PtrByteSize - Size, DL: dl,
6501 VT: PtrOff.getValueType());
6502 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6503 }
6504 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6505 CallSeqStart,
6506 Flags, DAG, dl);
6507 ArgOffset += PtrByteSize;
6508 continue;
6509 }
6510 // Copy the object to parameter save area if it can not be entirely passed
6511 // by registers.
6512 // FIXME: we only need to copy the parts which need to be passed in
6513 // parameter save area. For the parts passed by registers, we don't need
6514 // to copy them to the stack although we need to allocate space for them
6515 // in parameter save area.
6516 if ((NumGPRs - GPR_idx) * PtrByteSize < Size)
6517 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
6518 CallSeqStart,
6519 Flags, DAG, dl);
6520
6521 // When a register is available, pass a small aggregate right-justified.
6522 if (Size < 8 && GPR_idx != NumGPRs) {
6523 // The easiest way to get this right-justified in a register
6524 // is to copy the structure into the rightmost portion of a
6525 // local variable slot, then load the whole slot into the
6526 // register.
6527 // FIXME: The memcpy seems to produce pretty awful code for
6528 // small aggregates, particularly for packed ones.
6529 // FIXME: It would be preferable to use the slot in the
6530 // parameter save area instead of a new local variable.
6531 SDValue AddPtr = PtrOff;
6532 if (!isLittleEndian) {
6533 SDValue Const = DAG.getConstant(Val: 8 - Size, DL: dl, VT: PtrOff.getValueType());
6534 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6535 }
6536 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6537 CallSeqStart,
6538 Flags, DAG, dl);
6539
6540 // Load the slot into the register.
6541 SDValue Load =
6542 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6543 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6544 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6545
6546 // Done with this argument.
6547 ArgOffset += PtrByteSize;
6548 continue;
6549 }
6550
6551 // For aggregates larger than PtrByteSize, copy the pieces of the
6552 // object that fit into registers from the parameter save area.
6553 for (unsigned j=0; j<Size; j+=PtrByteSize) {
6554 SDValue Const = DAG.getConstant(Val: j, DL: dl, VT: PtrOff.getValueType());
6555 SDValue AddArg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Arg, N2: Const);
6556 if (GPR_idx != NumGPRs) {
6557 unsigned LoadSizeInBits = std::min(a: PtrByteSize, b: (Size - j)) * 8;
6558 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadSizeInBits);
6559 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: AddArg,
6560 PtrInfo: MachinePointerInfo(), MemVT: ObjType);
6561
6562 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6563 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6564 ArgOffset += PtrByteSize;
6565 } else {
6566 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
6567 break;
6568 }
6569 }
6570 continue;
6571 }
6572
6573 switch (Arg.getSimpleValueType().SimpleTy) {
6574 default: llvm_unreachable("Unexpected ValueType for argument!");
6575 case MVT::i1:
6576 case MVT::i32:
6577 case MVT::i64:
6578 if (Flags.isNest()) {
6579 // The 'nest' parameter, if any, is passed in R11.
6580 RegsToPass.push_back(Elt: std::make_pair(x: PPC::X11, y&: Arg));
6581 break;
6582 }
6583
6584 // These can be scalar arguments or elements of an integer array type
6585 // passed directly. Clang may use those instead of "byval" aggregate
6586 // types to avoid forcing arguments to memory unnecessarily.
6587 if (GPR_idx != NumGPRs) {
6588 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Arg));
6589 } else {
6590 if (IsFastCall)
6591 ComputePtrOff();
6592
6593 assert(HasParameterArea &&
6594 "Parameter area must exist to pass an argument in memory.");
6595 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6596 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6597 TailCallArguments, dl);
6598 if (IsFastCall)
6599 ArgOffset += PtrByteSize;
6600 }
6601 if (!IsFastCall)
6602 ArgOffset += PtrByteSize;
6603 break;
6604 case MVT::f32:
6605 case MVT::f64: {
6606 // These can be scalar arguments or elements of a float array type
6607 // passed directly. The latter are used to implement ELFv2 homogenous
6608 // float aggregates.
6609
6610 // Named arguments go into FPRs first, and once they overflow, the
6611 // remaining arguments go into GPRs and then the parameter save area.
6612 // Unnamed arguments for vararg functions always go to GPRs and
6613 // then the parameter save area. For now, put all arguments to vararg
6614 // routines always in both locations (FPR *and* GPR or stack slot).
6615 bool NeedGPROrStack = CFlags.IsVarArg || FPR_idx == NumFPRs;
6616 bool NeededLoad = false;
6617
6618 // First load the argument into the next available FPR.
6619 if (FPR_idx != NumFPRs)
6620 RegsToPass.push_back(Elt: std::make_pair(x: FPR[FPR_idx++], y&: Arg));
6621
6622 // Next, load the argument into GPR or stack slot if needed.
6623 if (!NeedGPROrStack)
6624 ;
6625 else if (GPR_idx != NumGPRs && !IsFastCall) {
6626 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
6627 // once we support fp <-> gpr moves.
6628
6629 // In the non-vararg case, this can only ever happen in the
6630 // presence of f32 array types, since otherwise we never run
6631 // out of FPRs before running out of GPRs.
6632 SDValue ArgVal;
6633
6634 // Double values are always passed in a single GPR.
6635 if (Arg.getValueType() != MVT::f32) {
6636 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: Arg);
6637
6638 // Non-array float values are extended and passed in a GPR.
6639 } else if (!Flags.isInConsecutiveRegs()) {
6640 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6641 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6642
6643 // If we have an array of floats, we collect every odd element
6644 // together with its predecessor into one GPR.
6645 } else if (ArgOffset % PtrByteSize != 0) {
6646 SDValue Lo, Hi;
6647 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: OutVals[i - 1]);
6648 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6649 if (!isLittleEndian)
6650 std::swap(a&: Lo, b&: Hi);
6651 ArgVal = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6652
6653 // The final element, if even, goes into the first half of a GPR.
6654 } else if (Flags.isInConsecutiveRegsLast()) {
6655 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6656 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6657 if (!isLittleEndian)
6658 ArgVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: ArgVal,
6659 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
6660
6661 // Non-final even elements are skipped; they will be handled
6662 // together the with subsequent argument on the next go-around.
6663 } else
6664 ArgVal = SDValue();
6665
6666 if (ArgVal.getNode())
6667 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: ArgVal));
6668 } else {
6669 if (IsFastCall)
6670 ComputePtrOff();
6671
6672 // Single-precision floating-point values are mapped to the
6673 // second (rightmost) word of the stack doubleword.
6674 if (Arg.getValueType() == MVT::f32 &&
6675 !isLittleEndian && !Flags.isInConsecutiveRegs()) {
6676 SDValue ConstFour = DAG.getConstant(Val: 4, DL: dl, VT: PtrOff.getValueType());
6677 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: ConstFour);
6678 }
6679
6680 assert(HasParameterArea &&
6681 "Parameter area must exist to pass an argument in memory.");
6682 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6683 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6684 TailCallArguments, dl);
6685
6686 NeededLoad = true;
6687 }
6688 // When passing an array of floats, the array occupies consecutive
6689 // space in the argument area; only round up to the next doubleword
6690 // at the end of the array. Otherwise, each float takes 8 bytes.
6691 if (!IsFastCall || NeededLoad) {
6692 ArgOffset += (Arg.getValueType() == MVT::f32 &&
6693 Flags.isInConsecutiveRegs()) ? 4 : 8;
6694 if (Flags.isInConsecutiveRegsLast())
6695 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6696 }
6697 break;
6698 }
6699 case MVT::v4f32:
6700 case MVT::v4i32:
6701 case MVT::v8i16:
6702 case MVT::v16i8:
6703 case MVT::v2f64:
6704 case MVT::v2i64:
6705 case MVT::v1i128:
6706 case MVT::f128:
6707 // These can be scalar arguments or elements of a vector array type
6708 // passed directly. The latter are used to implement ELFv2 homogenous
6709 // vector aggregates.
6710
6711 // For a varargs call, named arguments go into VRs or on the stack as
6712 // usual; unnamed arguments always go to the stack or the corresponding
6713 // GPRs when within range. For now, we always put the value in both
6714 // locations (or even all three).
6715 if (CFlags.IsVarArg) {
6716 assert(HasParameterArea &&
6717 "Parameter area must exist if we have a varargs call.");
6718 // We could elide this store in the case where the object fits
6719 // entirely in R registers. Maybe later.
6720 SDValue Store =
6721 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6722 MemOpChains.push_back(Elt: Store);
6723 if (VR_idx != NumVRs) {
6724 SDValue Load =
6725 DAG.getLoad(VT: MVT::v4f32, dl, Chain: Store, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6726 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6727 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Load));
6728 }
6729 ArgOffset += 16;
6730 for (unsigned i=0; i<16; i+=PtrByteSize) {
6731 if (GPR_idx == NumGPRs)
6732 break;
6733 SDValue Ix = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
6734 N2: DAG.getConstant(Val: i, DL: dl, VT: PtrVT));
6735 SDValue Load =
6736 DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Ix, PtrInfo: MachinePointerInfo());
6737 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6738 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6739 }
6740 break;
6741 }
6742
6743 // Non-varargs Altivec params go into VRs or on the stack.
6744 if (VR_idx != NumVRs) {
6745 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Arg));
6746 } else {
6747 if (IsFastCall)
6748 ComputePtrOff();
6749
6750 assert(HasParameterArea &&
6751 "Parameter area must exist to pass an argument in memory.");
6752 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6753 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: true, MemOpChains,
6754 TailCallArguments, dl);
6755 if (IsFastCall)
6756 ArgOffset += 16;
6757 }
6758
6759 if (!IsFastCall)
6760 ArgOffset += 16;
6761 break;
6762 }
6763 }
6764
6765 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) &&
6766 "mismatch in size of parameter area");
6767 (void)NumBytesActuallyUsed;
6768
6769 if (!MemOpChains.empty())
6770 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6771
6772 // Check if this is an indirect call (MTCTR/BCTRL).
6773 // See prepareDescriptorIndirectCall and buildCallOperands for more
6774 // information about calls through function pointers in the 64-bit SVR4 ABI.
6775 if (CFlags.IsIndirect) {
6776 // For 64-bit ELFv2 ABI with PCRel, do not save the TOC of the
6777 // caller in the TOC save area.
6778 if (isTOCSaveRestoreRequired(Subtarget)) {
6779 assert(!CFlags.IsTailCall && "Indirect tails calls not supported");
6780 // Load r2 into a virtual register and store it to the TOC save area.
6781 setUsesTOCBasePtr(DAG);
6782 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: PPC::X2, VT: MVT::i64);
6783 // TOC save area offset.
6784 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
6785 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
6786 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6787 Chain = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
6788 PtrInfo: MachinePointerInfo::getStack(
6789 MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
6790 }
6791 // In the ELFv2 ABI, R12 must contain the address of an indirect callee.
6792 // This does not mean the MTCTR instruction must use R12; it's easier
6793 // to model this as an extra parameter, so do that.
6794 if (isELFv2ABI && !CFlags.IsPatchPoint)
6795 RegsToPass.push_back(Elt: std::make_pair(x: (unsigned)PPC::X12, y&: Callee));
6796 }
6797
6798 // Build a sequence of copy-to-reg nodes chained together with token chain
6799 // and flag operands which copy the outgoing args into the appropriate regs.
6800 SDValue InGlue;
6801 for (const auto &[Reg, N] : RegsToPass) {
6802 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6803 InGlue = Chain.getValue(R: 1);
6804 }
6805
6806 if (CFlags.IsTailCall && !IsSibCall)
6807 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6808 TailCallArguments);
6809
6810 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6811 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6812}
6813
6814// Returns true when the shadow of a general purpose argument register
6815// in the parameter save area is aligned to at least 'RequiredAlign'.
6816static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) {
6817 assert(RequiredAlign.value() <= 16 &&
6818 "Required alignment greater than stack alignment.");
6819 switch (Reg) {
6820 default:
6821 report_fatal_error(reason: "called on invalid register.");
6822 case PPC::R5:
6823 case PPC::R9:
6824 case PPC::X3:
6825 case PPC::X5:
6826 case PPC::X7:
6827 case PPC::X9:
6828 // These registers are 16 byte aligned which is the most strict aligment
6829 // we can support.
6830 return true;
6831 case PPC::R3:
6832 case PPC::R7:
6833 case PPC::X4:
6834 case PPC::X6:
6835 case PPC::X8:
6836 case PPC::X10:
6837 // The shadow of these registers in the PSA is 8 byte aligned.
6838 return RequiredAlign <= 8;
6839 case PPC::R4:
6840 case PPC::R6:
6841 case PPC::R8:
6842 case PPC::R10:
6843 return RequiredAlign <= 4;
6844 }
6845}
6846
6847static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
6848 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
6849 Type *OrigTy, CCState &State) {
6850 const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(
6851 State.getMachineFunction().getSubtarget());
6852 const bool IsPPC64 = Subtarget.isPPC64();
6853 const unsigned PtrSize = IsPPC64 ? 8 : 4;
6854 const Align PtrAlign(PtrSize);
6855 const Align StackAlign(16);
6856 const MVT RegVT = Subtarget.getScalarIntVT();
6857
6858 if (ValVT == MVT::f128)
6859 report_fatal_error(reason: "f128 is unimplemented on AIX.");
6860
6861 static const MCPhysReg GPR_32[] = {// 32-bit registers.
6862 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
6863 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
6864 static const MCPhysReg GPR_64[] = {// 64-bit registers.
6865 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6866 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
6867
6868 static const MCPhysReg VR[] = {// Vector registers.
6869 PPC::V2, PPC::V3, PPC::V4, PPC::V5,
6870 PPC::V6, PPC::V7, PPC::V8, PPC::V9,
6871 PPC::V10, PPC::V11, PPC::V12, PPC::V13};
6872
6873 const ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32;
6874
6875 if (ArgFlags.isNest()) {
6876 MCRegister EnvReg = State.AllocateReg(Reg: IsPPC64 ? PPC::X11 : PPC::R11);
6877 if (!EnvReg)
6878 report_fatal_error(reason: "More then one nest argument.");
6879 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: EnvReg, LocVT: RegVT, HTP: LocInfo));
6880 return false;
6881 }
6882
6883 if (ArgFlags.isByVal()) {
6884 const Align ByValAlign(ArgFlags.getNonZeroByValAlign());
6885 if (ByValAlign > StackAlign)
6886 report_fatal_error(reason: "Pass-by-value arguments with alignment greater than "
6887 "16 are not supported.");
6888
6889 const unsigned ByValSize = ArgFlags.getByValSize();
6890 const Align ObjAlign = ByValAlign > PtrAlign ? ByValAlign : PtrAlign;
6891
6892 // An empty aggregate parameter takes up no storage and no registers,
6893 // but needs a MemLoc for a stack slot for the formal arguments side.
6894 if (ByValSize == 0) {
6895 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6896 Offset: State.getStackSize(), LocVT: RegVT, HTP: LocInfo));
6897 return false;
6898 }
6899
6900 // Shadow allocate any registers that are not properly aligned.
6901 unsigned NextReg = State.getFirstUnallocated(Regs: GPRs);
6902 while (NextReg != GPRs.size() &&
6903 !isGPRShadowAligned(Reg: GPRs[NextReg], RequiredAlign: ObjAlign)) {
6904 // Shadow allocate next registers since its aligment is not strict enough.
6905 MCRegister Reg = State.AllocateReg(Regs: GPRs);
6906 // Allocate the stack space shadowed by said register.
6907 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6908 assert(Reg && "Alocating register unexpectedly failed.");
6909 (void)Reg;
6910 NextReg = State.getFirstUnallocated(Regs: GPRs);
6911 }
6912
6913 const unsigned StackSize = alignTo(Size: ByValSize, A: ObjAlign);
6914 unsigned Offset = State.AllocateStack(Size: StackSize, Alignment: ObjAlign);
6915 for (const unsigned E = Offset + StackSize; Offset < E; Offset += PtrSize) {
6916 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6917 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6918 else {
6919 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6920 Offset, LocVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6921 HTP: LocInfo));
6922 break;
6923 }
6924 }
6925 return false;
6926 }
6927
6928 // Arguments always reserve parameter save area.
6929 switch (ValVT.SimpleTy) {
6930 default:
6931 report_fatal_error(reason: "Unhandled value type for argument.");
6932 case MVT::i64:
6933 // i64 arguments should have been split to i32 for PPC32.
6934 assert(IsPPC64 && "PPC32 should have split i64 values.");
6935 [[fallthrough]];
6936 case MVT::i1:
6937 case MVT::i32: {
6938 const unsigned Offset = State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6939 // AIX integer arguments are always passed in register width.
6940 if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits())
6941 LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt
6942 : CCValAssign::LocInfo::ZExt;
6943 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6944 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6945 else
6946 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT: RegVT, HTP: LocInfo));
6947
6948 return false;
6949 }
6950 case MVT::f32:
6951 case MVT::f64: {
6952 // Parameter save area (PSA) is reserved even if the float passes in fpr.
6953 const unsigned StoreSize = LocVT.getStoreSize();
6954 // Floats are always 4-byte aligned in the PSA on AIX.
6955 // This includes f64 in 64-bit mode for ABI compatibility.
6956 const unsigned Offset =
6957 State.AllocateStack(Size: IsPPC64 ? 8 : StoreSize, Alignment: Align(4));
6958 MCRegister FReg = State.AllocateReg(Regs: FPR);
6959 if (FReg)
6960 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: FReg, LocVT, HTP: LocInfo));
6961
6962 // Reserve and initialize GPRs or initialize the PSA as required.
6963 for (unsigned I = 0; I < StoreSize; I += PtrSize) {
6964 if (MCRegister Reg = State.AllocateReg(Regs: GPRs)) {
6965 assert(FReg && "An FPR should be available when a GPR is reserved.");
6966 if (State.isVarArg()) {
6967 // Successfully reserved GPRs are only initialized for vararg calls.
6968 // Custom handling is required for:
6969 // f64 in PPC32 needs to be split into 2 GPRs.
6970 // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR.
6971 State.addLoc(
6972 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6973 }
6974 } else {
6975 // If there are insufficient GPRs, the PSA needs to be initialized.
6976 // Initialization occurs even if an FPR was initialized for
6977 // compatibility with the AIX XL compiler. The full memory for the
6978 // argument will be initialized even if a prior word is saved in GPR.
6979 // A custom memLoc is used when the argument also passes in FPR so
6980 // that the callee handling can skip over it easily.
6981 State.addLoc(
6982 V: FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT,
6983 HTP: LocInfo)
6984 : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
6985 break;
6986 }
6987 }
6988
6989 return false;
6990 }
6991 case MVT::v4f32:
6992 case MVT::v4i32:
6993 case MVT::v8i16:
6994 case MVT::v16i8:
6995 case MVT::v2i64:
6996 case MVT::v2f64:
6997 case MVT::v1i128: {
6998 const unsigned VecSize = 16;
6999 const Align VecAlign(VecSize);
7000
7001 if (!State.isVarArg()) {
7002 // If there are vector registers remaining we don't consume any stack
7003 // space.
7004 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
7005 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
7006 return false;
7007 }
7008 // Vectors passed on the stack do not shadow GPRs or FPRs even though they
7009 // might be allocated in the portion of the PSA that is shadowed by the
7010 // GPRs.
7011 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7012 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7013 return false;
7014 }
7015
7016 unsigned NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7017 // Burn any underaligned registers and their shadowed stack space until
7018 // we reach the required alignment.
7019 while (NextRegIndex != GPRs.size() &&
7020 !isGPRShadowAligned(Reg: GPRs[NextRegIndex], RequiredAlign: VecAlign)) {
7021 // Shadow allocate register and its stack shadow.
7022 MCRegister Reg = State.AllocateReg(Regs: GPRs);
7023 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
7024 assert(Reg && "Allocating register unexpectedly failed.");
7025 (void)Reg;
7026 NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7027 }
7028
7029 // Vectors that are passed as fixed arguments are handled differently.
7030 // They are passed in VRs if any are available (unlike arguments passed
7031 // through ellipses) and shadow GPRs (unlike arguments to non-vaarg
7032 // functions)
7033 if (!ArgFlags.isVarArg()) {
7034 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
7035 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
7036 // Shadow allocate GPRs and stack space even though we pass in a VR.
7037 for (unsigned I = 0; I != VecSize; I += PtrSize)
7038 State.AllocateReg(Regs: GPRs);
7039 State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7040 return false;
7041 }
7042 // No vector registers remain so pass on the stack.
7043 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7044 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7045 return false;
7046 }
7047
7048 // If all GPRS are consumed then we pass the argument fully on the stack.
7049 if (NextRegIndex == GPRs.size()) {
7050 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7051 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7052 return false;
7053 }
7054
7055 // Corner case for 32-bit codegen. We have 2 registers to pass the first
7056 // half of the argument, and then need to pass the remaining half on the
7057 // stack.
7058 if (GPRs[NextRegIndex] == PPC::R9) {
7059 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7060 State.addLoc(
7061 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7062
7063 const MCRegister FirstReg = State.AllocateReg(Reg: PPC::R9);
7064 const MCRegister SecondReg = State.AllocateReg(Reg: PPC::R10);
7065 assert(FirstReg && SecondReg &&
7066 "Allocating R9 or R10 unexpectedly failed.");
7067 State.addLoc(
7068 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: FirstReg, LocVT: RegVT, HTP: LocInfo));
7069 State.addLoc(
7070 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: SecondReg, LocVT: RegVT, HTP: LocInfo));
7071 return false;
7072 }
7073
7074 // We have enough GPRs to fully pass the vector argument, and we have
7075 // already consumed any underaligned registers. Start with the custom
7076 // MemLoc and then the custom RegLocs.
7077 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7078 State.addLoc(
7079 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7080 for (unsigned I = 0; I != VecSize; I += PtrSize) {
7081 const MCRegister Reg = State.AllocateReg(Regs: GPRs);
7082 assert(Reg && "Failed to allocated register for vararg vector argument");
7083 State.addLoc(
7084 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
7085 }
7086 return false;
7087 }
7088 }
7089 return true;
7090}
7091
7092// So far, this function is only used by LowerFormalArguments_AIX()
7093static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT,
7094 bool IsPPC64,
7095 bool HasP8Vector,
7096 bool HasVSX) {
7097 assert((IsPPC64 || SVT != MVT::i64) &&
7098 "i64 should have been split for 32-bit codegen.");
7099
7100 switch (SVT) {
7101 default:
7102 report_fatal_error(reason: "Unexpected value type for formal argument");
7103 case MVT::i1:
7104 case MVT::i32:
7105 case MVT::i64:
7106 return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7107 case MVT::f32:
7108 return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass;
7109 case MVT::f64:
7110 return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass;
7111 case MVT::v4f32:
7112 case MVT::v4i32:
7113 case MVT::v8i16:
7114 case MVT::v16i8:
7115 case MVT::v2i64:
7116 case MVT::v2f64:
7117 case MVT::v1i128:
7118 return &PPC::VRRCRegClass;
7119 }
7120}
7121
7122static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT,
7123 SelectionDAG &DAG, SDValue ArgValue,
7124 MVT LocVT, const SDLoc &dl) {
7125 assert(ValVT.isScalarInteger() && LocVT.isScalarInteger());
7126 assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits());
7127
7128 if (Flags.isSExt())
7129 ArgValue = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: LocVT, N1: ArgValue,
7130 N2: DAG.getValueType(ValVT));
7131 else if (Flags.isZExt())
7132 ArgValue = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: LocVT, N1: ArgValue,
7133 N2: DAG.getValueType(ValVT));
7134
7135 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ValVT, Operand: ArgValue);
7136}
7137
7138static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
7139 const unsigned LASize = FL->getLinkageSize();
7140
7141 if (PPC::GPRCRegClass.contains(Reg)) {
7142 assert(Reg >= PPC::R3 && Reg <= PPC::R10 &&
7143 "Reg must be a valid argument register!");
7144 return LASize + 4 * (Reg - PPC::R3);
7145 }
7146
7147 if (PPC::G8RCRegClass.contains(Reg)) {
7148 assert(Reg >= PPC::X3 && Reg <= PPC::X10 &&
7149 "Reg must be a valid argument register!");
7150 return LASize + 8 * (Reg - PPC::X3);
7151 }
7152
7153 llvm_unreachable("Only general purpose registers expected.");
7154}
7155
7156// AIX ABI Stack Frame Layout:
7157//
7158// Low Memory +--------------------------------------------+
7159// SP +---> | Back chain | ---+
7160// | +--------------------------------------------+ |
7161// | | Saved Condition Register | |
7162// | +--------------------------------------------+ |
7163// | | Saved Linkage Register | |
7164// | +--------------------------------------------+ | Linkage Area
7165// | | Reserved for compilers | |
7166// | +--------------------------------------------+ |
7167// | | Reserved for binders | |
7168// | +--------------------------------------------+ |
7169// | | Saved TOC pointer | ---+
7170// | +--------------------------------------------+
7171// | | Parameter save area |
7172// | +--------------------------------------------+
7173// | | Alloca space |
7174// | +--------------------------------------------+
7175// | | Local variable space |
7176// | +--------------------------------------------+
7177// | | Float/int conversion temporary |
7178// | +--------------------------------------------+
7179// | | Save area for AltiVec registers |
7180// | +--------------------------------------------+
7181// | | AltiVec alignment padding |
7182// | +--------------------------------------------+
7183// | | Save area for VRSAVE register |
7184// | +--------------------------------------------+
7185// | | Save area for General Purpose registers |
7186// | +--------------------------------------------+
7187// | | Save area for Floating Point registers |
7188// | +--------------------------------------------+
7189// +---- | Back chain |
7190// High Memory +--------------------------------------------+
7191//
7192// Specifications:
7193// AIX 7.2 Assembler Language Reference
7194// Subroutine linkage convention
7195
7196SDValue PPCTargetLowering::LowerFormalArguments_AIX(
7197 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
7198 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7199 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
7200
7201 assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold ||
7202 CallConv == CallingConv::Fast) &&
7203 "Unexpected calling convention!");
7204
7205 if (getTargetMachine().Options.GuaranteedTailCallOpt)
7206 report_fatal_error(reason: "Tail call support is unimplemented on AIX.");
7207
7208 if (useSoftFloat())
7209 report_fatal_error(reason: "Soft float support is unimplemented on AIX.");
7210
7211 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7212
7213 const bool IsPPC64 = Subtarget.isPPC64();
7214 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7215
7216 // Assign locations to all of the incoming arguments.
7217 SmallVector<CCValAssign, 16> ArgLocs;
7218 MachineFunction &MF = DAG.getMachineFunction();
7219 MachineFrameInfo &MFI = MF.getFrameInfo();
7220 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
7221 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
7222
7223 const EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
7224 // Reserve space for the linkage area on the stack.
7225 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7226 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7227 uint64_t SaveStackPos = CCInfo.getStackSize();
7228 bool SaveParams = MF.getFunction().hasFnAttribute(Kind: "save-reg-params");
7229 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_AIX);
7230
7231 SmallVector<SDValue, 8> MemOps;
7232
7233 for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) {
7234 CCValAssign &VA = ArgLocs[I++];
7235 MVT LocVT = VA.getLocVT();
7236 MVT ValVT = VA.getValVT();
7237 ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags;
7238
7239 EVT ArgVT = Ins[VA.getValNo()].ArgVT;
7240 bool ArgSignExt = Ins[VA.getValNo()].Flags.isSExt();
7241 // For compatibility with the AIX XL compiler, the float args in the
7242 // parameter save area are initialized even if the argument is available
7243 // in register. The caller is required to initialize both the register
7244 // and memory, however, the callee can choose to expect it in either.
7245 // The memloc is dismissed here because the argument is retrieved from
7246 // the register.
7247 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint())
7248 continue;
7249
7250 if (SaveParams && VA.isRegLoc() && !Flags.isByVal() && !VA.needsCustom()) {
7251 const TargetRegisterClass *RegClass = getRegClassForSVT(
7252 SVT: LocVT.SimpleTy, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(), HasVSX: Subtarget.hasVSX());
7253 // On PPC64, debugger assumes extended 8-byte values are stored from GPR.
7254 MVT SaveVT = RegClass == &PPC::G8RCRegClass ? MVT::i64 : LocVT;
7255 const Register VReg = MF.addLiveIn(PReg: VA.getLocReg(), RC: RegClass);
7256 SDValue Parm = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: SaveVT);
7257 int FI = MFI.CreateFixedObject(Size: SaveVT.getStoreSize(), SPOffset: SaveStackPos, IsImmutable: true);
7258 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7259 SDValue StoreReg = DAG.getStore(Chain, dl, Val: Parm, Ptr: FIN,
7260 PtrInfo: MachinePointerInfo(), Alignment: Align(PtrByteSize));
7261 SaveStackPos = alignTo(Value: SaveStackPos + SaveVT.getStoreSize(), Align: PtrByteSize);
7262 MemOps.push_back(Elt: StoreReg);
7263 }
7264
7265 if (SaveParams && (VA.isMemLoc() || Flags.isByVal()) && !VA.needsCustom()) {
7266 unsigned StoreSize =
7267 Flags.isByVal() ? Flags.getByValSize() : LocVT.getStoreSize();
7268 SaveStackPos = alignTo(Value: SaveStackPos + StoreSize, Align: PtrByteSize);
7269 }
7270
7271 auto HandleMemLoc = [&]() {
7272 const unsigned LocSize = LocVT.getStoreSize();
7273 const unsigned ValSize = ValVT.getStoreSize();
7274 assert((ValSize <= LocSize) &&
7275 "Object size is larger than size of MemLoc");
7276 int CurArgOffset = VA.getLocMemOffset();
7277 // Objects are right-justified because AIX is big-endian.
7278 if (LocSize > ValSize)
7279 CurArgOffset += LocSize - ValSize;
7280 // Potential tail calls could cause overwriting of argument stack slots.
7281 const bool IsImmutable =
7282 !(getTargetMachine().Options.GuaranteedTailCallOpt &&
7283 (CallConv == CallingConv::Fast));
7284 int FI = MFI.CreateFixedObject(Size: ValSize, SPOffset: CurArgOffset, IsImmutable);
7285 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7286 SDValue ArgValue =
7287 DAG.getLoad(VT: ValVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
7288
7289 // While the ABI specifies the argument type is (sign or zero) extended
7290 // out to register width, not all code is compliant. We truncate and
7291 // re-extend to be more forgiving of these callers when the argument type
7292 // is smaller than register width.
7293 if (!ArgVT.isVector() && !ValVT.isVector() && ArgVT.isInteger() &&
7294 ValVT.isInteger() &&
7295 ArgVT.getScalarSizeInBits() < ValVT.getScalarSizeInBits()) {
7296 // It is possible to have either real integer values
7297 // or integers that were not originally integers.
7298 // In the latter case, these could have came from structs,
7299 // and these integers would not have an extend on the parameter.
7300 // Since these types of integers do not have an extend specified
7301 // in the first place, the type of extend that we do should not matter.
7302 EVT TruncatedArgVT = ArgVT.isSimple() && ArgVT.getSimpleVT() == MVT::i1
7303 ? MVT::i8
7304 : ArgVT;
7305 SDValue ArgValueTrunc =
7306 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncatedArgVT, Operand: ArgValue);
7307 SDValue ArgValueExt =
7308 ArgSignExt ? DAG.getSExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT)
7309 : DAG.getZExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT);
7310 InVals.push_back(Elt: ArgValueExt);
7311 } else {
7312 InVals.push_back(Elt: ArgValue);
7313 }
7314 };
7315
7316 // Vector arguments to VaArg functions are passed both on the stack, and
7317 // in any available GPRs. Load the value from the stack and add the GPRs
7318 // as live ins.
7319 if (VA.isMemLoc() && VA.needsCustom()) {
7320 assert(ValVT.isVector() && "Unexpected Custom MemLoc type.");
7321 assert(isVarArg && "Only use custom memloc for vararg.");
7322 // ValNo of the custom MemLoc, so we can compare it to the ValNo of the
7323 // matching custom RegLocs.
7324 const unsigned OriginalValNo = VA.getValNo();
7325 (void)OriginalValNo;
7326
7327 auto HandleCustomVecRegLoc = [&]() {
7328 assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7329 "Missing custom RegLoc.");
7330 VA = ArgLocs[I++];
7331 assert(VA.getValVT().isVector() &&
7332 "Unexpected Val type for custom RegLoc.");
7333 assert(VA.getValNo() == OriginalValNo &&
7334 "ValNo mismatch between custom MemLoc and RegLoc.");
7335 MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy;
7336 MF.addLiveIn(PReg: VA.getLocReg(),
7337 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7338 HasVSX: Subtarget.hasVSX()));
7339 };
7340
7341 HandleMemLoc();
7342 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7343 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7344 // R10.
7345 HandleCustomVecRegLoc();
7346 HandleCustomVecRegLoc();
7347
7348 // If we are targeting 32-bit, there might be 2 extra custom RegLocs if
7349 // we passed the vector in R5, R6, R7 and R8.
7350 if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) {
7351 assert(!IsPPC64 &&
7352 "Only 2 custom RegLocs expected for 64-bit codegen.");
7353 HandleCustomVecRegLoc();
7354 HandleCustomVecRegLoc();
7355 }
7356
7357 continue;
7358 }
7359
7360 if (VA.isRegLoc()) {
7361 if (VA.getValVT().isScalarInteger())
7362 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7363 else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) {
7364 switch (VA.getValVT().SimpleTy) {
7365 default:
7366 report_fatal_error(reason: "Unhandled value type for argument.");
7367 case MVT::f32:
7368 FuncInfo->appendParameterType(Type: PPCFunctionInfo::ShortFloatingPoint);
7369 break;
7370 case MVT::f64:
7371 FuncInfo->appendParameterType(Type: PPCFunctionInfo::LongFloatingPoint);
7372 break;
7373 }
7374 } else if (VA.getValVT().isVector()) {
7375 switch (VA.getValVT().SimpleTy) {
7376 default:
7377 report_fatal_error(reason: "Unhandled value type for argument.");
7378 case MVT::v16i8:
7379 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorChar);
7380 break;
7381 case MVT::v8i16:
7382 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorShort);
7383 break;
7384 case MVT::v4i32:
7385 case MVT::v2i64:
7386 case MVT::v1i128:
7387 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorInt);
7388 break;
7389 case MVT::v4f32:
7390 case MVT::v2f64:
7391 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorFloat);
7392 break;
7393 }
7394 }
7395 }
7396
7397 if (Flags.isByVal() && VA.isMemLoc()) {
7398 const unsigned Size =
7399 alignTo(Value: Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize,
7400 Align: PtrByteSize);
7401 const int FI = MF.getFrameInfo().CreateFixedObject(
7402 Size, SPOffset: VA.getLocMemOffset(), /* IsImmutable */ false,
7403 /* IsAliased */ isAliased: true);
7404 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7405 InVals.push_back(Elt: FIN);
7406
7407 continue;
7408 }
7409
7410 if (Flags.isByVal()) {
7411 assert(VA.isRegLoc() && "MemLocs should already be handled.");
7412
7413 const MCPhysReg ArgReg = VA.getLocReg();
7414 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
7415
7416 const unsigned StackSize = alignTo(Value: Flags.getByValSize(), Align: PtrByteSize);
7417 const int FI = MF.getFrameInfo().CreateFixedObject(
7418 Size: StackSize, SPOffset: mapArgRegToOffsetAIX(Reg: ArgReg, FL), /* IsImmutable */ false,
7419 /* IsAliased */ isAliased: true);
7420 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7421 InVals.push_back(Elt: FIN);
7422
7423 // Add live ins for all the RegLocs for the same ByVal.
7424 const TargetRegisterClass *RegClass =
7425 IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7426
7427 auto HandleRegLoc = [&, RegClass, LocVT](const MCPhysReg PhysReg,
7428 unsigned Offset) {
7429 const Register VReg = MF.addLiveIn(PReg: PhysReg, RC: RegClass);
7430 // Since the callers side has left justified the aggregate in the
7431 // register, we can simply store the entire register into the stack
7432 // slot.
7433 SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7434 // The store to the fixedstack object is needed becuase accessing a
7435 // field of the ByVal will use a gep and load. Ideally we will optimize
7436 // to extracting the value from the register directly, and elide the
7437 // stores when the arguments address is not taken, but that will need to
7438 // be future work.
7439 SDValue Store = DAG.getStore(
7440 Chain: CopyFrom.getValue(R: 1), dl, Val: CopyFrom,
7441 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: FIN, Offset: TypeSize::getFixed(ExactSize: Offset)),
7442 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset));
7443
7444 MemOps.push_back(Elt: Store);
7445 };
7446
7447 unsigned Offset = 0;
7448 HandleRegLoc(VA.getLocReg(), Offset);
7449 Offset += PtrByteSize;
7450 for (; Offset != StackSize && ArgLocs[I].isRegLoc();
7451 Offset += PtrByteSize) {
7452 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7453 "RegLocs should be for ByVal argument.");
7454
7455 const CCValAssign RL = ArgLocs[I++];
7456 HandleRegLoc(RL.getLocReg(), Offset);
7457 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7458 }
7459
7460 if (Offset != StackSize) {
7461 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7462 "Expected MemLoc for remaining bytes.");
7463 assert(ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes.");
7464 // Consume the MemLoc.The InVal has already been emitted, so nothing
7465 // more needs to be done.
7466 ++I;
7467 }
7468
7469 continue;
7470 }
7471
7472 if (VA.isRegLoc() && !VA.needsCustom()) {
7473 MVT::SimpleValueType SVT = ValVT.SimpleTy;
7474 Register VReg =
7475 MF.addLiveIn(PReg: VA.getLocReg(),
7476 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7477 HasVSX: Subtarget.hasVSX()));
7478 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7479 if (ValVT.isScalarInteger() &&
7480 (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) {
7481 ArgValue =
7482 truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl);
7483 }
7484 InVals.push_back(Elt: ArgValue);
7485 continue;
7486 }
7487 if (VA.isMemLoc()) {
7488 HandleMemLoc();
7489 continue;
7490 }
7491 }
7492
7493 // On AIX a minimum of 8 words is saved to the parameter save area.
7494 const unsigned MinParameterSaveArea = 8 * PtrByteSize;
7495 // Area that is at least reserved in the caller of this function.
7496 unsigned CallerReservedArea = std::max<unsigned>(
7497 a: CCInfo.getStackSize(), b: LinkageSize + MinParameterSaveArea);
7498
7499 // Set the size that is at least reserved in caller of this function. Tail
7500 // call optimized function's reserved stack space needs to be aligned so
7501 // that taking the difference between two stack areas will result in an
7502 // aligned stack.
7503 CallerReservedArea =
7504 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: CallerReservedArea);
7505 FuncInfo->setMinReservedArea(CallerReservedArea);
7506
7507 if (isVarArg) {
7508 int VAListIndex = 0;
7509 // If any of the optional arguments are passed in register then the fixed
7510 // stack object we spill into is not immutable. Create a fixed stack object
7511 // that overlaps the remainder of the parameter save area.
7512 if (CCInfo.getStackSize() < (LinkageSize + MinParameterSaveArea)) {
7513 unsigned FixedStackSize =
7514 LinkageSize + MinParameterSaveArea - CCInfo.getStackSize();
7515 VAListIndex =
7516 MFI.CreateFixedObject(Size: FixedStackSize, SPOffset: CCInfo.getStackSize(),
7517 /* IsImmutable */ false, /* IsAliased */ isAliased: true);
7518 } else {
7519 // All the arguments passed through ellipses are on the stack. Create a
7520 // dummy fixed stack object the same size as a pointer since we don't
7521 // know the actual size.
7522 VAListIndex =
7523 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: CCInfo.getStackSize(),
7524 /* IsImmutable */ true, /* IsAliased */ isAliased: true);
7525 }
7526
7527 FuncInfo->setVarArgsFrameIndex(VAListIndex);
7528 SDValue FIN = DAG.getFrameIndex(FI: VAListIndex, VT: PtrVT);
7529
7530 static const MCPhysReg GPR_32[] = {PPC::R3, PPC::R4, PPC::R5, PPC::R6,
7531 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
7532
7533 static const MCPhysReg GPR_64[] = {PPC::X3, PPC::X4, PPC::X5, PPC::X6,
7534 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
7535 const unsigned NumGPArgRegs = std::size(IsPPC64 ? GPR_64 : GPR_32);
7536
7537 // The fixed integer arguments of a variadic function are stored to the
7538 // VarArgsFrameIndex on the stack so that they may be loaded by
7539 // dereferencing the result of va_next.
7540 for (unsigned
7541 GPRIndex = (CCInfo.getStackSize() - LinkageSize) / PtrByteSize,
7542 Offset = 0;
7543 GPRIndex < NumGPArgRegs; ++GPRIndex, Offset += PtrByteSize) {
7544
7545 const Register VReg =
7546 IsPPC64 ? MF.addLiveIn(PReg: GPR_64[GPRIndex], RC: &PPC::G8RCRegClass)
7547 : MF.addLiveIn(PReg: GPR_32[GPRIndex], RC: &PPC::GPRCRegClass);
7548
7549 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
7550 MachinePointerInfo MPI =
7551 MachinePointerInfo::getFixedStack(MF, FI: VAListIndex, Offset);
7552 SDValue Store = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MPI);
7553 MemOps.push_back(Elt: Store);
7554 // Increment the address for the next argument to store.
7555 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
7556 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
7557 }
7558 }
7559
7560 if (!MemOps.empty())
7561 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
7562
7563 return Chain;
7564}
7565
7566SDValue PPCTargetLowering::LowerCall_AIX(
7567 SDValue Chain, SDValue Callee, CallFlags CFlags,
7568 const SmallVectorImpl<ISD::OutputArg> &Outs,
7569 const SmallVectorImpl<SDValue> &OutVals,
7570 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7571 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
7572 const CallBase *CB) const {
7573 // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the
7574 // AIX ABI stack frame layout.
7575
7576 assert((CFlags.CallConv == CallingConv::C ||
7577 CFlags.CallConv == CallingConv::Cold ||
7578 CFlags.CallConv == CallingConv::Fast) &&
7579 "Unexpected calling convention!");
7580
7581 if (CFlags.IsPatchPoint)
7582 report_fatal_error(reason: "This call type is unimplemented on AIX.");
7583
7584 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7585
7586 MachineFunction &MF = DAG.getMachineFunction();
7587 SmallVector<CCValAssign, 16> ArgLocs;
7588 CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
7589 *DAG.getContext());
7590
7591 // Reserve space for the linkage save area (LSA) on the stack.
7592 // In both PPC32 and PPC64 there are 6 reserved slots in the LSA:
7593 // [SP][CR][LR][2 x reserved][TOC].
7594 // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64.
7595 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7596 const bool IsPPC64 = Subtarget.isPPC64();
7597 const EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7598 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7599 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7600 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_AIX);
7601
7602 // The prolog code of the callee may store up to 8 GPR argument registers to
7603 // the stack, allowing va_start to index over them in memory if the callee
7604 // is variadic.
7605 // Because we cannot tell if this is needed on the caller side, we have to
7606 // conservatively assume that it is needed. As such, make sure we have at
7607 // least enough stack space for the caller to store the 8 GPRs.
7608 const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize;
7609 const unsigned NumBytes = std::max<unsigned>(
7610 a: LinkageSize + MinParameterSaveAreaSize, b: CCInfo.getStackSize());
7611
7612 // Adjust the stack pointer for the new arguments...
7613 // These operations are automatically eliminated by the prolog/epilog pass.
7614 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
7615 SDValue CallSeqStart = Chain;
7616
7617 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
7618 SmallVector<SDValue, 8> MemOpChains;
7619
7620 // Set up a copy of the stack pointer for loading and storing any
7621 // arguments that may not fit in the registers available for argument
7622 // passing.
7623 const SDValue StackPtr = IsPPC64 ? DAG.getRegister(Reg: PPC::X1, VT: MVT::i64)
7624 : DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
7625
7626 for (unsigned I = 0, E = ArgLocs.size(); I != E;) {
7627 const unsigned ValNo = ArgLocs[I].getValNo();
7628 SDValue Arg = OutVals[ValNo];
7629 ISD::ArgFlagsTy Flags = Outs[ValNo].Flags;
7630
7631 if (Flags.isByVal()) {
7632 const unsigned ByValSize = Flags.getByValSize();
7633
7634 // Nothing to do for zero-sized ByVals on the caller side.
7635 if (!ByValSize) {
7636 ++I;
7637 continue;
7638 }
7639
7640 auto GetLoad = [&](EVT VT, unsigned LoadOffset) {
7641 return DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: PtrVT, Chain,
7642 Ptr: (LoadOffset != 0)
7643 ? DAG.getObjectPtrOffset(
7644 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7645 : Arg,
7646 PtrInfo: MachinePointerInfo(), MemVT: VT);
7647 };
7648
7649 unsigned LoadOffset = 0;
7650
7651 // Initialize registers, which are fully occupied by the by-val argument.
7652 while (LoadOffset + PtrByteSize <= ByValSize && ArgLocs[I].isRegLoc()) {
7653 SDValue Load = GetLoad(PtrVT, LoadOffset);
7654 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7655 LoadOffset += PtrByteSize;
7656 const CCValAssign &ByValVA = ArgLocs[I++];
7657 assert(ByValVA.getValNo() == ValNo &&
7658 "Unexpected location for pass-by-value argument.");
7659 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: Load));
7660 }
7661
7662 if (LoadOffset == ByValSize)
7663 continue;
7664
7665 // There must be one more loc to handle the remainder.
7666 assert(ArgLocs[I].getValNo() == ValNo &&
7667 "Expected additional location for by-value argument.");
7668
7669 if (ArgLocs[I].isMemLoc()) {
7670 assert(LoadOffset < ByValSize && "Unexpected memloc for by-val arg.");
7671 const CCValAssign &ByValVA = ArgLocs[I++];
7672 ISD::ArgFlagsTy MemcpyFlags = Flags;
7673 // Only memcpy the bytes that don't pass in register.
7674 MemcpyFlags.setByValSize(ByValSize - LoadOffset);
7675 Chain = CallSeqStart = createMemcpyOutsideCallSeq(
7676 Arg: (LoadOffset != 0) ? DAG.getObjectPtrOffset(
7677 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7678 : Arg,
7679 PtrOff: DAG.getObjectPtrOffset(
7680 SL: dl, Ptr: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByValVA.getLocMemOffset())),
7681 CallSeqStart, Flags: MemcpyFlags, DAG, dl);
7682 continue;
7683 }
7684
7685 // Initialize the final register residue.
7686 // Any residue that occupies the final by-val arg register must be
7687 // left-justified on AIX. Loads must be a power-of-2 size and cannot be
7688 // larger than the ByValSize. For example: a 7 byte by-val arg requires 4,
7689 // 2 and 1 byte loads.
7690 const unsigned ResidueBytes = ByValSize % PtrByteSize;
7691 assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize &&
7692 "Unexpected register residue for by-value argument.");
7693 SDValue ResidueVal;
7694 for (unsigned Bytes = 0; Bytes != ResidueBytes;) {
7695 const unsigned N = llvm::bit_floor(Value: ResidueBytes - Bytes);
7696 const MVT VT =
7697 N == 1 ? MVT::i8
7698 : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64));
7699 SDValue Load = GetLoad(VT, LoadOffset);
7700 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7701 LoadOffset += N;
7702 Bytes += N;
7703
7704 // By-val arguments are passed left-justfied in register.
7705 // Every load here needs to be shifted, otherwise a full register load
7706 // should have been used.
7707 assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) &&
7708 "Unexpected load emitted during handling of pass-by-value "
7709 "argument.");
7710 unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8);
7711 EVT ShiftAmountTy =
7712 getShiftAmountTy(LHSTy: Load->getValueType(ResNo: 0), DL: DAG.getDataLayout());
7713 SDValue SHLAmt = DAG.getConstant(Val: NumSHLBits, DL: dl, VT: ShiftAmountTy);
7714 SDValue ShiftedLoad =
7715 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Load.getValueType(), N1: Load, N2: SHLAmt);
7716 ResidueVal = ResidueVal ? DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PtrVT, N1: ResidueVal,
7717 N2: ShiftedLoad)
7718 : ShiftedLoad;
7719 }
7720
7721 const CCValAssign &ByValVA = ArgLocs[I++];
7722 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: ResidueVal));
7723 continue;
7724 }
7725
7726 CCValAssign &VA = ArgLocs[I++];
7727 const MVT LocVT = VA.getLocVT();
7728 const MVT ValVT = VA.getValVT();
7729
7730 switch (VA.getLocInfo()) {
7731 default:
7732 report_fatal_error(reason: "Unexpected argument extension type.");
7733 case CCValAssign::Full:
7734 break;
7735 case CCValAssign::ZExt:
7736 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7737 break;
7738 case CCValAssign::SExt:
7739 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7740 break;
7741 }
7742
7743 if (VA.isRegLoc() && !VA.needsCustom()) {
7744 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
7745 continue;
7746 }
7747
7748 // Vector arguments passed to VarArg functions need custom handling when
7749 // they are passed (at least partially) in GPRs.
7750 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) {
7751 assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args.");
7752 // Store value to its stack slot.
7753 SDValue PtrOff =
7754 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7755 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7756 SDValue Store =
7757 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
7758 MemOpChains.push_back(Elt: Store);
7759 const unsigned OriginalValNo = VA.getValNo();
7760 // Then load the GPRs from the stack
7761 unsigned LoadOffset = 0;
7762 auto HandleCustomVecRegLoc = [&]() {
7763 assert(I != E && "Unexpected end of CCvalAssigns.");
7764 assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7765 "Expected custom RegLoc.");
7766 CCValAssign RegVA = ArgLocs[I++];
7767 assert(RegVA.getValNo() == OriginalValNo &&
7768 "Custom MemLoc ValNo and custom RegLoc ValNo must match.");
7769 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
7770 N2: DAG.getConstant(Val: LoadOffset, DL: dl, VT: PtrVT));
7771 SDValue Load = DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Add, PtrInfo: MachinePointerInfo());
7772 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7773 RegsToPass.push_back(Elt: std::make_pair(x: RegVA.getLocReg(), y&: Load));
7774 LoadOffset += PtrByteSize;
7775 };
7776
7777 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7778 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7779 // R10.
7780 HandleCustomVecRegLoc();
7781 HandleCustomVecRegLoc();
7782
7783 if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7784 ArgLocs[I].getValNo() == OriginalValNo) {
7785 assert(!IsPPC64 &&
7786 "Only 2 custom RegLocs expected for 64-bit codegen.");
7787 HandleCustomVecRegLoc();
7788 HandleCustomVecRegLoc();
7789 }
7790
7791 continue;
7792 }
7793
7794 if (VA.isMemLoc()) {
7795 SDValue PtrOff =
7796 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7797 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7798 MemOpChains.push_back(
7799 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff,
7800 PtrInfo: MachinePointerInfo::getStack(MF, Offset: VA.getLocMemOffset()),
7801 Alignment: Subtarget.getFrameLowering()->getStackAlign()));
7802
7803 continue;
7804 }
7805
7806 if (!ValVT.isFloatingPoint())
7807 report_fatal_error(
7808 reason: "Unexpected register handling for calling convention.");
7809
7810 // Custom handling is used for GPR initializations for vararg float
7811 // arguments.
7812 assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg &&
7813 LocVT.isInteger() &&
7814 "Custom register handling only expected for VarArg.");
7815
7816 SDValue ArgAsInt =
7817 DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), V: Arg);
7818
7819 if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize())
7820 // f32 in 32-bit GPR
7821 // f64 in 64-bit GPR
7822 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgAsInt));
7823 else if (Arg.getValueType().getFixedSizeInBits() <
7824 LocVT.getFixedSizeInBits())
7825 // f32 in 64-bit GPR.
7826 RegsToPass.push_back(Elt: std::make_pair(
7827 x: VA.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: LocVT)));
7828 else {
7829 // f64 in two 32-bit GPRs
7830 // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs.
7831 assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 &&
7832 "Unexpected custom register for argument!");
7833 CCValAssign &GPR1 = VA;
7834 SDValue MSWAsI64 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgAsInt,
7835 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i8));
7836 RegsToPass.push_back(Elt: std::make_pair(
7837 x: GPR1.getLocReg(), y: DAG.getZExtOrTrunc(Op: MSWAsI64, DL: dl, VT: MVT::i32)));
7838
7839 if (I != E) {
7840 // If only 1 GPR was available, there will only be one custom GPR and
7841 // the argument will also pass in memory.
7842 CCValAssign &PeekArg = ArgLocs[I];
7843 if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) {
7844 assert(PeekArg.needsCustom() && "A second custom GPR is expected.");
7845 CCValAssign &GPR2 = ArgLocs[I++];
7846 RegsToPass.push_back(Elt: std::make_pair(
7847 x: GPR2.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: MVT::i32)));
7848 }
7849 }
7850 }
7851 }
7852
7853 if (!MemOpChains.empty())
7854 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
7855
7856 // For indirect calls, we need to save the TOC base to the stack for
7857 // restoration after the call.
7858 if (CFlags.IsIndirect && !Subtarget.usePointerGlueHelper()) {
7859 assert(!CFlags.IsTailCall && "Indirect tail-calls not supported.");
7860 const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister();
7861 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
7862 const MVT PtrVT = Subtarget.getScalarIntVT();
7863 const unsigned TOCSaveOffset =
7864 Subtarget.getFrameLowering()->getTOCSaveOffset();
7865
7866 setUsesTOCBasePtr(DAG);
7867 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: TOCBaseReg, VT: PtrVT);
7868 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
7869 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: PtrVT);
7870 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7871 Chain = DAG.getStore(
7872 Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
7873 PtrInfo: MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
7874 }
7875
7876 // Build a sequence of copy-to-reg nodes chained together with token chain
7877 // and flag operands which copy the outgoing args into the appropriate regs.
7878 SDValue InGlue;
7879 for (auto Reg : RegsToPass) {
7880 Chain = DAG.getCopyToReg(Chain, dl, Reg: Reg.first, N: Reg.second, Glue: InGlue);
7881 InGlue = Chain.getValue(R: 1);
7882 }
7883
7884 const int SPDiff = 0;
7885 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
7886 Callee, SPDiff, NumBytes, Ins, InVals, CB);
7887}
7888
7889bool
7890PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
7891 MachineFunction &MF, bool isVarArg,
7892 const SmallVectorImpl<ISD::OutputArg> &Outs,
7893 LLVMContext &Context,
7894 const Type *RetTy) const {
7895 SmallVector<CCValAssign, 16> RVLocs;
7896 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
7897 return CCInfo.CheckReturn(
7898 Outs, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7899 ? RetCC_PPC_Cold
7900 : RetCC_PPC);
7901}
7902
7903SDValue
7904PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
7905 bool isVarArg,
7906 const SmallVectorImpl<ISD::OutputArg> &Outs,
7907 const SmallVectorImpl<SDValue> &OutVals,
7908 const SDLoc &dl, SelectionDAG &DAG) const {
7909 SmallVector<CCValAssign, 16> RVLocs;
7910 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
7911 *DAG.getContext());
7912 CCInfo.AnalyzeReturn(Outs,
7913 Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7914 ? RetCC_PPC_Cold
7915 : RetCC_PPC);
7916
7917 SDValue Glue;
7918 SmallVector<SDValue, 4> RetOps(1, Chain);
7919
7920 // Copy the result values into the output registers.
7921 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) {
7922 CCValAssign &VA = RVLocs[i];
7923 assert(VA.isRegLoc() && "Can only return in registers!");
7924
7925 SDValue Arg = OutVals[RealResIdx];
7926
7927 switch (VA.getLocInfo()) {
7928 default: llvm_unreachable("Unknown loc info!");
7929 case CCValAssign::Full: break;
7930 case CCValAssign::AExt:
7931 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7932 break;
7933 case CCValAssign::ZExt:
7934 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7935 break;
7936 case CCValAssign::SExt:
7937 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7938 break;
7939 }
7940 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
7941 bool isLittleEndian = Subtarget.isLittleEndian();
7942 // Legalize ret f64 -> ret 2 x i32.
7943 SDValue SVal =
7944 DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7945 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 0 : 1, DL: dl));
7946 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7947 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7948 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7949 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 1 : 0, DL: dl));
7950 Glue = Chain.getValue(R: 1);
7951 VA = RVLocs[++i]; // skip ahead to next loc
7952 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7953 } else
7954 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Arg, Glue);
7955 Glue = Chain.getValue(R: 1);
7956 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7957 }
7958
7959 RetOps[0] = Chain; // Update chain.
7960
7961 // Add the glue if we have it.
7962 if (Glue.getNode())
7963 RetOps.push_back(Elt: Glue);
7964
7965 return DAG.getNode(Opcode: PPCISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
7966}
7967
7968SDValue
7969PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,
7970 SelectionDAG &DAG) const {
7971 SDLoc dl(Op);
7972
7973 // Get the correct type for integers.
7974 EVT IntVT = Op.getValueType();
7975
7976 // Get the inputs.
7977 SDValue Chain = Op.getOperand(i: 0);
7978 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
7979 // Build a DYNAREAOFFSET node.
7980 SDValue Ops[2] = {Chain, FPSIdx};
7981 SDVTList VTs = DAG.getVTList(VT: IntVT);
7982 return DAG.getNode(Opcode: PPCISD::DYNAREAOFFSET, DL: dl, VTList: VTs, Ops);
7983}
7984
7985SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
7986 SelectionDAG &DAG) const {
7987 // When we pop the dynamic allocation we need to restore the SP link.
7988 SDLoc dl(Op);
7989
7990 // Get the correct type for pointers.
7991 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7992
7993 // Construct the stack pointer operand.
7994 bool isPPC64 = Subtarget.isPPC64();
7995 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1;
7996 SDValue StackPtr = DAG.getRegister(Reg: SP, VT: PtrVT);
7997
7998 // Get the operands for the STACKRESTORE.
7999 SDValue Chain = Op.getOperand(i: 0);
8000 SDValue SaveSP = Op.getOperand(i: 1);
8001
8002 // Load the old link SP.
8003 SDValue LoadLinkSP =
8004 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
8005
8006 // Restore the stack pointer.
8007 Chain = DAG.getCopyToReg(Chain: LoadLinkSP.getValue(R: 1), dl, Reg: SP, N: SaveSP);
8008
8009 // Store the old link SP.
8010 return DAG.getStore(Chain, dl, Val: LoadLinkSP, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
8011}
8012
8013SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const {
8014 MachineFunction &MF = DAG.getMachineFunction();
8015 bool isPPC64 = Subtarget.isPPC64();
8016 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8017
8018 // Get current frame pointer save index. The users of this index will be
8019 // primarily DYNALLOC instructions.
8020 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8021 int RASI = FI->getReturnAddrSaveIndex();
8022
8023 // If the frame pointer save index hasn't been defined yet.
8024 if (!RASI) {
8025 // Find out what the fix offset of the frame pointer save area.
8026 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset();
8027 // Allocate the frame index for frame pointer save area.
8028 RASI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: LROffset, IsImmutable: false);
8029 // Save the result.
8030 FI->setReturnAddrSaveIndex(RASI);
8031 }
8032 return DAG.getFrameIndex(FI: RASI, VT: PtrVT);
8033}
8034
8035SDValue
8036PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
8037 MachineFunction &MF = DAG.getMachineFunction();
8038 bool isPPC64 = Subtarget.isPPC64();
8039 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8040
8041 // Get current frame pointer save index. The users of this index will be
8042 // primarily DYNALLOC instructions.
8043 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8044 int FPSI = FI->getFramePointerSaveIndex();
8045
8046 // If the frame pointer save index hasn't been defined yet.
8047 if (!FPSI) {
8048 // Find out what the fix offset of the frame pointer save area.
8049 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset();
8050 // Allocate the frame index for frame pointer save area.
8051 FPSI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: FPOffset, IsImmutable: true);
8052 // Save the result.
8053 FI->setFramePointerSaveIndex(FPSI);
8054 }
8055 return DAG.getFrameIndex(FI: FPSI, VT: PtrVT);
8056}
8057
8058SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
8059 SelectionDAG &DAG) const {
8060 MachineFunction &MF = DAG.getMachineFunction();
8061 // Get the inputs.
8062 SDValue Chain = Op.getOperand(i: 0);
8063 SDValue Size = Op.getOperand(i: 1);
8064 SDLoc dl(Op);
8065
8066 // Get the correct type for pointers.
8067 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8068 // Negate the size.
8069 SDValue NegSize = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: PtrVT,
8070 N1: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), N2: Size);
8071 // Construct a node for the frame pointer save index.
8072 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
8073 SDValue Ops[3] = { Chain, NegSize, FPSIdx };
8074 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::Other);
8075 if (hasInlineStackProbe(MF))
8076 return DAG.getNode(Opcode: PPCISD::PROBED_ALLOCA, DL: dl, VTList: VTs, Ops);
8077 return DAG.getNode(Opcode: PPCISD::DYNALLOC, DL: dl, VTList: VTs, Ops);
8078}
8079
8080SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
8081 SelectionDAG &DAG) const {
8082 MachineFunction &MF = DAG.getMachineFunction();
8083
8084 bool isPPC64 = Subtarget.isPPC64();
8085 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8086
8087 int FI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64 ? 8 : 4, SPOffset: 0, IsImmutable: false);
8088 return DAG.getFrameIndex(FI, VT: PtrVT);
8089}
8090
8091SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
8092 SelectionDAG &DAG) const {
8093 SDLoc DL(Op);
8094 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_SETJMP, DL,
8095 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8096 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8097}
8098
8099SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
8100 SelectionDAG &DAG) const {
8101 SDLoc DL(Op);
8102 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_LONGJMP, DL, VT: MVT::Other,
8103 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8104}
8105
8106SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
8107 if (Op.getValueType().isVector())
8108 return LowerVectorLoad(Op, DAG);
8109
8110 assert(Op.getValueType() == MVT::i1 &&
8111 "Custom lowering only for i1 loads");
8112
8113 // First, load 8 bits into 32 bits, then truncate to 1 bit.
8114
8115 SDLoc dl(Op);
8116 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
8117
8118 SDValue Chain = LD->getChain();
8119 SDValue BasePtr = LD->getBasePtr();
8120 MachineMemOperand *MMO = LD->getMemOperand();
8121
8122 SDValue NewLD =
8123 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: getPointerTy(DL: DAG.getDataLayout()), Chain,
8124 Ptr: BasePtr, MemVT: MVT::i8, MMO);
8125 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewLD);
8126
8127 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
8128 return DAG.getMergeValues(Ops, dl);
8129}
8130
8131SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
8132 if (Op.getOperand(i: 1).getValueType().isVector())
8133 return LowerVectorStore(Op, DAG);
8134
8135 assert(Op.getOperand(1).getValueType() == MVT::i1 &&
8136 "Custom lowering only for i1 stores");
8137
8138 // First, zero extend to 32 bits, then use a truncating store to 8 bits.
8139
8140 SDLoc dl(Op);
8141 StoreSDNode *ST = cast<StoreSDNode>(Val&: Op);
8142
8143 SDValue Chain = ST->getChain();
8144 SDValue BasePtr = ST->getBasePtr();
8145 SDValue Value = ST->getValue();
8146 MachineMemOperand *MMO = ST->getMemOperand();
8147
8148 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
8149 Operand: Value);
8150 return DAG.getTruncStore(Chain, dl, Val: Value, Ptr: BasePtr, SVT: MVT::i8, MMO);
8151}
8152
8153// FIXME: Remove this once the ANDI glue bug is fixed:
8154SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
8155 assert(Op.getValueType() == MVT::i1 &&
8156 "Custom lowering only for i1 results");
8157
8158 SDLoc DL(Op);
8159 return DAG.getNode(Opcode: PPCISD::ANDI_rec_1_GT_BIT, DL, VT: MVT::i1, Operand: Op.getOperand(i: 0));
8160}
8161
8162SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
8163 SelectionDAG &DAG) const {
8164
8165 // Implements a vector truncate that fits in a vector register as a shuffle.
8166 // We want to legalize vector truncates down to where the source fits in
8167 // a vector register (and target is therefore smaller than vector register
8168 // size). At that point legalization will try to custom lower the sub-legal
8169 // result and get here - where we can contain the truncate as a single target
8170 // operation.
8171
8172 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows:
8173 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2>
8174 //
8175 // We will implement it for big-endian ordering as this (where x denotes
8176 // undefined):
8177 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to
8178 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u>
8179 //
8180 // The same operation in little-endian ordering will be:
8181 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to
8182 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1>
8183
8184 EVT TrgVT = Op.getValueType();
8185 assert(TrgVT.isVector() && "Vector type expected.");
8186 unsigned TrgNumElts = TrgVT.getVectorNumElements();
8187 EVT EltVT = TrgVT.getVectorElementType();
8188 if (!isOperationCustom(Op: Op.getOpcode(), VT: TrgVT) ||
8189 TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(Value: TrgNumElts) ||
8190 !llvm::has_single_bit<uint32_t>(Value: EltVT.getSizeInBits()))
8191 return SDValue();
8192
8193 SDValue N1 = Op.getOperand(i: 0);
8194 EVT SrcVT = N1.getValueType();
8195 unsigned SrcSize = SrcVT.getSizeInBits();
8196 if (SrcSize > 256 || !isPowerOf2_32(Value: SrcVT.getVectorNumElements()) ||
8197 !llvm::has_single_bit<uint32_t>(
8198 Value: SrcVT.getVectorElementType().getSizeInBits()))
8199 return SDValue();
8200 if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2)
8201 return SDValue();
8202
8203 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8204 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8205
8206 SDLoc DL(Op);
8207 SDValue Op1, Op2;
8208 if (SrcSize == 256) {
8209 EVT VecIdxTy = getVectorIdxTy(DL: DAG.getDataLayout());
8210 EVT SplitVT =
8211 N1.getValueType().getHalfNumVectorElementsVT(Context&: *DAG.getContext());
8212 unsigned SplitNumElts = SplitVT.getVectorNumElements();
8213 Op1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8214 N2: DAG.getConstant(Val: 0, DL, VT: VecIdxTy));
8215 Op2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8216 N2: DAG.getConstant(Val: SplitNumElts, DL, VT: VecIdxTy));
8217 }
8218 else {
8219 Op1 = SrcSize == 128 ? N1 : widenVec(DAG, Vec: N1, dl: DL);
8220 Op2 = DAG.getUNDEF(VT: WideVT);
8221 }
8222
8223 // First list the elements we want to keep.
8224 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits();
8225 SmallVector<int, 16> ShuffV;
8226 if (Subtarget.isLittleEndian())
8227 for (unsigned i = 0; i < TrgNumElts; ++i)
8228 ShuffV.push_back(Elt: i * SizeMult);
8229 else
8230 for (unsigned i = 1; i <= TrgNumElts; ++i)
8231 ShuffV.push_back(Elt: i * SizeMult - 1);
8232
8233 // Populate the remaining elements with undefs.
8234 for (unsigned i = TrgNumElts; i < WideNumElts; ++i)
8235 // ShuffV.push_back(i + WideNumElts);
8236 ShuffV.push_back(Elt: WideNumElts + 1);
8237
8238 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op1);
8239 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op2);
8240 return DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: Op1, N2: Op2, Mask: ShuffV);
8241}
8242
8243/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
8244/// possible.
8245SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
8246 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
8247 EVT ResVT = Op.getValueType();
8248 EVT CmpVT = Op.getOperand(i: 0).getValueType();
8249 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
8250 SDValue TV = Op.getOperand(i: 2), FV = Op.getOperand(i: 3);
8251 SDLoc dl(Op);
8252
8253 // Without power9-vector, we don't have native instruction for f128 comparison.
8254 // Following transformation to libcall is needed for setcc:
8255 // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE
8256 if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) {
8257 SDValue Z = DAG.getSetCC(
8258 DL: dl, VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: CmpVT),
8259 LHS, RHS, Cond: CC);
8260 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Z.getValueType());
8261 return DAG.getSelectCC(DL: dl, LHS: Z, RHS: Zero, True: TV, False: FV, Cond: ISD::SETNE);
8262 }
8263
8264 // Not FP, or using SPE? Not a fsel.
8265 if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() ||
8266 Subtarget.hasSPE())
8267 return Op;
8268
8269 SDNodeFlags Flags = Op.getNode()->getFlags();
8270
8271 // We have xsmaxc[dq]p/xsminc[dq]p which are OK to emit even in the
8272 // presence of infinities.
8273 if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) {
8274 switch (CC) {
8275 default:
8276 break;
8277 case ISD::SETOGT:
8278 case ISD::SETGT:
8279 return DAG.getNode(Opcode: PPCISD::XSMAXC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8280 case ISD::SETOLT:
8281 case ISD::SETLT:
8282 return DAG.getNode(Opcode: PPCISD::XSMINC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8283 }
8284 }
8285
8286 // We might be able to do better than this under some circumstances, but in
8287 // general, fsel-based lowering of select is a finite-math-only optimization.
8288 // For more information, see section F.3 of the 2.06 ISA specification.
8289 // With ISA 3.0
8290 if (!Flags.hasNoInfs() || !Flags.hasNoNaNs() || ResVT == MVT::f128)
8291 return Op;
8292
8293 // If the RHS of the comparison is a 0.0, we don't need to do the
8294 // subtraction at all.
8295 SDValue Sel1;
8296 if (isFloatingPointZero(Op: RHS))
8297 switch (CC) {
8298 default: break; // SETUO etc aren't handled by fsel.
8299 case ISD::SETNE:
8300 std::swap(a&: TV, b&: FV);
8301 [[fallthrough]];
8302 case ISD::SETEQ:
8303 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8304 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8305 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8306 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
8307 Sel1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Sel1);
8308 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8309 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: Sel1, N3: FV);
8310 case ISD::SETULT:
8311 case ISD::SETLT:
8312 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8313 [[fallthrough]];
8314 case ISD::SETOGE:
8315 case ISD::SETGE:
8316 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8317 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8318 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8319 case ISD::SETUGT:
8320 case ISD::SETGT:
8321 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8322 [[fallthrough]];
8323 case ISD::SETOLE:
8324 case ISD::SETLE:
8325 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8326 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8327 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8328 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: TV, N3: FV);
8329 }
8330
8331 SDValue Cmp;
8332 switch (CC) {
8333 default: break; // SETUO etc aren't handled by fsel.
8334 case ISD::SETNE:
8335 std::swap(a&: TV, b&: FV);
8336 [[fallthrough]];
8337 case ISD::SETEQ:
8338 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8339 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8340 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8341 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8342 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
8343 Sel1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Sel1);
8344 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8345 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: Cmp), N2: Sel1, N3: FV);
8346 case ISD::SETULT:
8347 case ISD::SETLT:
8348 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8349 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8350 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8351 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8352 case ISD::SETOGE:
8353 case ISD::SETGE:
8354 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8355 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8356 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8357 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8358 case ISD::SETUGT:
8359 case ISD::SETGT:
8360 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8361 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8362 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8363 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8364 case ISD::SETOLE:
8365 case ISD::SETLE:
8366 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8367 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8368 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8369 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8370 }
8371 return Op;
8372}
8373
8374static unsigned getPPCStrictOpcode(unsigned Opc) {
8375 switch (Opc) {
8376 default:
8377 llvm_unreachable("No strict version of this opcode!");
8378 case PPCISD::FCTIDZ:
8379 return PPCISD::STRICT_FCTIDZ;
8380 case PPCISD::FCTIWZ:
8381 return PPCISD::STRICT_FCTIWZ;
8382 case PPCISD::FCTIDUZ:
8383 return PPCISD::STRICT_FCTIDUZ;
8384 case PPCISD::FCTIWUZ:
8385 return PPCISD::STRICT_FCTIWUZ;
8386 case PPCISD::FCFID:
8387 return PPCISD::STRICT_FCFID;
8388 case PPCISD::FCFIDU:
8389 return PPCISD::STRICT_FCFIDU;
8390 case PPCISD::FCFIDS:
8391 return PPCISD::STRICT_FCFIDS;
8392 case PPCISD::FCFIDUS:
8393 return PPCISD::STRICT_FCFIDUS;
8394 }
8395}
8396
8397static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG,
8398 const PPCSubtarget &Subtarget) {
8399 SDLoc dl(Op);
8400 bool IsStrict = Op->isStrictFPOpcode();
8401 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8402 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8403
8404 // TODO: Any other flags to propagate?
8405 SDNodeFlags Flags;
8406 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8407
8408 // For strict nodes, source is the second operand.
8409 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8410 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8411 MVT DestTy = Op.getSimpleValueType();
8412 assert(Src.getValueType().isFloatingPoint() &&
8413 (DestTy == MVT::i8 || DestTy == MVT::i16 || DestTy == MVT::i32 ||
8414 DestTy == MVT::i64) &&
8415 "Invalid FP_TO_INT types");
8416 if (Src.getValueType() == MVT::f32) {
8417 if (IsStrict) {
8418 Src =
8419 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
8420 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8421 Chain = Src.getValue(R: 1);
8422 } else
8423 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
8424 }
8425 if ((DestTy == MVT::i8 || DestTy == MVT::i16) && Subtarget.hasP9Vector())
8426 DestTy = Subtarget.getScalarIntVT();
8427 unsigned Opc = ISD::DELETED_NODE;
8428 switch (DestTy.SimpleTy) {
8429 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
8430 case MVT::i32:
8431 Opc = IsSigned ? PPCISD::FCTIWZ
8432 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ);
8433 break;
8434 case MVT::i64:
8435 assert((IsSigned || Subtarget.hasFPCVT()) &&
8436 "i64 FP_TO_UINT is supported only with FPCVT");
8437 Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ;
8438 }
8439 EVT ConvTy = Src.getValueType() == MVT::f128 ? MVT::f128 : MVT::f64;
8440 SDValue Conv;
8441 if (IsStrict) {
8442 Opc = getPPCStrictOpcode(Opc);
8443 Conv = DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src},
8444 Flags);
8445 } else {
8446 Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: ConvTy, Operand: Src);
8447 }
8448 return Conv;
8449}
8450
8451void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
8452 SelectionDAG &DAG,
8453 const SDLoc &dl) const {
8454 SDValue Tmp = convertFPToInt(Op, DAG, Subtarget);
8455 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8456 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8457 bool IsStrict = Op->isStrictFPOpcode();
8458
8459 // Convert the FP value to an int value through memory.
8460 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
8461 (IsSigned || Subtarget.hasFPCVT());
8462 SDValue FIPtr = DAG.CreateStackTemporary(VT: i32Stack ? MVT::i32 : MVT::f64);
8463 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
8464 MachinePointerInfo MPI =
8465 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
8466
8467 // Emit a store to the stack slot.
8468 SDValue Chain = IsStrict ? Tmp.getValue(R: 1) : DAG.getEntryNode();
8469 Align Alignment(DAG.getEVTAlign(MemoryVT: Tmp.getValueType()));
8470 if (i32Stack) {
8471 MachineFunction &MF = DAG.getMachineFunction();
8472 Alignment = Align(4);
8473 MachineMemOperand *MMO =
8474 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Alignment);
8475 SDValue Ops[] = { Chain, Tmp, FIPtr };
8476 Chain = DAG.getMemIntrinsicNode(Opcode: PPCISD::STFIWX, dl,
8477 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT: MVT::i32, MMO);
8478 } else
8479 Chain = DAG.getStore(Chain, dl, Val: Tmp, Ptr: FIPtr, PtrInfo: MPI, Alignment);
8480
8481 // Result is a load from the stack slot. If loading 4 bytes, make sure to
8482 // add in a bias on big endian.
8483 if (Op.getValueType() == MVT::i32 && !i32Stack &&
8484 !Subtarget.isLittleEndian()) {
8485 FIPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FIPtr.getValueType(), N1: FIPtr,
8486 N2: DAG.getConstant(Val: 4, DL: dl, VT: FIPtr.getValueType()));
8487 MPI = MPI.getWithOffset(O: 4);
8488 }
8489
8490 RLI.Chain = Chain;
8491 RLI.Ptr = FIPtr;
8492 RLI.MPI = MPI;
8493 RLI.Alignment = Alignment;
8494}
8495
8496/// Custom lowers floating point to integer conversions to use
8497/// the direct move instructions available in ISA 2.07 to avoid the
8498/// need for load/store combinations.
8499SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
8500 SelectionDAG &DAG,
8501 const SDLoc &dl) const {
8502 SDValue Conv = convertFPToInt(Op, DAG, Subtarget);
8503 SDValue Mov = DAG.getNode(Opcode: PPCISD::MFVSR, DL: dl, VT: Op.getValueType(), Operand: Conv);
8504 if (Op->isStrictFPOpcode())
8505 return DAG.getMergeValues(Ops: {Mov, Conv.getValue(R: 1)}, dl);
8506 else
8507 return Mov;
8508}
8509
8510SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
8511 const SDLoc &dl) const {
8512 bool IsStrict = Op->isStrictFPOpcode();
8513 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8514 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8515 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8516 EVT SrcVT = Src.getValueType();
8517 EVT DstVT = Op.getValueType();
8518
8519 // FP to INT conversions are legal for f128.
8520 if (SrcVT == MVT::f128)
8521 return Subtarget.hasP9Vector() ? Op : SDValue();
8522
8523 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
8524 // PPC (the libcall is not available).
8525 if (SrcVT == MVT::ppcf128) {
8526 if (DstVT == MVT::i32) {
8527 // TODO: Conservatively pass only nofpexcept flag here. Need to check and
8528 // set other fast-math flags to FP operations in both strict and
8529 // non-strict cases. (FP_TO_SINT, FSUB)
8530 SDNodeFlags Flags;
8531 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8532
8533 if (IsSigned) {
8534 SDValue Lo, Hi;
8535 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: dl, LoVT: MVT::f64, HiVT: MVT::f64);
8536
8537 // Add the two halves of the long double in round-to-zero mode, and use
8538 // a smaller FP_TO_SINT.
8539 if (IsStrict) {
8540 SDValue Res = DAG.getNode(Opcode: PPCISD::STRICT_FADDRTZ, DL: dl,
8541 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8542 Ops: {Op.getOperand(i: 0), Lo, Hi}, Flags);
8543 return DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8544 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8545 Ops: {Res.getValue(R: 1), Res}, Flags);
8546 } else {
8547 SDValue Res = DAG.getNode(Opcode: PPCISD::FADDRTZ, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
8548 return DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Res);
8549 }
8550 } else {
8551 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
8552 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31));
8553 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
8554 SDValue SignMask = DAG.getConstant(Val: 0x80000000, DL: dl, VT: DstVT);
8555 if (IsStrict) {
8556 // Sel = Src < 0x80000000
8557 // FltOfs = select Sel, 0.0, 0x80000000
8558 // IntOfs = select Sel, 0, 0x80000000
8559 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
8560 SDValue Chain = Op.getOperand(i: 0);
8561 EVT SetCCVT =
8562 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: SrcVT);
8563 EVT DstSetCCVT =
8564 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: DstVT);
8565 SDValue Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
8566 Chain, IsSignaling: true);
8567 Chain = Sel.getValue(R: 1);
8568
8569 SDValue FltOfs = DAG.getSelect(
8570 DL: dl, VT: SrcVT, Cond: Sel, LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
8571 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
8572
8573 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl,
8574 VTList: DAG.getVTList(VT1: SrcVT, VT2: MVT::Other),
8575 Ops: {Chain, Src, FltOfs}, Flags);
8576 Chain = Val.getValue(R: 1);
8577 SDValue SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8578 VTList: DAG.getVTList(VT1: DstVT, VT2: MVT::Other),
8579 Ops: {Chain, Val}, Flags);
8580 Chain = SInt.getValue(R: 1);
8581 SDValue IntOfs = DAG.getSelect(
8582 DL: dl, VT: DstVT, Cond: Sel, LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), RHS: SignMask);
8583 SDValue Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
8584 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
8585 } else {
8586 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
8587 // FIXME: generated code sucks.
8588 SDValue True = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::ppcf128, N1: Src, N2: Cst);
8589 True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: True);
8590 True = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: True, N2: SignMask);
8591 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Src);
8592 return DAG.getSelectCC(DL: dl, LHS: Src, RHS: Cst, True, False, Cond: ISD::SETGE);
8593 }
8594 }
8595 }
8596
8597 return SDValue();
8598 }
8599
8600 if (Subtarget.hasDirectMove() && Subtarget.isPPC64())
8601 return LowerFP_TO_INTDirectMove(Op, DAG, dl);
8602
8603 ReuseLoadInfo RLI;
8604 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8605
8606 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI,
8607 Alignment: RLI.Alignment, MMOFlags: RLI.MMOFlags(),
8608 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8609}
8610
8611// We're trying to insert a regular store, S, and then a load, L. If the
8612// incoming value, O, is a load, we might just be able to have our load use the
8613// address used by O. However, we don't know if anything else will store to
8614// that address before we can load from it. To prevent this situation, we need
8615// to insert our load, L, into the chain as a peer of O. To do this, we give L
8616// the same chain operand as O, we create a token factor from the chain results
8617// of O and L, and we replace all uses of O's chain result with that token
8618// factor (this last part is handled by makeEquivalentMemoryOrdering).
8619bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
8620 ReuseLoadInfo &RLI,
8621 SelectionDAG &DAG,
8622 ISD::LoadExtType ET) const {
8623 // Conservatively skip reusing for constrained FP nodes.
8624 if (Op->isStrictFPOpcode())
8625 return false;
8626
8627 SDLoc dl(Op);
8628 bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT &&
8629 (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32);
8630 if (ET == ISD::NON_EXTLOAD &&
8631 (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) &&
8632 isOperationLegalOrCustom(Op: Op.getOpcode(),
8633 VT: Op.getOperand(i: 0).getValueType())) {
8634
8635 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8636 return true;
8637 }
8638
8639 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op);
8640 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() ||
8641 LD->isNonTemporal())
8642 return false;
8643 if (LD->getMemoryVT() != MemVT)
8644 return false;
8645
8646 // If the result of the load is an illegal type, then we can't build a
8647 // valid chain for reuse since the legalised loads and token factor node that
8648 // ties the legalised loads together uses a different output chain then the
8649 // illegal load.
8650 if (!isTypeLegal(VT: LD->getValueType(ResNo: 0)))
8651 return false;
8652
8653 RLI.Ptr = LD->getBasePtr();
8654 if (LD->isIndexed() && !LD->getOffset().isUndef()) {
8655 assert(LD->getAddressingMode() == ISD::PRE_INC &&
8656 "Non-pre-inc AM on PPC?");
8657 RLI.Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RLI.Ptr.getValueType(), N1: RLI.Ptr,
8658 N2: LD->getOffset());
8659 }
8660
8661 RLI.Chain = LD->getChain();
8662 RLI.MPI = LD->getPointerInfo();
8663 RLI.IsDereferenceable = LD->isDereferenceable();
8664 RLI.IsInvariant = LD->isInvariant();
8665 RLI.Alignment = LD->getAlign();
8666 RLI.AAInfo = LD->getAAInfo();
8667 RLI.Ranges = LD->getRanges();
8668
8669 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1);
8670 return true;
8671}
8672
8673/// Analyze profitability of direct move
8674/// prefer float load to int load plus direct move
8675/// when there is no integer use of int load
8676bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
8677 SDNode *Origin = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0).getNode();
8678 if (Origin->getOpcode() != ISD::LOAD)
8679 return true;
8680
8681 // If there is no LXSIBZX/LXSIHZX, like Power8,
8682 // prefer direct move if the memory size is 1 or 2 bytes.
8683 MachineMemOperand *MMO = cast<LoadSDNode>(Val: Origin)->getMemOperand();
8684 if (!Subtarget.hasP9Vector() &&
8685 (!MMO->getSize().hasValue() || MMO->getSize().getValue() <= 2))
8686 return true;
8687
8688 for (SDUse &Use : Origin->uses()) {
8689
8690 // Only look at the users of the loaded value.
8691 if (Use.getResNo() != 0)
8692 continue;
8693
8694 SDNode *User = Use.getUser();
8695 if (User->getOpcode() != ISD::SINT_TO_FP &&
8696 User->getOpcode() != ISD::UINT_TO_FP &&
8697 User->getOpcode() != ISD::STRICT_SINT_TO_FP &&
8698 User->getOpcode() != ISD::STRICT_UINT_TO_FP)
8699 return true;
8700 }
8701
8702 return false;
8703}
8704
8705static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG,
8706 const PPCSubtarget &Subtarget,
8707 SDValue Chain = SDValue()) {
8708 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8709 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8710 SDLoc dl(Op);
8711
8712 // TODO: Any other flags to propagate?
8713 SDNodeFlags Flags;
8714 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8715
8716 // If we have FCFIDS, then use it when converting to single-precision.
8717 // Otherwise, convert to double-precision and then round.
8718 bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT();
8719 unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS)
8720 : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU);
8721 EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64;
8722 if (Op->isStrictFPOpcode()) {
8723 if (!Chain)
8724 Chain = Op.getOperand(i: 0);
8725 return DAG.getNode(Opcode: getPPCStrictOpcode(Opc: ConvOpc), DL: dl,
8726 VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8727 } else
8728 return DAG.getNode(Opcode: ConvOpc, DL: dl, VT: ConvTy, Operand: Src);
8729}
8730
8731/// Custom lowers integer to floating point conversions to use
8732/// the direct move instructions available in ISA 2.07 to avoid the
8733/// need for load/store combinations.
8734SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
8735 SelectionDAG &DAG,
8736 const SDLoc &dl) const {
8737 assert((Op.getValueType() == MVT::f32 ||
8738 Op.getValueType() == MVT::f64) &&
8739 "Invalid floating point type as target of conversion");
8740 assert(Subtarget.hasFPCVT() &&
8741 "Int to FP conversions with direct moves require FPCVT");
8742 SDValue Src = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
8743 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32;
8744 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP ||
8745 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8746 unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA;
8747 SDValue Mov = DAG.getNode(Opcode: MovOpc, DL: dl, VT: MVT::f64, Operand: Src);
8748 return convertIntToFP(Op, Src: Mov, DAG, Subtarget);
8749}
8750
8751static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
8752
8753 EVT VecVT = Vec.getValueType();
8754 assert(VecVT.isVector() && "Expected a vector type.");
8755 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width.");
8756
8757 EVT EltVT = VecVT.getVectorElementType();
8758 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8759 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8760
8761 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements();
8762 SmallVector<SDValue, 16> Ops(NumConcat);
8763 Ops[0] = Vec;
8764 SDValue UndefVec = DAG.getUNDEF(VT: VecVT);
8765 for (unsigned i = 1; i < NumConcat; ++i)
8766 Ops[i] = UndefVec;
8767
8768 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WideVT, Ops);
8769}
8770
8771SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
8772 const SDLoc &dl) const {
8773 bool IsStrict = Op->isStrictFPOpcode();
8774 unsigned Opc = Op.getOpcode();
8775 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8776 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP ||
8777 Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) &&
8778 "Unexpected conversion type");
8779 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) &&
8780 "Supports conversions to v2f64/v4f32 only.");
8781
8782 // TODO: Any other flags to propagate?
8783 SDNodeFlags Flags;
8784 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8785
8786 bool SignedConv = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
8787 bool FourEltRes = Op.getValueType() == MVT::v4f32;
8788
8789 SDValue Wide = widenVec(DAG, Vec: Src, dl);
8790 EVT WideVT = Wide.getValueType();
8791 unsigned WideNumElts = WideVT.getVectorNumElements();
8792 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64;
8793
8794 SmallVector<int, 16> ShuffV;
8795 for (unsigned i = 0; i < WideNumElts; ++i)
8796 ShuffV.push_back(Elt: i + WideNumElts);
8797
8798 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2;
8799 int SaveElts = FourEltRes ? 4 : 2;
8800 if (Subtarget.isLittleEndian())
8801 for (int i = 0; i < SaveElts; i++)
8802 ShuffV[i * Stride] = i;
8803 else
8804 for (int i = 1; i <= SaveElts; i++)
8805 ShuffV[i * Stride - 1] = i - 1;
8806
8807 SDValue ShuffleSrc2 =
8808 SignedConv ? DAG.getUNDEF(VT: WideVT) : DAG.getConstant(Val: 0, DL: dl, VT: WideVT);
8809 SDValue Arrange = DAG.getVectorShuffle(VT: WideVT, dl, N1: Wide, N2: ShuffleSrc2, Mask: ShuffV);
8810
8811 SDValue Extend;
8812 if (SignedConv) {
8813 Arrange = DAG.getBitcast(VT: IntermediateVT, V: Arrange);
8814 EVT ExtVT = Src.getValueType();
8815 if (Subtarget.hasP9Altivec())
8816 ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideVT.getVectorElementType(),
8817 NumElements: IntermediateVT.getVectorNumElements());
8818
8819 Extend = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: IntermediateVT, N1: Arrange,
8820 N2: DAG.getValueType(ExtVT));
8821 } else
8822 Extend = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntermediateVT, Operand: Arrange);
8823
8824 if (IsStrict)
8825 return DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other),
8826 Ops: {Op.getOperand(i: 0), Extend}, Flags);
8827
8828 return DAG.getNode(Opcode: Opc, DL: dl, VT: Op.getValueType(), Operand: Extend);
8829}
8830
8831SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
8832 SelectionDAG &DAG) const {
8833 SDLoc dl(Op);
8834 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8835 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8836 bool IsStrict = Op->isStrictFPOpcode();
8837 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8838 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
8839
8840 // TODO: Any other flags to propagate?
8841 SDNodeFlags Flags;
8842 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8843
8844 EVT InVT = Src.getValueType();
8845 EVT OutVT = Op.getValueType();
8846 if (OutVT.isVector() && OutVT.isFloatingPoint() &&
8847 isOperationCustom(Op: Op.getOpcode(), VT: InVT))
8848 return LowerINT_TO_FPVector(Op, DAG, dl);
8849
8850 // Conversions to f128 are legal.
8851 if (Op.getValueType() == MVT::f128)
8852 return Subtarget.hasP9Vector() ? Op : SDValue();
8853
8854 // Don't handle ppc_fp128 here; let it be lowered to a libcall.
8855 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
8856 return SDValue();
8857
8858 if (Src.getValueType() == MVT::i1) {
8859 SDValue Sel = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: Op.getValueType(), N1: Src,
8860 N2: DAG.getConstantFP(Val: 1.0, DL: dl, VT: Op.getValueType()),
8861 N3: DAG.getConstantFP(Val: 0.0, DL: dl, VT: Op.getValueType()));
8862 if (IsStrict)
8863 return DAG.getMergeValues(Ops: {Sel, Chain}, dl);
8864 else
8865 return Sel;
8866 }
8867
8868 // If we have direct moves, we can do all the conversion, skip the store/load
8869 // however, without FPCVT we can't do most conversions.
8870 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) &&
8871 Subtarget.isPPC64() && Subtarget.hasFPCVT())
8872 return LowerINT_TO_FPDirectMove(Op, DAG, dl);
8873
8874 assert((IsSigned || Subtarget.hasFPCVT()) &&
8875 "UINT_TO_FP is supported only with FPCVT");
8876
8877 if (Src.getValueType() == MVT::i64) {
8878 SDValue SINT = Src;
8879 // When converting to single-precision, we actually need to convert
8880 // to double-precision first and then round to single-precision.
8881 // To avoid double-rounding effects during that operation, we have
8882 // to prepare the input operand. Bits that might be truncated when
8883 // converting to double-precision are replaced by a bit that won't
8884 // be lost at this stage, but is below the single-precision rounding
8885 // position.
8886 //
8887 // However, if afn is in effect, accept double
8888 // rounding to avoid the extra overhead.
8889 // FIXME: Currently INT_TO_FP can't support fast math flags because
8890 // of nneg flag, thus Op->getFlags().hasApproximateFuncs() is always
8891 // false.
8892 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT() &&
8893 !Op->getFlags().hasApproximateFuncs()) {
8894
8895 // Twiddle input to make sure the low 11 bits are zero. (If this
8896 // is the case, we are guaranteed the value will fit into the 53 bit
8897 // mantissa of an IEEE double-precision value without rounding.)
8898 // If any of those low 11 bits were not zero originally, make sure
8899 // bit 12 (value 2048) is set instead, so that the final rounding
8900 // to single-precision gets the correct result.
8901 SDValue Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64,
8902 N1: SINT, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8903 Round = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8904 N1: Round, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8905 Round = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i64, N1: Round, N2: SINT);
8906 Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64, N1: Round,
8907 N2: DAG.getSignedConstant(Val: -2048, DL: dl, VT: MVT::i64));
8908
8909 // However, we cannot use that value unconditionally: if the magnitude
8910 // of the input value is small, the bit-twiddling we did above might
8911 // end up visibly changing the output. Fortunately, in that case, we
8912 // don't need to twiddle bits since the original input will convert
8913 // exactly to double-precision floating-point already. Therefore,
8914 // construct a conditional to use the original value if the top 11
8915 // bits are all sign-bit copies, and use the rounded value computed
8916 // above otherwise.
8917 SDValue Cond = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: MVT::i64,
8918 N1: SINT, N2: DAG.getConstant(Val: 53, DL: dl, VT: MVT::i32));
8919 Cond = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8920 N1: Cond, N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64));
8921 Cond = DAG.getSetCC(
8922 DL: dl,
8923 VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: MVT::i64),
8924 LHS: Cond, RHS: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64), Cond: ISD::SETUGT);
8925
8926 SINT = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i64, N1: Cond, N2: Round, N3: SINT);
8927 }
8928
8929 ReuseLoadInfo RLI;
8930 SDValue Bits;
8931
8932 MachineFunction &MF = DAG.getMachineFunction();
8933 if (canReuseLoadAddress(Op: SINT, MemVT: MVT::i64, RLI, DAG)) {
8934 // Drop range metadata, as this metadata becomes invalid for f64 bit
8935 // reinterpretation of i64 values.
8936 Bits =
8937 DAG.getLoad(VT: MVT::f64, dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI, Alignment: RLI.Alignment,
8938 MMOFlags: RLI.MMOFlags(), Metadata: MMOMetadata(RLI.AAInfo));
8939 if (RLI.ResChain)
8940 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8941 } else if (Subtarget.hasLFIWAX() &&
8942 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::SEXTLOAD)) {
8943 MachineMemOperand *MMO = MF.getMachineMemOperand(
8944 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8945 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8946 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8947 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWAX, dl,
8948 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8949 Ops, MemVT: MVT::i32, MMO);
8950 if (RLI.ResChain)
8951 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8952 } else if (Subtarget.hasFPCVT() &&
8953 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::ZEXTLOAD)) {
8954 MachineMemOperand *MMO = MF.getMachineMemOperand(
8955 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8956 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8957 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8958 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWZX, dl,
8959 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8960 Ops, MemVT: MVT::i32, MMO);
8961 if (RLI.ResChain)
8962 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8963 } else if (((Subtarget.hasLFIWAX() &&
8964 SINT.getOpcode() == ISD::SIGN_EXTEND) ||
8965 (Subtarget.hasFPCVT() &&
8966 SINT.getOpcode() == ISD::ZERO_EXTEND)) &&
8967 SINT.getOperand(i: 0).getValueType() == MVT::i32) {
8968 MachineFrameInfo &MFI = MF.getFrameInfo();
8969 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8970
8971 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
8972 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
8973
8974 SDValue Store = DAG.getStore(Chain, dl, Val: SINT.getOperand(i: 0), Ptr: FIdx,
8975 PtrInfo: MachinePointerInfo::getFixedStack(
8976 MF&: DAG.getMachineFunction(), FI: FrameIdx));
8977 Chain = Store;
8978
8979 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
8980 "Expected an i32 store");
8981
8982 RLI.Ptr = FIdx;
8983 RLI.Chain = Chain;
8984 RLI.MPI =
8985 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
8986 RLI.Alignment = Align(4);
8987
8988 MachineMemOperand *MMO = MF.getMachineMemOperand(
8989 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8990 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8991 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8992 Bits = DAG.getMemIntrinsicNode(Opcode: SINT.getOpcode() == ISD::ZERO_EXTEND ?
8993 PPCISD::LFIWZX : PPCISD::LFIWAX,
8994 dl, VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8995 Ops, MemVT: MVT::i32, MMO);
8996 Chain = Bits.getValue(R: 1);
8997 } else
8998 Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: SINT);
8999
9000 SDValue FP = convertIntToFP(Op, Src: Bits, DAG, Subtarget, Chain);
9001 if (IsStrict)
9002 Chain = FP.getValue(R: 1);
9003
9004 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
9005 if (IsStrict)
9006 FP = DAG.getNode(
9007 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
9008 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)},
9009 Flags);
9010 else
9011 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9012 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9013 }
9014 return FP;
9015 }
9016
9017 assert(Src.getValueType() == MVT::i32 &&
9018 "Unhandled INT_TO_FP type in custom expander!");
9019 // Since we only generate this in 64-bit mode, we can take advantage of
9020 // 64-bit registers. In particular, sign extend the input value into the
9021 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack
9022 // then lfd it and fcfid it.
9023 MachineFunction &MF = DAG.getMachineFunction();
9024 MachineFrameInfo &MFI = MF.getFrameInfo();
9025 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9026
9027 SDValue Ld;
9028 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
9029 ReuseLoadInfo RLI;
9030 bool ReusingLoad;
9031 if (!(ReusingLoad = canReuseLoadAddress(Op: Src, MemVT: MVT::i32, RLI, DAG))) {
9032 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
9033 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9034
9035 SDValue Store = DAG.getStore(Chain, dl, Val: Src, Ptr: FIdx,
9036 PtrInfo: MachinePointerInfo::getFixedStack(
9037 MF&: DAG.getMachineFunction(), FI: FrameIdx));
9038 Chain = Store;
9039
9040 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
9041 "Expected an i32 store");
9042
9043 RLI.Ptr = FIdx;
9044 RLI.Chain = Chain;
9045 RLI.MPI =
9046 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
9047 RLI.Alignment = Align(4);
9048 }
9049
9050 MachineMemOperand *MMO = MF.getMachineMemOperand(
9051 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
9052 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
9053 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
9054 Ld = DAG.getMemIntrinsicNode(Opcode: IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
9055 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops,
9056 MemVT: MVT::i32, MMO);
9057 Chain = Ld.getValue(R: 1);
9058 if (ReusingLoad && RLI.ResChain) {
9059 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Ld.getValue(R: 1));
9060 }
9061 } else {
9062 assert(Subtarget.isPPC64() &&
9063 "i32->FP without LFIWAX supported only on PPC64");
9064
9065 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9066 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9067
9068 SDValue Ext64 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i64, Operand: Src);
9069
9070 // STD the extended value into the stack slot.
9071 SDValue Store = DAG.getStore(
9072 Chain, dl, Val: Ext64, Ptr: FIdx,
9073 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9074 Chain = Store;
9075
9076 // Load the value as a double.
9077 Ld = DAG.getLoad(
9078 VT: MVT::f64, dl, Chain, Ptr: FIdx,
9079 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9080 Chain = Ld.getValue(R: 1);
9081 }
9082
9083 // FCFID it and return it.
9084 SDValue FP = convertIntToFP(Op, Src: Ld, DAG, Subtarget, Chain);
9085 if (IsStrict)
9086 Chain = FP.getValue(R: 1);
9087 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
9088 if (IsStrict)
9089 FP = DAG.getNode(
9090 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
9091 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)}, Flags);
9092 else
9093 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9094 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9095 }
9096 return FP;
9097}
9098
9099SDValue PPCTargetLowering::LowerSET_ROUNDING(SDValue Op,
9100 SelectionDAG &DAG) const {
9101 SDLoc Dl(Op);
9102 MachineFunction &MF = DAG.getMachineFunction();
9103 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9104 SDValue Chain = Op.getOperand(i: 0);
9105
9106 // If requested mode is constant, just use simpler mtfsb/mffscrni
9107 if (auto *CVal = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
9108 uint64_t Mode = CVal->getZExtValue();
9109 assert(Mode < 4 && "Unsupported rounding mode!");
9110 unsigned InternalRnd = Mode ^ (~(Mode >> 1) & 1);
9111 if (Subtarget.isISA3_0())
9112 return SDValue(
9113 DAG.getMachineNode(
9114 Opcode: PPC::MFFSCRNI, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9115 Ops: {DAG.getConstant(Val: InternalRnd, DL: Dl, VT: MVT::i32, isTarget: true), Chain}),
9116 1);
9117 SDNode *SetHi = DAG.getMachineNode(
9118 Opcode: (InternalRnd & 2) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9119 Ops: {DAG.getConstant(Val: 30, DL: Dl, VT: MVT::i32, isTarget: true), Chain});
9120 SDNode *SetLo = DAG.getMachineNode(
9121 Opcode: (InternalRnd & 1) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9122 Ops: {DAG.getConstant(Val: 31, DL: Dl, VT: MVT::i32, isTarget: true), SDValue(SetHi, 0)});
9123 return SDValue(SetLo, 0);
9124 }
9125
9126 // Use x ^ (~(x >> 1) & 1) to transform LLVM rounding mode to Power format.
9127 SDValue One = DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32);
9128 SDValue SrcFlag = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
9129 N2: DAG.getConstant(Val: 3, DL: Dl, VT: MVT::i32));
9130 SDValue DstFlag = DAG.getNode(
9131 Opcode: ISD::XOR, DL: Dl, VT: MVT::i32, N1: SrcFlag,
9132 N2: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32,
9133 N1: DAG.getNOT(DL: Dl,
9134 Val: DAG.getNode(Opcode: ISD::SRL, DL: Dl, VT: MVT::i32, N1: SrcFlag, N2: One),
9135 VT: MVT::i32),
9136 N2: One));
9137 // For Power9, there's faster mffscrn, and we don't need to read FPSCR
9138 SDValue MFFS;
9139 if (!Subtarget.isISA3_0()) {
9140 MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: Dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9141 Chain = MFFS.getValue(R: 1);
9142 }
9143 SDValue NewFPSCR;
9144 if (Subtarget.isPPC64()) {
9145 if (Subtarget.isISA3_0()) {
9146 NewFPSCR = DAG.getAnyExtOrTrunc(Op: DstFlag, DL: Dl, VT: MVT::i64);
9147 } else {
9148 // Set the last two bits (rounding mode) of bitcasted FPSCR.
9149 SDNode *InsertRN = DAG.getMachineNode(
9150 Opcode: PPC::RLDIMI, dl: Dl, VT: MVT::i64,
9151 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::i64, Operand: MFFS),
9152 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: Dl, VT: MVT::i64, Operand: DstFlag),
9153 DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9154 DAG.getTargetConstant(Val: 62, DL: Dl, VT: MVT::i32)});
9155 NewFPSCR = SDValue(InsertRN, 0);
9156 }
9157 NewFPSCR = DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::f64, Operand: NewFPSCR);
9158 } else {
9159 // In 32-bit mode, store f64, load and update the lower half.
9160 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9161 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9162 SDValue Addr = Subtarget.isLittleEndian()
9163 ? StackSlot
9164 : DAG.getNode(Opcode: ISD::ADD, DL: Dl, VT: PtrVT, N1: StackSlot,
9165 N2: DAG.getConstant(Val: 4, DL: Dl, VT: PtrVT));
9166 if (Subtarget.isISA3_0()) {
9167 Chain = DAG.getStore(Chain, dl: Dl, Val: DstFlag, Ptr: Addr, PtrInfo: MachinePointerInfo());
9168 } else {
9169 Chain = DAG.getStore(Chain, dl: Dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9170 SDValue Tmp =
9171 DAG.getLoad(VT: MVT::i32, dl: Dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9172 Chain = Tmp.getValue(R: 1);
9173 Tmp = SDValue(DAG.getMachineNode(
9174 Opcode: PPC::RLWIMI, dl: Dl, VT: MVT::i32,
9175 Ops: {Tmp, DstFlag, DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9176 DAG.getTargetConstant(Val: 30, DL: Dl, VT: MVT::i32),
9177 DAG.getTargetConstant(Val: 31, DL: Dl, VT: MVT::i32)}),
9178 0);
9179 Chain = DAG.getStore(Chain, dl: Dl, Val: Tmp, Ptr: Addr, PtrInfo: MachinePointerInfo());
9180 }
9181 NewFPSCR =
9182 DAG.getLoad(VT: MVT::f64, dl: Dl, Chain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9183 Chain = NewFPSCR.getValue(R: 1);
9184 }
9185 if (Subtarget.isISA3_0())
9186 return SDValue(DAG.getMachineNode(Opcode: PPC::MFFSCRN, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9187 Ops: {NewFPSCR, Chain}),
9188 1);
9189 SDValue Zero = DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32, isTarget: true);
9190 SDNode *MTFSF = DAG.getMachineNode(
9191 Opcode: PPC::MTFSF, dl: Dl, VT: MVT::Other,
9192 Ops: {DAG.getConstant(Val: 255, DL: Dl, VT: MVT::i32, isTarget: true), NewFPSCR, Zero, Zero, Chain});
9193 return SDValue(MTFSF, 0);
9194}
9195
9196SDValue PPCTargetLowering::LowerGET_ROUNDING(SDValue Op,
9197 SelectionDAG &DAG) const {
9198 SDLoc dl(Op);
9199 /*
9200 The rounding mode is in bits 30:31 of FPSR, and has the following
9201 settings:
9202 00 Round to nearest
9203 01 Round to 0
9204 10 Round to +inf
9205 11 Round to -inf
9206
9207 GET_ROUNDING, on the other hand, expects the following:
9208 -1 Undefined
9209 0 Round to 0
9210 1 Round to nearest
9211 2 Round to +inf
9212 3 Round to -inf
9213
9214 To perform the conversion, we do:
9215 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1))
9216 */
9217
9218 MachineFunction &MF = DAG.getMachineFunction();
9219 EVT VT = Op.getValueType();
9220 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9221
9222 // Save FP Control Word to register
9223 SDValue Chain = Op.getOperand(i: 0);
9224 SDValue MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9225 Chain = MFFS.getValue(R: 1);
9226
9227 SDValue CWD;
9228 if (isTypeLegal(VT: MVT::i64)) {
9229 CWD = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
9230 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: MFFS));
9231 } else {
9232 // Save FP register to stack slot
9233 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9234 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9235 Chain = DAG.getStore(Chain, dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9236
9237 // Load FP Control Word from low 32 bits of stack slot.
9238 assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) &&
9239 "Stack slot adjustment is valid only on big endian subtargets!");
9240 SDValue Four = DAG.getConstant(Val: 4, DL: dl, VT: PtrVT);
9241 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackSlot, N2: Four);
9242 CWD = DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9243 Chain = CWD.getValue(R: 1);
9244 }
9245
9246 // Transform as necessary
9247 SDValue CWD1 =
9248 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9249 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
9250 SDValue CWD2 =
9251 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9252 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9253 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32,
9254 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9255 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9256 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
9257
9258 SDValue RetVal =
9259 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1, N2: CWD2);
9260
9261 RetVal =
9262 DAG.getNode(Opcode: (VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND),
9263 DL: dl, VT, Operand: RetVal);
9264
9265 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
9266}
9267
9268SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9269 EVT VT = Op.getValueType();
9270 uint64_t BitWidth = VT.getSizeInBits();
9271 SDLoc dl(Op);
9272 assert(Op.getNumOperands() == 3 &&
9273 VT == Op.getOperand(1).getValueType() &&
9274 "Unexpected SHL!");
9275
9276 // Expand into a bunch of logical ops. Note that these ops
9277 // depend on the PPC behavior for oversized shift amounts.
9278 SDValue Lo = Op.getOperand(i: 0);
9279 SDValue Hi = Op.getOperand(i: 1);
9280 SDValue Amt = Op.getOperand(i: 2);
9281 EVT AmtVT = Amt.getValueType();
9282
9283 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9284 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9285 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Amt);
9286 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Tmp1);
9287 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR , DL: dl, VT, N1: Tmp2, N2: Tmp3);
9288 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9289 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9290 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Tmp5);
9291 SDValue OutHi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9292 SDValue OutLo = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Amt);
9293 SDValue OutOps[] = { OutLo, OutHi };
9294 return DAG.getMergeValues(Ops: OutOps, dl);
9295}
9296
9297SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9298 EVT VT = Op.getValueType();
9299 SDLoc dl(Op);
9300 uint64_t BitWidth = VT.getSizeInBits();
9301 assert(Op.getNumOperands() == 3 &&
9302 VT == Op.getOperand(1).getValueType() &&
9303 "Unexpected SRL!");
9304
9305 // Expand into a bunch of logical ops. Note that these ops
9306 // depend on the PPC behavior for oversized shift amounts.
9307 SDValue Lo = Op.getOperand(i: 0);
9308 SDValue Hi = Op.getOperand(i: 1);
9309 SDValue Amt = Op.getOperand(i: 2);
9310 EVT AmtVT = Amt.getValueType();
9311
9312 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9313 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9314 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9315 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9316 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9317 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9318 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9319 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Tmp5);
9320 SDValue OutLo = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9321 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Amt);
9322 SDValue OutOps[] = { OutLo, OutHi };
9323 return DAG.getMergeValues(Ops: OutOps, dl);
9324}
9325
9326SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
9327 SDLoc dl(Op);
9328 EVT VT = Op.getValueType();
9329 uint64_t BitWidth = VT.getSizeInBits();
9330 assert(Op.getNumOperands() == 3 &&
9331 VT == Op.getOperand(1).getValueType() &&
9332 "Unexpected SRA!");
9333
9334 // Expand into a bunch of logical ops, followed by a select_cc.
9335 SDValue Lo = Op.getOperand(i: 0);
9336 SDValue Hi = Op.getOperand(i: 1);
9337 SDValue Amt = Op.getOperand(i: 2);
9338 EVT AmtVT = Amt.getValueType();
9339
9340 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9341 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9342 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9343 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9344 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9345 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9346 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9347 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Tmp5);
9348 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Amt);
9349 SDValue OutLo = DAG.getSelectCC(DL: dl, LHS: Tmp5, RHS: DAG.getConstant(Val: 0, DL: dl, VT: AmtVT),
9350 True: Tmp4, False: Tmp6, Cond: ISD::SETLE);
9351 SDValue OutOps[] = { OutLo, OutHi };
9352 return DAG.getMergeValues(Ops: OutOps, dl);
9353}
9354
9355SDValue PPCTargetLowering::LowerFunnelShift(SDValue Op,
9356 SelectionDAG &DAG) const {
9357 SDLoc dl(Op);
9358 EVT VT = Op.getValueType();
9359 unsigned BitWidth = VT.getSizeInBits();
9360
9361 bool IsFSHL = Op.getOpcode() == ISD::FSHL;
9362 SDValue X = Op.getOperand(i: 0);
9363 SDValue Y = Op.getOperand(i: 1);
9364 SDValue Z = Op.getOperand(i: 2);
9365 EVT AmtVT = Z.getValueType();
9366
9367 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
9368 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
9369 // This is simpler than TargetLowering::expandFunnelShift because we can rely
9370 // on PowerPC shift by BW being well defined.
9371 Z = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Z,
9372 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
9373 SDValue SubZ =
9374 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT, N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Z);
9375 X = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: X, N2: IsFSHL ? Z : SubZ);
9376 Y = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Y, N2: IsFSHL ? SubZ : Z);
9377 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: X, N2: Y);
9378}
9379
9380//===----------------------------------------------------------------------===//
9381// Vector related lowering.
9382//
9383
9384/// getCanonicalConstSplat - Build a canonical splat immediate of Val with an
9385/// element size of SplatSize. Cast the result to VT.
9386static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT,
9387 SelectionDAG &DAG, const SDLoc &dl) {
9388 static const MVT VTys[] = { // canonical VT to use for each size.
9389 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32
9390 };
9391
9392 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1];
9393
9394 // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize.
9395 if (Val == ((1LLU << (SplatSize * 8)) - 1)) {
9396 SplatSize = 1;
9397 Val = 0xFF;
9398 }
9399
9400 EVT CanonicalVT = VTys[SplatSize-1];
9401
9402 // Build a canonical splat for this value.
9403 // Explicitly truncate APInt here, as this API is used with a mix of
9404 // signed and unsigned values.
9405 return DAG.getBitcast(
9406 VT: ReqVT,
9407 V: DAG.getConstant(Val: APInt(64, Val).trunc(width: SplatSize * 8), DL: dl, VT: CanonicalVT));
9408}
9409
9410/// BuildIntrinsicOp - Return a unary operator intrinsic node with the
9411/// specified intrinsic ID.
9412static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG,
9413 const SDLoc &dl, EVT DestVT = MVT::Other) {
9414 if (DestVT == MVT::Other) DestVT = Op.getValueType();
9415 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9416 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op);
9417}
9418
9419/// BuildIntrinsicOp - Return a binary operator intrinsic node with the
9420/// specified intrinsic ID.
9421static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
9422 SelectionDAG &DAG, const SDLoc &dl,
9423 EVT DestVT = MVT::Other) {
9424 if (DestVT == MVT::Other) DestVT = LHS.getValueType();
9425 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9426 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: LHS, N3: RHS);
9427}
9428
9429/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the
9430/// specified intrinsic ID.
9431static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
9432 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl,
9433 EVT DestVT = MVT::Other) {
9434 if (DestVT == MVT::Other) DestVT = Op0.getValueType();
9435 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9436 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op0, N3: Op1, N4: Op2);
9437}
9438
9439/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified
9440/// amount. The result has the specified value type.
9441static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
9442 SelectionDAG &DAG, const SDLoc &dl) {
9443 // Force LHS/RHS to be the right type.
9444 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: LHS);
9445 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: RHS);
9446
9447 int Ops[16];
9448 for (unsigned i = 0; i != 16; ++i)
9449 Ops[i] = i + Amt;
9450 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: LHS, N2: RHS, Mask: Ops);
9451 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
9452}
9453
9454/// Do we have an efficient pattern in a .td file for this node?
9455///
9456/// \param V - pointer to the BuildVectorSDNode being matched
9457/// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves?
9458///
9459/// There are some patterns where it is beneficial to keep a BUILD_VECTOR
9460/// node as a BUILD_VECTOR node rather than expanding it. The patterns where
9461/// the opposite is true (expansion is beneficial) are:
9462/// - The node builds a vector out of integers that are not 32 or 64-bits
9463/// - The node builds a vector out of constants
9464/// - The node is a "load-and-splat"
9465/// In all other cases, we will choose to keep the BUILD_VECTOR.
9466static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V,
9467 bool HasDirectMove,
9468 bool HasP8Vector) {
9469 EVT VecVT = V->getValueType(ResNo: 0);
9470 bool RightType = VecVT == MVT::v2f64 ||
9471 (HasP8Vector && VecVT == MVT::v4f32) ||
9472 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32));
9473 if (!RightType)
9474 return false;
9475
9476 bool IsSplat = true;
9477 bool IsLoad = false;
9478 SDValue Op0 = V->getOperand(Num: 0);
9479
9480 // This function is called in a block that confirms the node is not a constant
9481 // splat. So a constant BUILD_VECTOR here means the vector is built out of
9482 // different constants.
9483 if (V->isConstant())
9484 return false;
9485 for (int i = 0, e = V->getNumOperands(); i < e; ++i) {
9486 if (V->getOperand(Num: i).isUndef())
9487 return false;
9488 // We want to expand nodes that represent load-and-splat even if the
9489 // loaded value is a floating point truncation or conversion to int.
9490 if (V->getOperand(Num: i).getOpcode() == ISD::LOAD ||
9491 (V->getOperand(Num: i).getOpcode() == ISD::FP_ROUND &&
9492 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9493 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_SINT &&
9494 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9495 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_UINT &&
9496 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD))
9497 IsLoad = true;
9498 // If the operands are different or the input is not a load and has more
9499 // uses than just this BV node, then it isn't a splat.
9500 if (V->getOperand(Num: i) != Op0 ||
9501 (!IsLoad && !V->isOnlyUserOf(N: V->getOperand(Num: i).getNode())))
9502 IsSplat = false;
9503 }
9504 return !(IsSplat && IsLoad);
9505}
9506
9507// Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128.
9508SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
9509
9510 SDLoc dl(Op);
9511 SDValue Op0 = Op->getOperand(Num: 0);
9512
9513 if (!Subtarget.isPPC64() || (Op0.getOpcode() != ISD::BUILD_PAIR) ||
9514 (Op.getValueType() != MVT::f128))
9515 return SDValue();
9516
9517 SDValue Lo = Op0.getOperand(i: 0);
9518 SDValue Hi = Op0.getOperand(i: 1);
9519 if ((Lo.getValueType() != MVT::i64) || (Hi.getValueType() != MVT::i64))
9520 return SDValue();
9521
9522 if (!Subtarget.isLittleEndian())
9523 std::swap(a&: Lo, b&: Hi);
9524
9525 return DAG.getNode(Opcode: PPCISD::BUILD_FP128, DL: dl, VT: MVT::f128, N1: Lo, N2: Hi);
9526}
9527
9528static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) {
9529 const SDValue *InputLoad = &Op;
9530 while (InputLoad->getOpcode() == ISD::BITCAST)
9531 InputLoad = &InputLoad->getOperand(i: 0);
9532 if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR ||
9533 InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) {
9534 IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED;
9535 InputLoad = &InputLoad->getOperand(i: 0);
9536 }
9537 if (InputLoad->getOpcode() != ISD::LOAD)
9538 return nullptr;
9539 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9540 return ISD::isNormalLoad(N: LD) ? InputLoad : nullptr;
9541}
9542
9543// Convert the argument APFloat to a single precision APFloat if there is no
9544// loss in information during the conversion to single precision APFloat and the
9545// resulting number is not a denormal number. Return true if successful.
9546bool llvm::convertToNonDenormSingle(APFloat &ArgAPFloat) {
9547 APFloat APFloatToConvert = ArgAPFloat;
9548 bool LosesInfo = true;
9549 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9550 losesInfo: &LosesInfo);
9551 bool Success = (!LosesInfo && !APFloatToConvert.isDenormal());
9552 if (Success)
9553 ArgAPFloat = APFloatToConvert;
9554 return Success;
9555}
9556
9557// Bitcast the argument APInt to a double and convert it to a single precision
9558// APFloat, bitcast the APFloat to an APInt and assign it to the original
9559// argument if there is no loss in information during the conversion from
9560// double to single precision APFloat and the resulting number is not a denormal
9561// number. Return true if successful.
9562bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) {
9563 double DpValue = ArgAPInt.bitsToDouble();
9564 APFloat APFloatDp(DpValue);
9565 bool Success = convertToNonDenormSingle(ArgAPFloat&: APFloatDp);
9566 if (Success)
9567 ArgAPInt = APFloatDp.bitcastToAPInt();
9568 return Success;
9569}
9570
9571// Nondestructive check for convertTonNonDenormSingle.
9572bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) {
9573 // Only convert if it loses info, since XXSPLTIDP should
9574 // handle the other case.
9575 APFloat APFloatToConvert = ArgAPFloat;
9576 bool LosesInfo = true;
9577 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9578 losesInfo: &LosesInfo);
9579
9580 return (!LosesInfo && !APFloatToConvert.isDenormal());
9581}
9582
9583static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op,
9584 unsigned &Opcode) {
9585 LoadSDNode *InputNode = dyn_cast<LoadSDNode>(Val: Op.getOperand(i: 0));
9586 if (!InputNode || !Subtarget.hasVSX() || !ISD::isUNINDEXEDLoad(N: InputNode))
9587 return false;
9588
9589 EVT Ty = Op->getValueType(ResNo: 0);
9590 // For v2f64, v4f32 and v4i32 types, we require the load to be non-extending
9591 // as we cannot handle extending loads for these types.
9592 if ((Ty == MVT::v2f64 || Ty == MVT::v4f32 || Ty == MVT::v4i32) &&
9593 ISD::isNON_EXTLoad(N: InputNode))
9594 return true;
9595
9596 EVT MemVT = InputNode->getMemoryVT();
9597 // For v8i16 and v16i8 types, extending loads can be handled as long as the
9598 // memory VT is the same vector element VT type.
9599 // The loads feeding into the v8i16 and v16i8 types will be extending because
9600 // scalar i8/i16 are not legal types.
9601 if ((Ty == MVT::v8i16 || Ty == MVT::v16i8) && ISD::isEXTLoad(N: InputNode) &&
9602 (MemVT == Ty.getVectorElementType()))
9603 return true;
9604
9605 if (Ty == MVT::v2i64) {
9606 // Check the extend type, when the input type is i32, and the output vector
9607 // type is v2i64.
9608 if (MemVT == MVT::i32) {
9609 if (ISD::isZEXTLoad(N: InputNode))
9610 Opcode = PPCISD::ZEXT_LD_SPLAT;
9611 if (ISD::isSEXTLoad(N: InputNode))
9612 Opcode = PPCISD::SEXT_LD_SPLAT;
9613 }
9614 return true;
9615 }
9616 return false;
9617}
9618
9619bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN,
9620 bool IsLittleEndian) {
9621 assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector");
9622
9623 BitMask.clearAllBits();
9624 EVT VT = BVN.getValueType(ResNo: 0);
9625 unsigned VTSize = VT.getSizeInBits();
9626 APInt ConstValue(VTSize, 0);
9627
9628 unsigned EltWidth = VT.getScalarSizeInBits();
9629
9630 unsigned BitPos = 0;
9631 for (auto OpVal : BVN.op_values()) {
9632 auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal);
9633
9634 if (!CN)
9635 return false;
9636 // The elements in a vector register are ordered in reverse byte order
9637 // between little-endian and big-endian modes.
9638 ConstValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth),
9639 bitPosition: IsLittleEndian ? BitPos : VTSize - EltWidth - BitPos);
9640 BitPos += EltWidth;
9641 }
9642
9643 for (unsigned J = 0; J < 16; ++J) {
9644 APInt ExtractValue = ConstValue.extractBits(numBits: 8, bitPosition: J * 8);
9645 if (ExtractValue != 0x00 && ExtractValue != 0xFF)
9646 return false;
9647 if (ExtractValue == 0xFF)
9648 BitMask.setBit(J);
9649 }
9650 return true;
9651}
9652
9653// If this is a case we can't handle, return null and let the default
9654// expansion code take care of it. If we CAN select this case, and if it
9655// selects to a single instruction, return Op. Otherwise, if we can codegen
9656// this case more efficiently than a constant pool load, lower it to the
9657// sequence of ops that should be used.
9658SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
9659 SelectionDAG &DAG) const {
9660 SDLoc dl(Op);
9661 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: Op.getNode());
9662 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
9663
9664 if (Subtarget.hasP10Vector()) {
9665 APInt BitMask(32, 0);
9666 // If the value of the vector is all zeros or all ones,
9667 // we do not convert it to MTVSRBMI.
9668 // The xxleqv instruction sets a vector with all ones.
9669 // The xxlxor instruction sets a vector with all zeros.
9670 if (isValidMtVsrBmi(BitMask, BVN&: *BVN, IsLittleEndian: Subtarget.isLittleEndian()) &&
9671 BitMask != 0 && BitMask != 0xffff) {
9672 SDValue SDConstant = DAG.getTargetConstant(Val: BitMask, DL: dl, VT: MVT::i32);
9673 MachineSDNode *MSDNode =
9674 DAG.getMachineNode(Opcode: PPC::MTVSRBMI, dl, VT: MVT::v16i8, Op1: SDConstant);
9675 SDValue SDV = SDValue(MSDNode, 0);
9676 EVT DVT = BVN->getValueType(ResNo: 0);
9677 EVT SVT = SDV.getValueType();
9678 if (SVT != DVT) {
9679 SDV = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DVT, Operand: SDV);
9680 }
9681 return SDV;
9682 }
9683 // Recognize build vector patterns to emit VSX vector instructions
9684 // instead of loading value from memory.
9685 if (SDValue VecPat = combineBVLoadsSpecialValue(Operand: Op, DAG))
9686 return VecPat;
9687 }
9688 // Check if this is a splat of a constant value.
9689 APInt APSplatBits, APSplatUndef;
9690 unsigned SplatBitSize = 0;
9691 bool HasAnyUndefs;
9692 bool BVNIsConstantSplat =
9693 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
9694 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
9695
9696 // If it is a splat of a double, check if we can shrink it to a 32 bit
9697 // non-denormal float which when converted back to double gives us the same
9698 // double. This is to exploit the XXSPLTIDP instruction.
9699 // If we lose precision, we use XXSPLTI32DX.
9700 if (BVNIsConstantSplat && (SplatBitSize == 64) &&
9701 Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
9702 // Check the type first to short-circuit so we don't modify APSplatBits if
9703 // this block isn't executed.
9704 if ((Op->getValueType(ResNo: 0) == MVT::v2f64) &&
9705 convertToNonDenormSingle(ArgAPInt&: APSplatBits)) {
9706 SDValue SplatNode = DAG.getNode(
9707 Opcode: PPCISD::XXSPLTI_SP_TO_DP, DL: dl, VT: MVT::v2f64,
9708 Operand: DAG.getTargetConstant(Val: APSplatBits.getZExtValue(), DL: dl, VT: MVT::i32));
9709 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9710 } else {
9711 // We may lose precision, so we have to use XXSPLTI32DX.
9712
9713 uint32_t Hi = Hi_32(Value: APSplatBits.getZExtValue());
9714 uint32_t Lo = Lo_32(Value: APSplatBits.getZExtValue());
9715 SDValue SplatNode = DAG.getUNDEF(VT: MVT::v2i64);
9716
9717 if (!Hi || !Lo)
9718 // If either load is 0, then we should generate XXLXOR to set to 0.
9719 SplatNode = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::v2i64);
9720
9721 if (Hi)
9722 SplatNode = DAG.getNode(
9723 Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9724 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32),
9725 N3: DAG.getTargetConstant(Val: Hi, DL: dl, VT: MVT::i32));
9726
9727 if (Lo)
9728 SplatNode =
9729 DAG.getNode(Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9730 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i32),
9731 N3: DAG.getTargetConstant(Val: Lo, DL: dl, VT: MVT::i32));
9732
9733 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9734 }
9735 }
9736
9737 if (SDValue V =
9738 LowerVecSplatSmallFP(Op, DAG, BVNIsConstantSplat, SplatBitSize))
9739 return V;
9740
9741 bool IsSplat64 = false;
9742 uint64_t SplatBits = 0;
9743 int32_t SextVal = 0;
9744 if (BVNIsConstantSplat && SplatBitSize <= 64) {
9745 SplatBits = APSplatBits.getZExtValue();
9746 if (SplatBitSize <= 32) {
9747 SextVal = SignExtend32(X: SplatBits, B: SplatBitSize);
9748 } else if (SplatBitSize == 64 && Subtarget.hasP8Altivec()) {
9749 int64_t Splat64Val = static_cast<int64_t>(SplatBits);
9750 bool P9Vector = Subtarget.hasP9Vector();
9751 int32_t Hi = P9Vector ? 127 : 15;
9752 int32_t Lo = P9Vector ? -128 : -16;
9753 IsSplat64 = Splat64Val >= Lo && Splat64Val <= Hi;
9754 SextVal = static_cast<int32_t>(SplatBits);
9755 }
9756 }
9757
9758 if (!BVNIsConstantSplat || (SplatBitSize > 32 && !IsSplat64)) {
9759 unsigned NewOpcode = PPCISD::LD_SPLAT;
9760
9761 // Handle load-and-splat patterns as we have instructions that will do this
9762 // in one go.
9763 if (DAG.isSplatValue(V: Op, AllowUndefs: true) &&
9764 isValidSplatLoad(Subtarget, Op, Opcode&: NewOpcode)) {
9765 const SDValue *InputLoad = &Op.getOperand(i: 0);
9766 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9767
9768 // If the input load is an extending load, it will be an i32 -> i64
9769 // extending load and isValidSplatLoad() will update NewOpcode.
9770 unsigned MemorySize = LD->getMemoryVT().getScalarSizeInBits();
9771 unsigned ElementSize =
9772 MemorySize * ((NewOpcode == PPCISD::LD_SPLAT) ? 1 : 2);
9773
9774 assert(((ElementSize == 2 * MemorySize)
9775 ? (NewOpcode == PPCISD::ZEXT_LD_SPLAT ||
9776 NewOpcode == PPCISD::SEXT_LD_SPLAT)
9777 : (NewOpcode == PPCISD::LD_SPLAT)) &&
9778 "Unmatched element size and opcode!\n");
9779
9780 // Checking for a single use of this load, we have to check for vector
9781 // width (128 bits) / ElementSize uses (since each operand of the
9782 // BUILD_VECTOR is a separate use of the value.
9783 unsigned NumUsesOfInputLD = 128 / ElementSize;
9784 for (SDValue BVInOp : Op->ops())
9785 if (BVInOp.isUndef())
9786 NumUsesOfInputLD--;
9787
9788 // Exclude somes case where LD_SPLAT is worse than scalar_to_vector:
9789 // Below cases should also happen for "lfiwzx/lfiwax + LE target + index
9790 // 1" and "lxvrhx + BE target + index 7" and "lxvrbx + BE target + index
9791 // 15", but function IsValidSplatLoad() now will only return true when
9792 // the data at index 0 is not nullptr. So we will not get into trouble for
9793 // these cases.
9794 //
9795 // case 1 - lfiwzx/lfiwax
9796 // 1.1: load result is i32 and is sign/zero extend to i64;
9797 // 1.2: build a v2i64 vector type with above loaded value;
9798 // 1.3: the vector has only one value at index 0, others are all undef;
9799 // 1.4: on BE target, so that lfiwzx/lfiwax does not need any permute.
9800 if (NumUsesOfInputLD == 1 &&
9801 (Op->getValueType(ResNo: 0) == MVT::v2i64 && NewOpcode != PPCISD::LD_SPLAT &&
9802 !Subtarget.isLittleEndian() && Subtarget.hasVSX() &&
9803 Subtarget.hasLFIWAX()))
9804 return SDValue();
9805
9806 // case 2 - lxvr[hb]x
9807 // 2.1: load result is at most i16;
9808 // 2.2: build a vector with above loaded value;
9809 // 2.3: the vector has only one value at index 0, others are all undef;
9810 // 2.4: on LE target, so that lxvr[hb]x does not need any permute.
9811 if (NumUsesOfInputLD == 1 && Subtarget.isLittleEndian() &&
9812 Subtarget.isISA3_1() && ElementSize <= 16)
9813 return SDValue();
9814
9815 assert(NumUsesOfInputLD > 0 && "No uses of input LD of a build_vector?");
9816 if (InputLoad->getNode()->hasNUsesOfValue(NUses: NumUsesOfInputLD, Value: 0) &&
9817 Subtarget.hasVSX()) {
9818 SDValue Ops[] = {
9819 LD->getChain(), // Chain
9820 LD->getBasePtr(), // Ptr
9821 DAG.getValueType(Op.getValueType()) // VT
9822 };
9823 SDValue LdSplt = DAG.getMemIntrinsicNode(
9824 Opcode: NewOpcode, dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other), Ops,
9825 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
9826 // Replace all uses of the output chain of the original load with the
9827 // output chain of the new load.
9828 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1),
9829 To: LdSplt.getValue(R: 1));
9830 return LdSplt;
9831 }
9832 }
9833
9834 // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to
9835 // 32-bits can be lowered to VSX instructions under certain conditions.
9836 // Without VSX, there is no pattern more efficient than expanding the node.
9837 if (Subtarget.hasVSX() && Subtarget.isPPC64() &&
9838 haveEfficientBuildVectorPattern(V: BVN, HasDirectMove: Subtarget.hasDirectMove(),
9839 HasP8Vector: Subtarget.hasP8Vector()))
9840 return Op;
9841 return SDValue();
9842 }
9843
9844 uint64_t SplatUndef = APSplatUndef.getZExtValue();
9845 unsigned SplatSize = SplatBitSize / 8;
9846
9847 // First, handle single instruction cases.
9848
9849 // All zeros?
9850 if (SplatBits == 0) {
9851 // Canonicalize all zero vectors to be v4i32.
9852 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) {
9853 SDValue Z = DAG.getConstant(Val: 0, DL: dl, VT: MVT::v4i32);
9854 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Z);
9855 }
9856 return Op;
9857 }
9858
9859 // We have XXSPLTIW for constant splats four bytes wide.
9860 // Given vector length is a multiple of 4, 2-byte splats can be replaced
9861 // with 4-byte splats. We replicate the SplatBits in case of 2-byte splat to
9862 // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be
9863 // turned into a 4-byte splat of 0xABABABAB.
9864 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 2)
9865 return getCanonicalConstSplat(Val: SplatBits | (SplatBits << 16), SplatSize: SplatSize * 2,
9866 VT: Op.getValueType(), DAG, dl);
9867
9868 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 4)
9869 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9870 dl);
9871
9872 // We have XXSPLTIB for constant splats one byte wide.
9873 if (Subtarget.hasP9Vector() && SplatSize == 1)
9874 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9875 dl);
9876
9877 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
9878 // Use VSPLTIW/VUPKLSW for v2i64 in range [-16,15].
9879 if (SextVal >= -16 && SextVal <= 15) {
9880 // SplatSize may be 1, 2, 4, or 8. Use size 4 instead of 8 for the splat to
9881 // generate a splat word with extend for size 8.
9882 unsigned UseSize = SplatSize == 8 ? 4 : SplatSize;
9883 SDValue Res =
9884 getCanonicalConstSplat(Val: SextVal, SplatSize: UseSize, VT: Op.getValueType(), DAG, dl);
9885 if (SplatSize != 8)
9886 return Res;
9887 SDValue IntrinsicOp =
9888 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vupklsw,
9889 Op: DAG.getBitcast(VT: MVT::v4i32, V: Res), DAG, dl, DestVT: MVT::v2i64);
9890 return DAG.getBitcast(VT: Op.getValueType(), V: IntrinsicOp);
9891 }
9892
9893 // Two instruction sequences.
9894
9895 if (Subtarget.hasP9Vector() && SextVal >= -128 && SextVal <= 127) {
9896 SDValue C = DAG.getConstant(Val: (unsigned char)SextVal, DL: dl, VT: MVT::i32);
9897 SmallVector<SDValue, 16> Ops(16, C);
9898 SDValue BV = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops);
9899 unsigned IID;
9900 EVT VT;
9901 switch (SplatSize) {
9902 default:
9903 llvm_unreachable("Unexpected type for vector constant.");
9904 case 2:
9905 IID = Intrinsic::ppc_altivec_vupklsb;
9906 VT = MVT::v8i16;
9907 break;
9908 case 4:
9909 IID = Intrinsic::ppc_altivec_vextsb2w;
9910 VT = MVT::v4i32;
9911 break;
9912 case 8:
9913 IID = Intrinsic::ppc_altivec_vextsb2d;
9914 VT = MVT::v2i64;
9915 break;
9916 }
9917 SDValue Extend = BuildIntrinsicOp(IID, Op: BV, DAG, dl, DestVT: VT);
9918 return DAG.getBitcast(VT: Op->getValueType(ResNo: 0), V: Extend);
9919 }
9920 assert(!IsSplat64 && "Unhandled 64-bit splat pattern");
9921
9922 // If this value is in the range [-32,30] and is even, use:
9923 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2)
9924 // If this value is in the range [17,31] and is odd, use:
9925 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16)
9926 // If this value is in the range [-31,-17] and is odd, use:
9927 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16)
9928 // Note the last two are three-instruction sequences.
9929 if (SextVal >= -32 && SextVal <= 31) {
9930 // To avoid having these optimizations undone by constant folding,
9931 // we convert to a pseudo that will be expanded later into one of
9932 // the above forms.
9933 SDValue Elt = DAG.getSignedConstant(Val: SextVal, DL: dl, VT: MVT::i32);
9934 EVT VT = (SplatSize == 1 ? MVT::v16i8 :
9935 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32));
9936 SDValue EltSize = DAG.getConstant(Val: SplatSize, DL: dl, VT: MVT::i32);
9937 SDValue RetVal = DAG.getNode(Opcode: PPCISD::VADD_SPLAT, DL: dl, VT, N1: Elt, N2: EltSize);
9938 if (VT == Op.getValueType())
9939 return RetVal;
9940 else
9941 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: RetVal);
9942 }
9943
9944 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is
9945 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important
9946 // for fneg/fabs.
9947 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) {
9948 // Make -1 and vspltisw -1:
9949 SDValue OnesV = getCanonicalConstSplat(Val: -1, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
9950
9951 // Make the VSLW intrinsic, computing 0x8000_0000.
9952 SDValue Res = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: OnesV,
9953 RHS: OnesV, DAG, dl);
9954
9955 // xor by OnesV to invert it.
9956 Res = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::v4i32, N1: Res, N2: OnesV);
9957 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9958 }
9959
9960 // Check to see if this is a wide variety of vsplti*, binop self cases.
9961 static const signed char SplatCsts[] = {
9962 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7,
9963 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16
9964 };
9965
9966 for (unsigned idx = 0; idx < std::size(SplatCsts); ++idx) {
9967 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for
9968 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1'
9969 int i = SplatCsts[idx];
9970
9971 // Figure out what shift amount will be used by altivec if shifted by i in
9972 // this splat size.
9973 unsigned TypeShiftAmt = i & (SplatBitSize-1);
9974
9975 // vsplti + shl self.
9976 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
9977 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9978 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9979 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
9980 Intrinsic::ppc_altivec_vslw
9981 };
9982 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9983 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9984 }
9985
9986 // vsplti + srl self.
9987 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
9988 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9989 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9990 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0,
9991 Intrinsic::ppc_altivec_vsrw
9992 };
9993 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9994 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9995 }
9996
9997 // vsplti + rol self.
9998 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) |
9999 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) {
10000 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
10001 static const unsigned IIDs[] = { // Intrinsic to use for each size.
10002 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0,
10003 Intrinsic::ppc_altivec_vrlw
10004 };
10005 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
10006 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
10007 }
10008
10009 // t = vsplti c, result = vsldoi t, t, 1
10010 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
10011 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10012 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1;
10013 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10014 }
10015 // t = vsplti c, result = vsldoi t, t, 2
10016 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
10017 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10018 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2;
10019 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10020 }
10021 // t = vsplti c, result = vsldoi t, t, 3
10022 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
10023 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10024 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3;
10025 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10026 }
10027 }
10028
10029 return SDValue();
10030}
10031
10032/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
10033/// the specified operations to build the shuffle.
10034static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
10035 SDValue RHS, SelectionDAG &DAG,
10036 const SDLoc &dl) {
10037 unsigned OpNum = (PFEntry >> 26) & 0x0F;
10038 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
10039 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
10040
10041 enum {
10042 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
10043 OP_VMRGHW,
10044 OP_VMRGLW,
10045 OP_VSPLTISW0,
10046 OP_VSPLTISW1,
10047 OP_VSPLTISW2,
10048 OP_VSPLTISW3,
10049 OP_VSLDOI4,
10050 OP_VSLDOI8,
10051 OP_VSLDOI12
10052 };
10053
10054 if (OpNum == OP_COPY) {
10055 if (LHSID == (1*9+2)*9+3) return LHS;
10056 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
10057 return RHS;
10058 }
10059
10060 SDValue OpLHS, OpRHS;
10061 OpLHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
10062 OpRHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
10063
10064 int ShufIdxs[16];
10065 switch (OpNum) {
10066 default: llvm_unreachable("Unknown i32 permute!");
10067 case OP_VMRGHW:
10068 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3;
10069 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19;
10070 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7;
10071 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23;
10072 break;
10073 case OP_VMRGLW:
10074 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11;
10075 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27;
10076 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15;
10077 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31;
10078 break;
10079 case OP_VSPLTISW0:
10080 for (unsigned i = 0; i != 16; ++i)
10081 ShufIdxs[i] = (i&3)+0;
10082 break;
10083 case OP_VSPLTISW1:
10084 for (unsigned i = 0; i != 16; ++i)
10085 ShufIdxs[i] = (i&3)+4;
10086 break;
10087 case OP_VSPLTISW2:
10088 for (unsigned i = 0; i != 16; ++i)
10089 ShufIdxs[i] = (i&3)+8;
10090 break;
10091 case OP_VSPLTISW3:
10092 for (unsigned i = 0; i != 16; ++i)
10093 ShufIdxs[i] = (i&3)+12;
10094 break;
10095 case OP_VSLDOI4:
10096 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 4, VT: OpLHS.getValueType(), DAG, dl);
10097 case OP_VSLDOI8:
10098 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 8, VT: OpLHS.getValueType(), DAG, dl);
10099 case OP_VSLDOI12:
10100 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 12, VT: OpLHS.getValueType(), DAG, dl);
10101 }
10102 EVT VT = OpLHS.getValueType();
10103 OpLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpLHS);
10104 OpRHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpRHS);
10105 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OpLHS, N2: OpRHS, Mask: ShufIdxs);
10106 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
10107}
10108
10109/// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled
10110/// by the VINSERTB instruction introduced in ISA 3.0, else just return default
10111/// SDValue.
10112SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N,
10113 SelectionDAG &DAG) const {
10114 const unsigned BytesInVector = 16;
10115 bool IsLE = Subtarget.isLittleEndian();
10116 SDLoc dl(N);
10117 SDValue V1 = N->getOperand(Num: 0);
10118 SDValue V2 = N->getOperand(Num: 1);
10119 unsigned ShiftElts = 0, InsertAtByte = 0;
10120 bool Swap = false;
10121
10122 // Shifts required to get the byte we want at element 7.
10123 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1,
10124 0, 15, 14, 13, 12, 11, 10, 9};
10125 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0,
10126 1, 2, 3, 4, 5, 6, 7, 8};
10127
10128 ArrayRef<int> Mask = N->getMask();
10129 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
10130
10131 // For each mask element, find out if we're just inserting something
10132 // from V2 into V1 or vice versa.
10133 // Possible permutations inserting an element from V2 into V1:
10134 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10135 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10136 // ...
10137 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X
10138 // Inserting from V1 into V2 will be similar, except mask range will be
10139 // [16,31].
10140
10141 bool FoundCandidate = false;
10142 // If both vector operands for the shuffle are the same vector, the mask
10143 // will contain only elements from the first one and the second one will be
10144 // undef.
10145 unsigned VINSERTBSrcElem = IsLE ? 8 : 7;
10146 // Go through the mask of half-words to find an element that's being moved
10147 // from one vector to the other.
10148 for (unsigned i = 0; i < BytesInVector; ++i) {
10149 unsigned CurrentElement = Mask[i];
10150 // If 2nd operand is undefined, we should only look for element 7 in the
10151 // Mask.
10152 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem)
10153 continue;
10154
10155 bool OtherElementsInOrder = true;
10156 // Examine the other elements in the Mask to see if they're in original
10157 // order.
10158 for (unsigned j = 0; j < BytesInVector; ++j) {
10159 if (j == i)
10160 continue;
10161 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be
10162 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined,
10163 // in which we always assume we're always picking from the 1st operand.
10164 int MaskOffset =
10165 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0;
10166 if (Mask[j] != OriginalOrder[j] + MaskOffset) {
10167 OtherElementsInOrder = false;
10168 break;
10169 }
10170 }
10171 // If other elements are in original order, we record the number of shifts
10172 // we need to get the element we want into element 7. Also record which byte
10173 // in the vector we should insert into.
10174 if (OtherElementsInOrder) {
10175 // If 2nd operand is undefined, we assume no shifts and no swapping.
10176 if (V2.isUndef()) {
10177 ShiftElts = 0;
10178 Swap = false;
10179 } else {
10180 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4.
10181 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF]
10182 : BigEndianShifts[CurrentElement & 0xF];
10183 Swap = CurrentElement < BytesInVector;
10184 }
10185 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i;
10186 FoundCandidate = true;
10187 break;
10188 }
10189 }
10190
10191 if (!FoundCandidate)
10192 return SDValue();
10193
10194 // Candidate found, construct the proper SDAG sequence with VINSERTB,
10195 // optionally with VECSHL if shift is required.
10196 if (Swap)
10197 std::swap(a&: V1, b&: V2);
10198 if (V2.isUndef())
10199 V2 = V1;
10200 if (ShiftElts) {
10201 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10202 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10203 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: Shl,
10204 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10205 }
10206 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: V2,
10207 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10208}
10209
10210/// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled
10211/// by the VINSERTH instruction introduced in ISA 3.0, else just return default
10212/// SDValue.
10213SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N,
10214 SelectionDAG &DAG) const {
10215 const unsigned NumHalfWords = 8;
10216 const unsigned BytesInVector = NumHalfWords * 2;
10217 // Check that the shuffle is on half-words.
10218 if (!isNByteElemShuffleMask(N, Width: 2, StepLen: 1))
10219 return SDValue();
10220
10221 bool IsLE = Subtarget.isLittleEndian();
10222 SDLoc dl(N);
10223 SDValue V1 = N->getOperand(Num: 0);
10224 SDValue V2 = N->getOperand(Num: 1);
10225 unsigned ShiftElts = 0, InsertAtByte = 0;
10226 bool Swap = false;
10227
10228 // Shifts required to get the half-word we want at element 3.
10229 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5};
10230 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4};
10231
10232 uint32_t Mask = 0;
10233 uint32_t OriginalOrderLow = 0x1234567;
10234 uint32_t OriginalOrderHigh = 0x89ABCDEF;
10235 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a
10236 // 32-bit space, only need 4-bit nibbles per element.
10237 for (unsigned i = 0; i < NumHalfWords; ++i) {
10238 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10239 Mask |= ((uint32_t)(N->getMaskElt(Idx: i * 2) / 2) << MaskShift);
10240 }
10241
10242 // For each mask element, find out if we're just inserting something
10243 // from V2 into V1 or vice versa. Possible permutations inserting an element
10244 // from V2 into V1:
10245 // X, 1, 2, 3, 4, 5, 6, 7
10246 // 0, X, 2, 3, 4, 5, 6, 7
10247 // 0, 1, X, 3, 4, 5, 6, 7
10248 // 0, 1, 2, X, 4, 5, 6, 7
10249 // 0, 1, 2, 3, X, 5, 6, 7
10250 // 0, 1, 2, 3, 4, X, 6, 7
10251 // 0, 1, 2, 3, 4, 5, X, 7
10252 // 0, 1, 2, 3, 4, 5, 6, X
10253 // Inserting from V1 into V2 will be similar, except mask range will be [8,15].
10254
10255 bool FoundCandidate = false;
10256 // Go through the mask of half-words to find an element that's being moved
10257 // from one vector to the other.
10258 for (unsigned i = 0; i < NumHalfWords; ++i) {
10259 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10260 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF;
10261 uint32_t MaskOtherElts = ~(0xF << MaskShift);
10262 uint32_t TargetOrder = 0x0;
10263
10264 // If both vector operands for the shuffle are the same vector, the mask
10265 // will contain only elements from the first one and the second one will be
10266 // undef.
10267 if (V2.isUndef()) {
10268 ShiftElts = 0;
10269 unsigned VINSERTHSrcElem = IsLE ? 4 : 3;
10270 TargetOrder = OriginalOrderLow;
10271 Swap = false;
10272 // Skip if not the correct element or mask of other elements don't equal
10273 // to our expected order.
10274 if (MaskOneElt == VINSERTHSrcElem &&
10275 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10276 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10277 FoundCandidate = true;
10278 break;
10279 }
10280 } else { // If both operands are defined.
10281 // Target order is [8,15] if the current mask is between [0,7].
10282 TargetOrder =
10283 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow;
10284 // Skip if mask of other elements don't equal our expected order.
10285 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10286 // We only need the last 3 bits for the number of shifts.
10287 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7]
10288 : BigEndianShifts[MaskOneElt & 0x7];
10289 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10290 Swap = MaskOneElt < NumHalfWords;
10291 FoundCandidate = true;
10292 break;
10293 }
10294 }
10295 }
10296
10297 if (!FoundCandidate)
10298 return SDValue();
10299
10300 // Candidate found, construct the proper SDAG sequence with VINSERTH,
10301 // optionally with VECSHL if shift is required.
10302 if (Swap)
10303 std::swap(a&: V1, b&: V2);
10304 if (V2.isUndef())
10305 V2 = V1;
10306 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10307 if (ShiftElts) {
10308 // Double ShiftElts because we're left shifting on v16i8 type.
10309 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10310 N3: DAG.getConstant(Val: 2 * ShiftElts, DL: dl, VT: MVT::i32));
10311 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: Shl);
10312 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10313 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10314 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10315 }
10316 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V2);
10317 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10318 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10319 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10320}
10321
10322/// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be
10323/// handled by the XXSPLTI32DX instruction introduced in ISA 3.1, otherwise
10324/// return the default SDValue.
10325SDValue PPCTargetLowering::lowerToXXSPLTI32DX(ShuffleVectorSDNode *SVN,
10326 SelectionDAG &DAG) const {
10327 // The LHS and RHS may be bitcasts to v16i8 as we canonicalize shuffles
10328 // to v16i8. Peek through the bitcasts to get the actual operands.
10329 SDValue LHS = peekThroughBitcasts(V: SVN->getOperand(Num: 0));
10330 SDValue RHS = peekThroughBitcasts(V: SVN->getOperand(Num: 1));
10331
10332 auto ShuffleMask = SVN->getMask();
10333 SDValue VecShuffle(SVN, 0);
10334 SDLoc DL(SVN);
10335
10336 // Check that we have a four byte shuffle.
10337 if (!isNByteElemShuffleMask(N: SVN, Width: 4, StepLen: 1))
10338 return SDValue();
10339
10340 // Canonicalize the RHS being a BUILD_VECTOR when lowering to xxsplti32dx.
10341 if (RHS->getOpcode() != ISD::BUILD_VECTOR) {
10342 std::swap(a&: LHS, b&: RHS);
10343 VecShuffle = peekThroughBitcasts(V: DAG.getCommutedVectorShuffle(SV: *SVN));
10344 ShuffleVectorSDNode *CommutedSV = dyn_cast<ShuffleVectorSDNode>(Val&: VecShuffle);
10345 if (!CommutedSV)
10346 return SDValue();
10347 ShuffleMask = CommutedSV->getMask();
10348 }
10349
10350 // Ensure that the RHS is a vector of constants.
10351 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: RHS.getNode());
10352 if (!BVN)
10353 return SDValue();
10354
10355 // Check if RHS is a splat of 4-bytes (or smaller).
10356 APInt APSplatValue, APSplatUndef;
10357 unsigned SplatBitSize;
10358 bool HasAnyUndefs;
10359 if (!BVN->isConstantSplat(SplatValue&: APSplatValue, SplatUndef&: APSplatUndef, SplatBitSize,
10360 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian()) ||
10361 SplatBitSize > 32)
10362 return SDValue();
10363
10364 // Check that the shuffle mask matches the semantics of XXSPLTI32DX.
10365 // The instruction splats a constant C into two words of the source vector
10366 // producing { C, Unchanged, C, Unchanged } or { Unchanged, C, Unchanged, C }.
10367 // Thus we check that the shuffle mask is the equivalent of
10368 // <0, [4-7], 2, [4-7]> or <[4-7], 1, [4-7], 3> respectively.
10369 // Note: the check above of isNByteElemShuffleMask() ensures that the bytes
10370 // within each word are consecutive, so we only need to check the first byte.
10371 SDValue Index;
10372 bool IsLE = Subtarget.isLittleEndian();
10373 if ((ShuffleMask[0] == 0 && ShuffleMask[8] == 8) &&
10374 (ShuffleMask[4] % 4 == 0 && ShuffleMask[12] % 4 == 0 &&
10375 ShuffleMask[4] > 15 && ShuffleMask[12] > 15))
10376 Index = DAG.getTargetConstant(Val: IsLE ? 0 : 1, DL, VT: MVT::i32);
10377 else if ((ShuffleMask[4] == 4 && ShuffleMask[12] == 12) &&
10378 (ShuffleMask[0] % 4 == 0 && ShuffleMask[8] % 4 == 0 &&
10379 ShuffleMask[0] > 15 && ShuffleMask[8] > 15))
10380 Index = DAG.getTargetConstant(Val: IsLE ? 1 : 0, DL, VT: MVT::i32);
10381 else
10382 return SDValue();
10383
10384 // If the splat is narrower than 32-bits, we need to get the 32-bit value
10385 // for XXSPLTI32DX.
10386 unsigned SplatVal = APSplatValue.getZExtValue();
10387 for (; SplatBitSize < 32; SplatBitSize <<= 1)
10388 SplatVal |= (SplatVal << SplatBitSize);
10389
10390 SDValue SplatNode = DAG.getNode(
10391 Opcode: PPCISD::XXSPLTI32DX, DL, VT: MVT::v2i64, N1: DAG.getBitcast(VT: MVT::v2i64, V: LHS),
10392 N2: Index, N3: DAG.getTargetConstant(Val: SplatVal, DL, VT: MVT::i32));
10393 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: SplatNode);
10394}
10395
10396/// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8).
10397/// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is
10398/// a multiple of 8. Otherwise convert it to a scalar rotation(i128)
10399/// i.e (or (shl x, C1), (srl x, 128-C1)).
10400SDValue PPCTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
10401 assert(Op.getOpcode() == ISD::ROTL && "Should only be called for ISD::ROTL");
10402 assert(Op.getValueType() == MVT::v1i128 &&
10403 "Only set v1i128 as custom, other type shouldn't reach here!");
10404 SDLoc dl(Op);
10405 SDValue N0 = peekThroughBitcasts(V: Op.getOperand(i: 0));
10406 SDValue N1 = peekThroughBitcasts(V: Op.getOperand(i: 1));
10407 unsigned SHLAmt = N1.getConstantOperandVal(i: 0);
10408 if (SHLAmt % 8 == 0) {
10409 std::array<int, 16> Mask;
10410 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
10411 std::rotate(first: Mask.begin(), middle: Mask.begin() + SHLAmt / 8, last: Mask.end());
10412 if (SDValue Shuffle =
10413 DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: DAG.getBitcast(VT: MVT::v16i8, V: N0),
10414 N2: DAG.getUNDEF(VT: MVT::v16i8), Mask))
10415 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: Shuffle);
10416 }
10417 SDValue ArgVal = DAG.getBitcast(VT: MVT::i128, V: N0);
10418 SDValue SHLOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ArgVal,
10419 N2: DAG.getConstant(Val: SHLAmt, DL: dl, VT: MVT::i32));
10420 SDValue SRLOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: ArgVal,
10421 N2: DAG.getConstant(Val: 128 - SHLAmt, DL: dl, VT: MVT::i32));
10422 SDValue OROp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i128, N1: SHLOp, N2: SRLOp);
10423 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: OROp);
10424}
10425
10426/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this
10427/// is a shuffle we can handle in a single instruction, return it. Otherwise,
10428/// return the code it can be lowered into. Worst case, it can always be
10429/// lowered into a vperm.
10430SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
10431 SelectionDAG &DAG) const {
10432 SDLoc dl(Op);
10433 SDValue V1 = Op.getOperand(i: 0);
10434 SDValue V2 = Op.getOperand(i: 1);
10435 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10436
10437 // Any nodes that were combined in the target-independent combiner prior
10438 // to vector legalization will not be sent to the target combine. Try to
10439 // combine it here.
10440 if (SDValue NewShuffle = combineVectorShuffle(SVN: SVOp, DAG)) {
10441 if (!isa<ShuffleVectorSDNode>(Val: NewShuffle))
10442 return NewShuffle;
10443 Op = NewShuffle;
10444 SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10445 V1 = Op.getOperand(i: 0);
10446 V2 = Op.getOperand(i: 1);
10447 }
10448 EVT VT = Op.getValueType();
10449 bool isLittleEndian = Subtarget.isLittleEndian();
10450
10451 unsigned ShiftElts, InsertAtByte;
10452 bool Swap = false;
10453
10454 // If this is a load-and-splat, we can do that with a single instruction
10455 // in some cases. However if the load has multiple uses, we don't want to
10456 // combine it because that will just produce multiple loads.
10457 bool IsPermutedLoad = false;
10458 const SDValue *InputLoad = getNormalLoadInput(Op: V1, IsPermuted&: IsPermutedLoad);
10459 if (InputLoad && Subtarget.hasVSX() && V2.isUndef() &&
10460 (PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) || PPC::isSplatShuffleMask(N: SVOp, EltSize: 8)) &&
10461 InputLoad->hasOneUse()) {
10462 bool IsFourByte = PPC::isSplatShuffleMask(N: SVOp, EltSize: 4);
10463 int SplatIdx =
10464 PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: IsFourByte ? 4 : 8, DAG);
10465
10466 // The splat index for permuted loads will be in the left half of the vector
10467 // which is strictly wider than the loaded value by 8 bytes. So we need to
10468 // adjust the splat index to point to the correct address in memory.
10469 if (IsPermutedLoad) {
10470 assert((isLittleEndian || IsFourByte) &&
10471 "Unexpected size for permuted load on big endian target");
10472 SplatIdx += IsFourByte ? 2 : 1;
10473 assert((SplatIdx < (IsFourByte ? 4 : 2)) &&
10474 "Splat of a value outside of the loaded memory");
10475 }
10476
10477 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
10478 // For 4-byte load-and-splat, we need Power9.
10479 if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) {
10480 uint64_t Offset = 0;
10481 if (IsFourByte)
10482 Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4;
10483 else
10484 Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8;
10485
10486 // If the width of the load is the same as the width of the splat,
10487 // loading with an offset would load the wrong memory.
10488 if (LD->getValueType(ResNo: 0).getSizeInBits() == (IsFourByte ? 32 : 64))
10489 Offset = 0;
10490
10491 SDValue BasePtr = LD->getBasePtr();
10492 if (Offset != 0)
10493 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
10494 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: Offset, DL: dl));
10495 SDValue Ops[] = {
10496 LD->getChain(), // Chain
10497 BasePtr, // BasePtr
10498 DAG.getValueType(Op.getValueType()) // VT
10499 };
10500 SDVTList VTL =
10501 DAG.getVTList(VT1: IsFourByte ? MVT::v4i32 : MVT::v2i64, VT2: MVT::Other);
10502 SDValue LdSplt =
10503 DAG.getMemIntrinsicNode(Opcode: PPCISD::LD_SPLAT, dl, VTList: VTL,
10504 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
10505 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1), To: LdSplt.getValue(R: 1));
10506 if (LdSplt.getValueType() != SVOp->getValueType(ResNo: 0))
10507 LdSplt = DAG.getBitcast(VT: SVOp->getValueType(ResNo: 0), V: LdSplt);
10508 return LdSplt;
10509 }
10510 }
10511
10512 // All v2i64 and v2f64 shuffles are legal
10513 if (VT == MVT::v2i64 || VT == MVT::v2f64)
10514 return Op;
10515
10516 if (Subtarget.hasP9Vector() &&
10517 PPC::isXXINSERTWMask(N: SVOp, ShiftElts, InsertAtByte, Swap,
10518 IsLE: isLittleEndian)) {
10519 if (V2.isUndef())
10520 V2 = V1;
10521 else if (Swap)
10522 std::swap(a&: V1, b&: V2);
10523 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10524 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2);
10525 if (ShiftElts) {
10526 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv2, N2: Conv2,
10527 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10528 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Shl,
10529 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10530 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10531 }
10532 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10533 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10534 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10535 }
10536
10537 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
10538 SDValue SplatInsertNode;
10539 if ((SplatInsertNode = lowerToXXSPLTI32DX(SVN: SVOp, DAG)))
10540 return SplatInsertNode;
10541 }
10542
10543 if (Subtarget.hasP9Altivec()) {
10544 SDValue NewISDNode;
10545 if ((NewISDNode = lowerToVINSERTH(N: SVOp, DAG)))
10546 return NewISDNode;
10547
10548 if ((NewISDNode = lowerToVINSERTB(N: SVOp, DAG)))
10549 return NewISDNode;
10550 }
10551
10552 if (Subtarget.hasVSX() &&
10553 PPC::isXXSLDWIShuffleMask(N: SVOp, ShiftElts, Swap, IsLE: isLittleEndian)) {
10554 if (Swap)
10555 std::swap(a&: V1, b&: V2);
10556 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10557 SDValue Conv2 =
10558 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2.isUndef() ? V1 : V2);
10559
10560 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10561 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10562 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Shl);
10563 }
10564
10565 if (Subtarget.hasVSX() &&
10566 PPC::isXXPERMDIShuffleMask(N: SVOp, DM&: ShiftElts, Swap, IsLE: isLittleEndian)) {
10567 if (Swap)
10568 std::swap(a&: V1, b&: V2);
10569 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10570 SDValue Conv2 =
10571 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V2.isUndef() ? V1 : V2);
10572
10573 SDValue PermDI = DAG.getNode(Opcode: PPCISD::XXPERMDI, DL: dl, VT: MVT::v2i64, N1: Conv1, N2: Conv2,
10574 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10575 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: PermDI);
10576 }
10577
10578 if (Subtarget.hasP9Vector()) {
10579 if (PPC::isXXBRHShuffleMask(N: SVOp)) {
10580 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10581 SDValue ReveHWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v8i16, Operand: Conv);
10582 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveHWord);
10583 } else if (PPC::isXXBRWShuffleMask(N: SVOp)) {
10584 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10585 SDValue ReveWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v4i32, Operand: Conv);
10586 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveWord);
10587 } else if (PPC::isXXBRDShuffleMask(N: SVOp)) {
10588 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10589 SDValue ReveDWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Conv);
10590 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveDWord);
10591 } else if (PPC::isXXBRQShuffleMask(N: SVOp)) {
10592 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: V1);
10593 SDValue ReveQWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v1i128, Operand: Conv);
10594 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveQWord);
10595 }
10596 }
10597
10598 if (Subtarget.hasVSX()) {
10599 if (V2.isUndef() && PPC::isSplatShuffleMask(N: SVOp, EltSize: 4)) {
10600 int SplatIdx = PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: 4, DAG);
10601
10602 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10603 SDValue Splat = DAG.getNode(Opcode: PPCISD::XXSPLT, DL: dl, VT: MVT::v4i32, N1: Conv,
10604 N2: DAG.getConstant(Val: SplatIdx, DL: dl, VT: MVT::i32));
10605 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Splat);
10606 }
10607
10608 // Left shifts of 8 bytes are actually swaps. Convert accordingly.
10609 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) == 8) {
10610 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: V1);
10611 SDValue Swap = DAG.getNode(Opcode: PPCISD::SWAP_NO_CHAIN, DL: dl, VT: MVT::v2f64, Operand: Conv);
10612 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Swap);
10613 }
10614 }
10615
10616 // Cases that are handled by instructions that take permute immediates
10617 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
10618 // selected by the instruction selector.
10619 if (V2.isUndef()) {
10620 if (PPC::isSplatShuffleMask(N: SVOp, EltSize: 1) ||
10621 PPC::isSplatShuffleMask(N: SVOp, EltSize: 2) ||
10622 PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) ||
10623 PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10624 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10625 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) != -1 ||
10626 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10627 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10628 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10629 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10630 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10631 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10632 (Subtarget.hasP8Altivec() && (
10633 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10634 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind: 1, DAG) ||
10635 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind: 1, DAG)))) {
10636 return Op;
10637 }
10638 }
10639
10640 // Altivec has a variety of "shuffle immediates" that take two vector inputs
10641 // and produce a fixed permutation. If any of these match, do not lower to
10642 // VPERM.
10643 unsigned int ShuffleKind = isLittleEndian ? 2 : 0;
10644 if (PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10645 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10646 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind, DAG) != -1 ||
10647 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10648 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10649 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10650 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10651 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10652 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10653 (Subtarget.hasP8Altivec() && (
10654 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10655 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind, DAG) ||
10656 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind, DAG))))
10657 return Op;
10658
10659 // Check to see if this is a shuffle of 4-byte values. If so, we can use our
10660 // perfect shuffle table to emit an optimal matching sequence.
10661 ArrayRef<int> PermMask = SVOp->getMask();
10662
10663 if (EnablePerfectShuffle && !isLittleEndian) {
10664 unsigned PFIndexes[4];
10665 bool isFourElementShuffle = true;
10666 for (unsigned i = 0; i != 4 && isFourElementShuffle;
10667 ++i) { // Element number
10668 unsigned EltNo = 8; // Start out undef.
10669 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte.
10670 if (PermMask[i * 4 + j] < 0)
10671 continue; // Undef, ignore it.
10672
10673 unsigned ByteSource = PermMask[i * 4 + j];
10674 if ((ByteSource & 3) != j) {
10675 isFourElementShuffle = false;
10676 break;
10677 }
10678
10679 if (EltNo == 8) {
10680 EltNo = ByteSource / 4;
10681 } else if (EltNo != ByteSource / 4) {
10682 isFourElementShuffle = false;
10683 break;
10684 }
10685 }
10686 PFIndexes[i] = EltNo;
10687 }
10688
10689 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the
10690 // perfect shuffle vector to determine if it is cost effective to do this as
10691 // discrete instructions, or whether we should use a vperm.
10692 // For now, we skip this for little endian until such time as we have a
10693 // little-endian perfect shuffle table.
10694 if (isFourElementShuffle) {
10695 // Compute the index in the perfect shuffle table.
10696 unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
10697 PFIndexes[2] * 9 + PFIndexes[3];
10698
10699 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
10700 unsigned Cost = (PFEntry >> 30);
10701
10702 // Determining when to avoid vperm is tricky. Many things affect the cost
10703 // of vperm, particularly how many times the perm mask needs to be
10704 // computed. For example, if the perm mask can be hoisted out of a loop or
10705 // is already used (perhaps because there are multiple permutes with the
10706 // same shuffle mask?) the vperm has a cost of 1. OTOH, hoisting the
10707 // permute mask out of the loop requires an extra register.
10708 //
10709 // As a compromise, we only emit discrete instructions if the shuffle can
10710 // be generated in 3 or fewer operations. When we have loop information
10711 // available, if this block is within a loop, we should avoid using vperm
10712 // for 3-operation perms and use a constant pool load instead.
10713 if (Cost < 3)
10714 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
10715 }
10716 }
10717
10718 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant
10719 // vector that will get spilled to the constant pool.
10720 if (V2.isUndef()) V2 = V1;
10721
10722 return LowerVPERM(Op, DAG, PermMask, VT, V1, V2);
10723}
10724
10725SDValue PPCTargetLowering::LowerVPERM(SDValue Op, SelectionDAG &DAG,
10726 ArrayRef<int> PermMask, EVT VT,
10727 SDValue V1, SDValue V2) const {
10728 unsigned Opcode = PPCISD::VPERM;
10729 EVT ValType = V1.getValueType();
10730 SDLoc dl(Op);
10731 bool NeedSwap = false;
10732 bool isLittleEndian = Subtarget.isLittleEndian();
10733 bool isPPC64 = Subtarget.isPPC64();
10734
10735 if (Subtarget.hasVSX() && Subtarget.hasP9Vector() &&
10736 (V1->hasOneUse() || V2->hasOneUse())) {
10737 LLVM_DEBUG(dbgs() << "At least one of two input vectors are dead - using "
10738 "XXPERM instead\n");
10739 Opcode = PPCISD::XXPERM;
10740
10741 // The second input to XXPERM is also an output so if the second input has
10742 // multiple uses then copying is necessary, as a result we want the
10743 // single-use operand to be used as the second input to prevent copying.
10744 if ((!isLittleEndian && !V2->hasOneUse() && V1->hasOneUse()) ||
10745 (isLittleEndian && !V1->hasOneUse() && V2->hasOneUse())) {
10746 std::swap(a&: V1, b&: V2);
10747 NeedSwap = !NeedSwap;
10748 }
10749 }
10750
10751 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except
10752 // that it is in input element units, not in bytes. Convert now.
10753
10754 // For little endian, the order of the input vectors is reversed, and
10755 // the permutation mask is complemented with respect to 31. This is
10756 // necessary to produce proper semantics with the big-endian-based vperm
10757 // instruction.
10758 EVT EltVT = V1.getValueType().getVectorElementType();
10759 unsigned BytesPerElement = EltVT.getSizeInBits() / 8;
10760
10761 bool V1HasXXSWAPD = V1->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10762 bool V2HasXXSWAPD = V2->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10763
10764 /*
10765 Vectors will be appended like so: [ V1 | v2 ]
10766 XXSWAPD on V1:
10767 [ A | B | C | D ] -> [ C | D | A | B ]
10768 0-3 4-7 8-11 12-15 0-3 4-7 8-11 12-15
10769 i.e. index of A, B += 8, and index of C, D -= 8.
10770 XXSWAPD on V2:
10771 [ E | F | G | H ] -> [ G | H | E | F ]
10772 16-19 20-23 24-27 28-31 16-19 20-23 24-27 28-31
10773 i.e. index of E, F += 8, index of G, H -= 8
10774 Swap V1 and V2:
10775 [ V1 | V2 ] -> [ V2 | V1 ]
10776 0-15 16-31 0-15 16-31
10777 i.e. index of V1 += 16, index of V2 -= 16
10778 */
10779
10780 SmallVector<SDValue, 16> ResultMask;
10781 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
10782 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i];
10783
10784 if (V1HasXXSWAPD) {
10785 if (SrcElt < 8)
10786 SrcElt += 8;
10787 else if (SrcElt < 16)
10788 SrcElt -= 8;
10789 }
10790 if (V2HasXXSWAPD) {
10791 if (SrcElt > 23)
10792 SrcElt -= 8;
10793 else if (SrcElt > 15)
10794 SrcElt += 8;
10795 }
10796 if (NeedSwap) {
10797 if (SrcElt < 16)
10798 SrcElt += 16;
10799 else
10800 SrcElt -= 16;
10801 }
10802 for (unsigned j = 0; j != BytesPerElement; ++j)
10803 if (isLittleEndian)
10804 ResultMask.push_back(
10805 Elt: DAG.getConstant(Val: 31 - (SrcElt * BytesPerElement + j), DL: dl, VT: MVT::i32));
10806 else
10807 ResultMask.push_back(
10808 Elt: DAG.getConstant(Val: SrcElt * BytesPerElement + j, DL: dl, VT: MVT::i32));
10809 }
10810
10811 if (V1HasXXSWAPD) {
10812 dl = SDLoc(V1->getOperand(Num: 0));
10813 V1 = V1->getOperand(Num: 0)->getOperand(Num: 1);
10814 }
10815 if (V2HasXXSWAPD) {
10816 dl = SDLoc(V2->getOperand(Num: 0));
10817 V2 = V2->getOperand(Num: 0)->getOperand(Num: 1);
10818 }
10819
10820 if (isPPC64 && (V1HasXXSWAPD || V2HasXXSWAPD)) {
10821 if (ValType != MVT::v2f64)
10822 V1 = DAG.getBitcast(VT: MVT::v2f64, V: V1);
10823 if (V2.getValueType() != MVT::v2f64)
10824 V2 = DAG.getBitcast(VT: MVT::v2f64, V: V2);
10825 }
10826
10827 ShufflesHandledWithVPERM++;
10828 SDValue VPermMask = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: ResultMask);
10829 LLVM_DEBUG({
10830 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10831 if (Opcode == PPCISD::XXPERM) {
10832 dbgs() << "Emitting a XXPERM for the following shuffle:\n";
10833 } else {
10834 dbgs() << "Emitting a VPERM for the following shuffle:\n";
10835 }
10836 SVOp->dump();
10837 dbgs() << "With the following permute control vector:\n";
10838 VPermMask.dump();
10839 });
10840
10841 if (Opcode == PPCISD::XXPERM)
10842 VPermMask = DAG.getBitcast(VT: MVT::v4i32, V: VPermMask);
10843
10844 // Only need to place items backwards in LE,
10845 // the mask was properly calculated.
10846 if (isLittleEndian)
10847 std::swap(a&: V1, b&: V2);
10848
10849 SDValue VPERMNode =
10850 DAG.getNode(Opcode, DL: dl, VT: V1.getValueType(), N1: V1, N2: V2, N3: VPermMask);
10851
10852 VPERMNode = DAG.getBitcast(VT: ValType, V: VPERMNode);
10853 return VPERMNode;
10854}
10855
10856/// getVectorCompareInfo - Given an intrinsic, return false if it is not a
10857/// vector comparison. If it is, return true and fill in Opc/isDot with
10858/// information about the intrinsic.
10859static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc,
10860 bool &isDot, const PPCSubtarget &Subtarget) {
10861 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 0);
10862 CompareOpc = -1;
10863 isDot = false;
10864 switch (IntrinsicID) {
10865 default:
10866 return false;
10867 // Comparison predicates.
10868 case Intrinsic::ppc_altivec_vcmpbfp_p:
10869 CompareOpc = 966;
10870 isDot = true;
10871 break;
10872 case Intrinsic::ppc_altivec_vcmpeqfp_p:
10873 CompareOpc = 198;
10874 isDot = true;
10875 break;
10876 case Intrinsic::ppc_altivec_vcmpequb_p:
10877 CompareOpc = 6;
10878 isDot = true;
10879 break;
10880 case Intrinsic::ppc_altivec_vcmpequh_p:
10881 CompareOpc = 70;
10882 isDot = true;
10883 break;
10884 case Intrinsic::ppc_altivec_vcmpequw_p:
10885 CompareOpc = 134;
10886 isDot = true;
10887 break;
10888 case Intrinsic::ppc_altivec_vcmpequd_p:
10889 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10890 CompareOpc = 199;
10891 isDot = true;
10892 } else
10893 return false;
10894 break;
10895 case Intrinsic::ppc_altivec_vcmpneb_p:
10896 case Intrinsic::ppc_altivec_vcmpneh_p:
10897 case Intrinsic::ppc_altivec_vcmpnew_p:
10898 case Intrinsic::ppc_altivec_vcmpnezb_p:
10899 case Intrinsic::ppc_altivec_vcmpnezh_p:
10900 case Intrinsic::ppc_altivec_vcmpnezw_p:
10901 if (Subtarget.hasP9Altivec()) {
10902 switch (IntrinsicID) {
10903 default:
10904 llvm_unreachable("Unknown comparison intrinsic.");
10905 case Intrinsic::ppc_altivec_vcmpneb_p:
10906 CompareOpc = 7;
10907 break;
10908 case Intrinsic::ppc_altivec_vcmpneh_p:
10909 CompareOpc = 71;
10910 break;
10911 case Intrinsic::ppc_altivec_vcmpnew_p:
10912 CompareOpc = 135;
10913 break;
10914 case Intrinsic::ppc_altivec_vcmpnezb_p:
10915 CompareOpc = 263;
10916 break;
10917 case Intrinsic::ppc_altivec_vcmpnezh_p:
10918 CompareOpc = 327;
10919 break;
10920 case Intrinsic::ppc_altivec_vcmpnezw_p:
10921 CompareOpc = 391;
10922 break;
10923 }
10924 isDot = true;
10925 } else
10926 return false;
10927 break;
10928 case Intrinsic::ppc_altivec_vcmpgefp_p:
10929 CompareOpc = 454;
10930 isDot = true;
10931 break;
10932 case Intrinsic::ppc_altivec_vcmpgtfp_p:
10933 CompareOpc = 710;
10934 isDot = true;
10935 break;
10936 case Intrinsic::ppc_altivec_vcmpgtsb_p:
10937 CompareOpc = 774;
10938 isDot = true;
10939 break;
10940 case Intrinsic::ppc_altivec_vcmpgtsh_p:
10941 CompareOpc = 838;
10942 isDot = true;
10943 break;
10944 case Intrinsic::ppc_altivec_vcmpgtsw_p:
10945 CompareOpc = 902;
10946 isDot = true;
10947 break;
10948 case Intrinsic::ppc_altivec_vcmpgtsd_p:
10949 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10950 CompareOpc = 967;
10951 isDot = true;
10952 } else
10953 return false;
10954 break;
10955 case Intrinsic::ppc_altivec_vcmpgtub_p:
10956 CompareOpc = 518;
10957 isDot = true;
10958 break;
10959 case Intrinsic::ppc_altivec_vcmpgtuh_p:
10960 CompareOpc = 582;
10961 isDot = true;
10962 break;
10963 case Intrinsic::ppc_altivec_vcmpgtuw_p:
10964 CompareOpc = 646;
10965 isDot = true;
10966 break;
10967 case Intrinsic::ppc_altivec_vcmpgtud_p:
10968 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10969 CompareOpc = 711;
10970 isDot = true;
10971 } else
10972 return false;
10973 break;
10974
10975 case Intrinsic::ppc_altivec_vcmpequq:
10976 case Intrinsic::ppc_altivec_vcmpgtsq:
10977 case Intrinsic::ppc_altivec_vcmpgtuq:
10978 if (!Subtarget.isISA3_1())
10979 return false;
10980 switch (IntrinsicID) {
10981 default:
10982 llvm_unreachable("Unknown comparison intrinsic.");
10983 case Intrinsic::ppc_altivec_vcmpequq:
10984 CompareOpc = 455;
10985 break;
10986 case Intrinsic::ppc_altivec_vcmpgtsq:
10987 CompareOpc = 903;
10988 break;
10989 case Intrinsic::ppc_altivec_vcmpgtuq:
10990 CompareOpc = 647;
10991 break;
10992 }
10993 break;
10994
10995 // VSX predicate comparisons use the same infrastructure
10996 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10997 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10998 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
10999 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
11000 case Intrinsic::ppc_vsx_xvcmpgesp_p:
11001 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
11002 if (Subtarget.hasVSX()) {
11003 switch (IntrinsicID) {
11004 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
11005 CompareOpc = 99;
11006 break;
11007 case Intrinsic::ppc_vsx_xvcmpgedp_p:
11008 CompareOpc = 115;
11009 break;
11010 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
11011 CompareOpc = 107;
11012 break;
11013 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
11014 CompareOpc = 67;
11015 break;
11016 case Intrinsic::ppc_vsx_xvcmpgesp_p:
11017 CompareOpc = 83;
11018 break;
11019 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
11020 CompareOpc = 75;
11021 break;
11022 }
11023 isDot = true;
11024 } else
11025 return false;
11026 break;
11027
11028 // Normal Comparisons.
11029 case Intrinsic::ppc_altivec_vcmpbfp:
11030 CompareOpc = 966;
11031 break;
11032 case Intrinsic::ppc_altivec_vcmpeqfp:
11033 CompareOpc = 198;
11034 break;
11035 case Intrinsic::ppc_altivec_vcmpequb:
11036 CompareOpc = 6;
11037 break;
11038 case Intrinsic::ppc_altivec_vcmpequh:
11039 CompareOpc = 70;
11040 break;
11041 case Intrinsic::ppc_altivec_vcmpequw:
11042 CompareOpc = 134;
11043 break;
11044 case Intrinsic::ppc_altivec_vcmpequd:
11045 if (Subtarget.hasP8Altivec())
11046 CompareOpc = 199;
11047 else
11048 return false;
11049 break;
11050 case Intrinsic::ppc_altivec_vcmpneb:
11051 case Intrinsic::ppc_altivec_vcmpneh:
11052 case Intrinsic::ppc_altivec_vcmpnew:
11053 case Intrinsic::ppc_altivec_vcmpnezb:
11054 case Intrinsic::ppc_altivec_vcmpnezh:
11055 case Intrinsic::ppc_altivec_vcmpnezw:
11056 if (Subtarget.hasP9Altivec())
11057 switch (IntrinsicID) {
11058 default:
11059 llvm_unreachable("Unknown comparison intrinsic.");
11060 case Intrinsic::ppc_altivec_vcmpneb:
11061 CompareOpc = 7;
11062 break;
11063 case Intrinsic::ppc_altivec_vcmpneh:
11064 CompareOpc = 71;
11065 break;
11066 case Intrinsic::ppc_altivec_vcmpnew:
11067 CompareOpc = 135;
11068 break;
11069 case Intrinsic::ppc_altivec_vcmpnezb:
11070 CompareOpc = 263;
11071 break;
11072 case Intrinsic::ppc_altivec_vcmpnezh:
11073 CompareOpc = 327;
11074 break;
11075 case Intrinsic::ppc_altivec_vcmpnezw:
11076 CompareOpc = 391;
11077 break;
11078 }
11079 else
11080 return false;
11081 break;
11082 case Intrinsic::ppc_altivec_vcmpgefp:
11083 CompareOpc = 454;
11084 break;
11085 case Intrinsic::ppc_altivec_vcmpgtfp:
11086 CompareOpc = 710;
11087 break;
11088 case Intrinsic::ppc_altivec_vcmpgtsb:
11089 CompareOpc = 774;
11090 break;
11091 case Intrinsic::ppc_altivec_vcmpgtsh:
11092 CompareOpc = 838;
11093 break;
11094 case Intrinsic::ppc_altivec_vcmpgtsw:
11095 CompareOpc = 902;
11096 break;
11097 case Intrinsic::ppc_altivec_vcmpgtsd:
11098 if (Subtarget.hasP8Altivec())
11099 CompareOpc = 967;
11100 else
11101 return false;
11102 break;
11103 case Intrinsic::ppc_altivec_vcmpgtub:
11104 CompareOpc = 518;
11105 break;
11106 case Intrinsic::ppc_altivec_vcmpgtuh:
11107 CompareOpc = 582;
11108 break;
11109 case Intrinsic::ppc_altivec_vcmpgtuw:
11110 CompareOpc = 646;
11111 break;
11112 case Intrinsic::ppc_altivec_vcmpgtud:
11113 if (Subtarget.hasP8Altivec())
11114 CompareOpc = 711;
11115 else
11116 return false;
11117 break;
11118 case Intrinsic::ppc_altivec_vcmpequq_p:
11119 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11120 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11121 if (!Subtarget.isISA3_1())
11122 return false;
11123 switch (IntrinsicID) {
11124 default:
11125 llvm_unreachable("Unknown comparison intrinsic.");
11126 case Intrinsic::ppc_altivec_vcmpequq_p:
11127 CompareOpc = 455;
11128 break;
11129 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11130 CompareOpc = 903;
11131 break;
11132 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11133 CompareOpc = 647;
11134 break;
11135 }
11136 isDot = true;
11137 break;
11138 }
11139 return true;
11140}
11141
11142/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom
11143/// lower, do it, otherwise return null.
11144SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11145 SelectionDAG &DAG) const {
11146 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11147
11148 SDLoc dl(Op);
11149 // Note: BCD instructions expect the immediate operand in vector form (v4i32),
11150 // but the builtin provides it as a scalar. To satisfy the instruction
11151 // encoding, we splat the scalar across all lanes using SPLAT_VECTOR.
11152 auto MapNodeWithSplatVector =
11153 [&](unsigned Opcode,
11154 std::initializer_list<SDValue> ExtraOps = {}) -> SDValue {
11155 SDValue SplatVal =
11156 DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: MVT::v4i32, Operand: Op.getOperand(i: 2));
11157
11158 SmallVector<SDValue, 4> Ops{SplatVal, Op.getOperand(i: 1)};
11159 Ops.append(in_start: ExtraOps.begin(), in_end: ExtraOps.end());
11160 return DAG.getNode(Opcode, DL: dl, VT: MVT::v16i8, Ops);
11161 };
11162
11163 switch (IntrinsicID) {
11164 case Intrinsic::thread_pointer:
11165 // Reads the thread pointer register, used for __builtin_thread_pointer.
11166 if (Subtarget.isPPC64())
11167 return DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
11168 return DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
11169
11170 case Intrinsic::ppc_rldimi: {
11171 assert(Subtarget.isPPC64() && "rldimi is only available in 64-bit!");
11172 SDValue Src = Op.getOperand(i: 1);
11173 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11174 if (Mask.isZero())
11175 return Op.getOperand(i: 2);
11176 if (Mask.isAllOnes())
11177 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src, N2: Op.getOperand(i: 3));
11178 uint64_t SH = Op.getConstantOperandVal(i: 3);
11179 unsigned MB = 0, ME = 0;
11180 if (!isRunOfOnes64(Val: Mask.getZExtValue(), MB, ME))
11181 report_fatal_error(reason: "invalid rldimi mask!");
11182 // rldimi requires ME=63-SH, otherwise rotation is needed before rldimi.
11183 if (ME < 63 - SH) {
11184 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11185 N2: DAG.getConstant(Val: ME + SH + 1, DL: dl, VT: MVT::i32));
11186 } else if (ME > 63 - SH) {
11187 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11188 N2: DAG.getConstant(Val: ME + SH - 63, DL: dl, VT: MVT::i32));
11189 }
11190 return SDValue(
11191 DAG.getMachineNode(Opcode: PPC::RLDIMI, dl, VT: MVT::i64,
11192 Ops: {Op.getOperand(i: 2), Src,
11193 DAG.getTargetConstant(Val: 63 - ME, DL: dl, VT: MVT::i32),
11194 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32)}),
11195 0);
11196 }
11197
11198 case Intrinsic::ppc_rlwimi: {
11199 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11200 if (Mask.isZero())
11201 return Op.getOperand(i: 2);
11202 if (Mask.isAllOnes())
11203 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
11204 N2: Op.getOperand(i: 3));
11205 unsigned MB = 0, ME = 0;
11206 if (!isRunOfOnes(Val: Mask.getZExtValue(), MB, ME))
11207 report_fatal_error(reason: "invalid rlwimi mask!");
11208 return SDValue(DAG.getMachineNode(
11209 Opcode: PPC::RLWIMI, dl, VT: MVT::i32,
11210 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1), Op.getOperand(i: 3),
11211 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11212 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11213 0);
11214 }
11215
11216 case Intrinsic::ppc_bcdshift:
11217 return MapNodeWithSplatVector(PPCISD::BCDSHIFT, {Op.getOperand(i: 3)});
11218 case Intrinsic::ppc_bcdshiftround:
11219 return MapNodeWithSplatVector(PPCISD::BCDSHIFTROUND, {Op.getOperand(i: 3)});
11220 case Intrinsic::ppc_bcdtruncate:
11221 return MapNodeWithSplatVector(PPCISD::BCDTRUNC, {Op.getOperand(i: 3)});
11222 case Intrinsic::ppc_bcdunsignedtruncate:
11223 return MapNodeWithSplatVector(PPCISD::BCDUTRUNC);
11224 case Intrinsic::ppc_bcdunsignedshift:
11225 return MapNodeWithSplatVector(PPCISD::BCDUSHIFT);
11226
11227 case Intrinsic::ppc_rlwnm: {
11228 if (Op.getConstantOperandVal(i: 3) == 0)
11229 return DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
11230 unsigned MB = 0, ME = 0;
11231 if (!isRunOfOnes(Val: Op.getConstantOperandVal(i: 3), MB, ME))
11232 report_fatal_error(reason: "invalid rlwnm mask!");
11233 return SDValue(
11234 DAG.getMachineNode(Opcode: PPC::RLWNM, dl, VT: MVT::i32,
11235 Ops: {Op.getOperand(i: 1), Op.getOperand(i: 2),
11236 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11237 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11238 0);
11239 }
11240
11241 case Intrinsic::ppc_mma_disassemble_acc: {
11242 if (Subtarget.isISAFuture()) {
11243 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11244 SDValue WideVec =
11245 SDValue(DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes,
11246 Ops: Op.getOperand(i: 1)),
11247 0);
11248 SmallVector<SDValue, 4> RetOps;
11249 SDValue Value = SDValue(WideVec.getNode(), 0);
11250 SDValue Value2 = SDValue(WideVec.getNode(), 1);
11251
11252 SDValue Extract;
11253 Extract = DAG.getNode(
11254 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11255 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11256 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11257 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11258 RetOps.push_back(Elt: Extract);
11259 Extract = DAG.getNode(
11260 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11261 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11262 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11263 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11264 RetOps.push_back(Elt: Extract);
11265 Extract = DAG.getNode(
11266 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11267 N1: Subtarget.isLittleEndian() ? Value : Value2,
11268 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11269 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11270 RetOps.push_back(Elt: Extract);
11271 Extract = DAG.getNode(
11272 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11273 N1: Subtarget.isLittleEndian() ? Value : Value2,
11274 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11275 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11276 RetOps.push_back(Elt: Extract);
11277 return DAG.getMergeValues(Ops: RetOps, dl);
11278 }
11279 [[fallthrough]];
11280 }
11281 case Intrinsic::ppc_vsx_disassemble_pair: {
11282 int NumVecs = 2;
11283 SDValue WideVec = Op.getOperand(i: 1);
11284 if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) {
11285 NumVecs = 4;
11286 WideVec = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: WideVec);
11287 }
11288 SmallVector<SDValue, 4> RetOps;
11289 for (int VecNo = 0; VecNo < NumVecs; VecNo++) {
11290 SDValue Extract = DAG.getNode(
11291 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: WideVec,
11292 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo
11293 : VecNo,
11294 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11295 RetOps.push_back(Elt: Extract);
11296 }
11297 return DAG.getMergeValues(Ops: RetOps, dl);
11298 }
11299
11300 case Intrinsic::ppc_build_dmr: {
11301 SmallVector<SDValue, 8> Pairs;
11302 SmallVector<SDValue, 8> Chains;
11303 for (int i = 1; i < 9; i += 2) {
11304 SDValue Hi = Op.getOperand(i);
11305 SDValue Lo = Op.getOperand(i: i + 1);
11306 if (Hi->getOpcode() == ISD::LOAD)
11307 Chains.push_back(Elt: Hi.getValue(R: 1));
11308 if (Lo->getOpcode() == ISD::LOAD)
11309 Chains.push_back(Elt: Lo.getValue(R: 1));
11310 Pairs.push_back(
11311 Elt: DAG.getNode(Opcode: PPCISD::PAIR_BUILD, DL: dl, VT: MVT::v256i1, Ops: {Hi, Lo}));
11312 }
11313 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
11314 SDValue Value = DMFInsert1024(Pairs, dl: SDLoc(Op), DAG);
11315 return DAG.getMergeValues(Ops: {Value, TF}, dl);
11316 }
11317
11318 case Intrinsic::ppc_mma_dmxxextfdmr512: {
11319 assert(Subtarget.isISAFuture() && "dmxxextfdmr512 requires ISA Future");
11320 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11321 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11322 "Specify P of 0 or 1 for lower or upper 512 bytes");
11323 unsigned HiLo = Idx->getSExtValue();
11324 unsigned Opcode;
11325 unsigned Subx;
11326 if (HiLo == 0) {
11327 Opcode = PPC::DMXXEXTFDMR512;
11328 Subx = PPC::sub_wacc_lo;
11329 } else {
11330 Opcode = PPC::DMXXEXTFDMR512_HI;
11331 Subx = PPC::sub_wacc_hi;
11332 }
11333 SDValue Subreg(
11334 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
11335 Op1: Op.getOperand(i: 1),
11336 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11337 0);
11338 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11339 return SDValue(DAG.getMachineNode(Opcode, dl, ResultTys: ReturnTypes, Ops: Subreg), 0);
11340 }
11341
11342 case Intrinsic::ppc_mma_dmxxextfdmr256: {
11343 assert(Subtarget.isISAFuture() && "dmxxextfdmr256 requires ISA Future");
11344 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11345 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11346 "Specify a dmr row pair 0-3");
11347 unsigned IdxVal = Idx->getSExtValue();
11348 unsigned Subx;
11349 switch (IdxVal) {
11350 case 0:
11351 Subx = PPC::sub_dmrrowp0;
11352 break;
11353 case 1:
11354 Subx = PPC::sub_dmrrowp1;
11355 break;
11356 case 2:
11357 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11358 break;
11359 case 3:
11360 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11361 break;
11362 }
11363 SDValue Subreg(
11364 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v256i1,
11365 Op1: Op.getOperand(i: 1),
11366 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11367 0);
11368 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11369 return SDValue(
11370 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR256, dl, VT: MVT::v256i1, Ops: {Subreg, P}),
11371 0);
11372 }
11373
11374 case Intrinsic::ppc_mma_dmxxinstdmr512: {
11375 assert(Subtarget.isISAFuture() && "dmxxinstdmr512 requires ISA Future");
11376 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 4));
11377 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11378 "Specify P of 0 or 1 for lower or upper 512 bytes");
11379 unsigned HiLo = Idx->getSExtValue();
11380 unsigned Opcode;
11381 unsigned Subx;
11382 if (HiLo == 0) {
11383 Opcode = PPCISD::INST512;
11384 Subx = PPC::sub_wacc_lo;
11385 } else {
11386 Opcode = PPCISD::INST512HI;
11387 Subx = PPC::sub_wacc_hi;
11388 }
11389 SDValue Wacc = DAG.getNode(Opcode, DL: dl, VT: MVT::v512i1, N1: Op.getOperand(i: 2),
11390 N2: Op.getOperand(i: 3));
11391 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11392 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11393 Op1: Op.getOperand(i: 1), Op2: Wacc, Op3: SubReg),
11394 0);
11395 }
11396
11397 case Intrinsic::ppc_mma_dmxxinstdmr256: {
11398 assert(Subtarget.isISAFuture() && "dmxxinstdmr256 requires ISA Future");
11399 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11400 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11401 "Specify a dmr row pair 0-3");
11402 unsigned IdxVal = Idx->getSExtValue();
11403 unsigned Subx;
11404 switch (IdxVal) {
11405 case 0:
11406 Subx = PPC::sub_dmrrowp0;
11407 break;
11408 case 1:
11409 Subx = PPC::sub_dmrrowp1;
11410 break;
11411 case 2:
11412 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11413 break;
11414 case 3:
11415 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11416 break;
11417 }
11418 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11419 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11420 SDValue DMRRowp =
11421 DAG.getNode(Opcode: PPCISD::INST256, DL: dl, VT: MVT::v256i1, N1: Op.getOperand(i: 2), N2: P);
11422 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11423 Op1: Op.getOperand(i: 1), Op2: DMRRowp, Op3: SubReg),
11424 0);
11425 }
11426
11427 case Intrinsic::ppc_mma_xxmfacc:
11428 case Intrinsic::ppc_mma_xxmtacc: {
11429 // Allow pre-isa-future subtargets to lower as normal.
11430 if (!Subtarget.isISAFuture())
11431 return SDValue();
11432 // The intrinsics for xxmtacc and xxmfacc take one argument of
11433 // type v512i1, for future cpu the corresponding wacc instruction
11434 // dmxx[inst|extf]dmr512 is always generated for type v512i1, negating
11435 // the need to produce the xxm[t|f]acc.
11436 SDValue WideVec = Op.getOperand(i: 1);
11437 DAG.ReplaceAllUsesWith(From: Op, To: WideVec);
11438 return SDValue();
11439 }
11440
11441 case Intrinsic::ppc_unpack_longdouble: {
11442 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11443 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11444 "Argument of long double unpack must be 0 or 1!");
11445 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::f64, N1: Op.getOperand(i: 1),
11446 N2: DAG.getConstant(Val: !!(Idx->getSExtValue()), DL: dl,
11447 VT: Idx->getValueType(ResNo: 0)));
11448 }
11449
11450 case Intrinsic::ppc_compare_exp_lt:
11451 case Intrinsic::ppc_compare_exp_gt:
11452 case Intrinsic::ppc_compare_exp_eq:
11453 case Intrinsic::ppc_compare_exp_uo: {
11454 unsigned Pred;
11455 switch (IntrinsicID) {
11456 case Intrinsic::ppc_compare_exp_lt:
11457 Pred = PPC::PRED_LT;
11458 break;
11459 case Intrinsic::ppc_compare_exp_gt:
11460 Pred = PPC::PRED_GT;
11461 break;
11462 case Intrinsic::ppc_compare_exp_eq:
11463 Pred = PPC::PRED_EQ;
11464 break;
11465 case Intrinsic::ppc_compare_exp_uo:
11466 Pred = PPC::PRED_UN;
11467 break;
11468 }
11469 return SDValue(
11470 DAG.getMachineNode(
11471 Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11472 Ops: {SDValue(DAG.getMachineNode(Opcode: PPC::XSCMPEXPDP, dl, VT: MVT::i32,
11473 Op1: Op.getOperand(i: 1), Op2: Op.getOperand(i: 2)),
11474 0),
11475 DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11476 DAG.getTargetConstant(Val: Pred, DL: dl, VT: MVT::i32)}),
11477 0);
11478 }
11479 case Intrinsic::ppc_test_data_class: {
11480 EVT OpVT = Op.getOperand(i: 1).getValueType();
11481 unsigned CmprOpc = OpVT == MVT::f128 ? PPC::XSTSTDCQP
11482 : (OpVT == MVT::f64 ? PPC::XSTSTDCDP
11483 : PPC::XSTSTDCSP);
11484 // Lower __builtin_ppc_test_data_class(value, mask) to XSTSTDC* instruction.
11485 // The XSTSTDC* instructions test if a floating-point value matches any of
11486 // the data classes specified in the mask, setting CR field bits
11487 // accordingly. We need to extract the EQ bit (bit 2) from the CR field and
11488 // convert it to an integer result (1 if match, 0 if no match).
11489 //
11490 // Note: Operands are swapped because XSTSTDC* expects (mask, value) but the
11491 // intrinsic provides (value, mask) as Op.getOperand(1) and
11492 // Op.getOperand(2).
11493 SDValue TestDataClass =
11494 SDValue(DAG.getMachineNode(Opcode: CmprOpc, dl, VT: MVT::i32,
11495 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1)}),
11496 0);
11497 if (Subtarget.isISA3_1()) {
11498 // ISA 3.1+: Use SETBC instruction to directly convert CR bit to integer.
11499 // This is more efficient than the SELECT_CC approach used in earlier
11500 // ISAs.
11501 SDValue SubRegIdx = DAG.getTargetConstant(Val: PPC::sub_eq, DL: dl, VT: MVT::i32);
11502 SDValue CRBit =
11503 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11504 Op1: TestDataClass, Op2: SubRegIdx),
11505 0);
11506
11507 return DAG.getNode(Opcode: PPCISD::SETBC, DL: dl, VT: MVT::i32, Operand: CRBit);
11508 }
11509
11510 // Pre-ISA 3.1: Use SELECT_CC to convert CR field to integer (1 or 0).
11511 return SDValue(
11512 DAG.getMachineNode(Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11513 Ops: {TestDataClass, DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32),
11514 DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11515 DAG.getTargetConstant(Val: PPC::PRED_EQ, DL: dl, VT: MVT::i32)}),
11516 0);
11517 }
11518 case Intrinsic::ppc_fnmsub: {
11519 EVT VT = Op.getOperand(i: 1).getValueType();
11520 if (!Subtarget.hasVSX() || (!Subtarget.hasFloat128() && VT == MVT::f128))
11521 return DAG.getNode(
11522 Opcode: ISD::FNEG, DL: dl, VT,
11523 Operand: DAG.getNode(Opcode: ISD::FMA, DL: dl, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11524 N3: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op.getOperand(i: 3))));
11525 return DAG.getNode(Opcode: PPCISD::FNMSUB, DL: dl, VT, N1: Op.getOperand(i: 1),
11526 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11527 }
11528 case Intrinsic::ppc_convert_f128_to_ppcf128:
11529 case Intrinsic::ppc_convert_ppcf128_to_f128: {
11530 RTLIB::Libcall LC = IntrinsicID == Intrinsic::ppc_convert_ppcf128_to_f128
11531 ? RTLIB::CONVERT_PPCF128_F128
11532 : RTLIB::CONVERT_F128_PPCF128;
11533 MakeLibCallOptions CallOptions;
11534 std::pair<SDValue, SDValue> Result =
11535 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op.getOperand(i: 1), CallOptions,
11536 dl, Chain: SDValue());
11537 return Result.first;
11538 }
11539 case Intrinsic::ppc_maxfe:
11540 case Intrinsic::ppc_maxfl:
11541 case Intrinsic::ppc_maxfs:
11542 case Intrinsic::ppc_minfe:
11543 case Intrinsic::ppc_minfl:
11544 case Intrinsic::ppc_minfs: {
11545 EVT VT = Op.getValueType();
11546 assert(
11547 all_of(Op->ops().drop_front(4),
11548 [VT](const SDUse &Use) { return Use.getValueType() == VT; }) &&
11549 "ppc_[max|min]f[e|l|s] must have uniform type arguments");
11550 (void)VT;
11551 ISD::CondCode CC = ISD::SETGT;
11552 if (IntrinsicID == Intrinsic::ppc_minfe ||
11553 IntrinsicID == Intrinsic::ppc_minfl ||
11554 IntrinsicID == Intrinsic::ppc_minfs)
11555 CC = ISD::SETLT;
11556 unsigned I = Op.getNumOperands() - 2, Cnt = I;
11557 SDValue Res = Op.getOperand(i: I);
11558 for (--I; Cnt != 0; --Cnt, I = (--I == 0 ? (Op.getNumOperands() - 1) : I)) {
11559 Res =
11560 DAG.getSelectCC(DL: dl, LHS: Res, RHS: Op.getOperand(i: I), True: Res, False: Op.getOperand(i: I), Cond: CC);
11561 }
11562 return Res;
11563 }
11564 }
11565
11566 // If this is a lowered altivec predicate compare, CompareOpc is set to the
11567 // opcode number of the comparison.
11568 int CompareOpc;
11569 bool isDot;
11570 if (!getVectorCompareInfo(Intrin: Op, CompareOpc, isDot, Subtarget))
11571 return SDValue(); // Don't custom lower most intrinsics.
11572
11573 // If this is a non-dot comparison, make the VCMP node and we are done.
11574 if (!isDot) {
11575 SDValue Tmp = DAG.getNode(Opcode: PPCISD::VCMP, DL: dl, VT: Op.getOperand(i: 2).getValueType(),
11576 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11577 N3: DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32));
11578 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Tmp);
11579 }
11580
11581 // Create the PPCISD altivec 'dot' comparison node.
11582 SDValue Ops[] = {
11583 Op.getOperand(i: 2), // LHS
11584 Op.getOperand(i: 3), // RHS
11585 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
11586 };
11587 EVT VTs[] = { Op.getOperand(i: 2).getValueType(), MVT::Glue };
11588 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
11589
11590 // Unpack the result based on how the target uses it.
11591 unsigned BitNo; // Bit # of CR6.
11592 bool InvertBit; // Invert result?
11593 unsigned Bitx;
11594 unsigned SetOp;
11595 switch (Op.getConstantOperandVal(i: 1)) {
11596 default: // Can't happen, don't crash on invalid number though.
11597 case 0: // Return the value of the EQ bit of CR6.
11598 BitNo = 0;
11599 InvertBit = false;
11600 Bitx = PPC::sub_eq;
11601 SetOp = PPCISD::SETBC;
11602 break;
11603 case 1: // Return the inverted value of the EQ bit of CR6.
11604 BitNo = 0;
11605 InvertBit = true;
11606 Bitx = PPC::sub_eq;
11607 SetOp = PPCISD::SETBCR;
11608 break;
11609 case 2: // Return the value of the LT bit of CR6.
11610 BitNo = 2;
11611 InvertBit = false;
11612 Bitx = PPC::sub_lt;
11613 SetOp = PPCISD::SETBC;
11614 break;
11615 case 3: // Return the inverted value of the LT bit of CR6.
11616 BitNo = 2;
11617 InvertBit = true;
11618 Bitx = PPC::sub_lt;
11619 SetOp = PPCISD::SETBCR;
11620 break;
11621 }
11622
11623 SDValue GlueOp = CompNode.getValue(R: 1);
11624 if (Subtarget.isISA3_1()) {
11625 SDValue SubRegIdx = DAG.getTargetConstant(Val: Bitx, DL: dl, VT: MVT::i32);
11626 SDValue CR6Reg = DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32);
11627 SDValue CRBit =
11628 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11629 Op1: CR6Reg, Op2: SubRegIdx, Op3: GlueOp),
11630 0);
11631 return DAG.getNode(Opcode: SetOp, DL: dl, VT: MVT::i32, Operand: CRBit);
11632 }
11633
11634 // Now that we have the comparison, emit a copy from the CR to a GPR.
11635 // This is flagged to the above dot comparison.
11636 SDValue Flags = DAG.getNode(Opcode: PPCISD::MFOCRF, DL: dl, VT: MVT::i32,
11637 N1: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32), N2: GlueOp);
11638
11639 // Shift the bit into the low position.
11640 Flags = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: Flags,
11641 N2: DAG.getConstant(Val: 8 - (3 - BitNo), DL: dl, VT: MVT::i32));
11642 // Isolate the bit.
11643 Flags = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Flags,
11644 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11645
11646 // If we are supposed to, toggle the bit.
11647 if (InvertBit)
11648 Flags = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Flags,
11649 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11650 return Flags;
11651}
11652
11653SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
11654 SelectionDAG &DAG) const {
11655 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to
11656 // the beginning of the argument list.
11657 int ArgStart = isa<ConstantSDNode>(Val: Op.getOperand(i: 0)) ? 0 : 1;
11658 SDLoc DL(Op);
11659 switch (Op.getConstantOperandVal(i: ArgStart)) {
11660 case Intrinsic::ppc_cfence: {
11661 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument.");
11662 SDValue Val = Op.getOperand(i: ArgStart + 1);
11663 EVT Ty = Val.getValueType();
11664 if (Ty == MVT::i128) {
11665 // FIXME: Testing one of two paired registers is sufficient to guarantee
11666 // ordering?
11667 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: Val);
11668 }
11669 unsigned Opcode = Subtarget.isPPC64() ? PPC::CFENCE8 : PPC::CFENCE;
11670 return SDValue(
11671 DAG.getMachineNode(
11672 Opcode, dl: DL, VT: MVT::Other,
11673 Op1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getScalarIntVT(), Operand: Val),
11674 Op2: Op.getOperand(i: 0)),
11675 0);
11676 }
11677 case Intrinsic::ppc_disassemble_dmr: {
11678 assert(ArgStart == 1 &&
11679 "llvm.ppc.disassemble.dmr must carry a chain argument.");
11680 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: Op.getOperand(i: ArgStart + 2),
11681 Ptr: Op.getOperand(i: ArgStart + 1), PtrInfo: MachinePointerInfo());
11682 }
11683 default:
11684 break;
11685 }
11686 return SDValue();
11687}
11688
11689// Lower scalar BSWAP64 to xxbrd.
11690SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
11691 SDLoc dl(Op);
11692 if (!Subtarget.isPPC64())
11693 return Op;
11694
11695 if (Subtarget.hasP9Vector()) {
11696 // MTVSRDD
11697 Op = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: MVT::v2i64, N1: Op.getOperand(i: 0),
11698 N2: Op.getOperand(i: 0));
11699 // XXBRD
11700 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Op);
11701 // MFVSRD
11702 int VectorIndex = 0;
11703 if (Subtarget.isLittleEndian())
11704 VectorIndex = 1;
11705 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Op,
11706 N2: DAG.getTargetConstant(Val: VectorIndex, DL: dl, VT: MVT::i32));
11707 return Op;
11708 }
11709
11710 // For Power8, use parallel rotate instructions for faster bswap64.
11711 SDValue Input = Op.getOperand(i: 0);
11712 // Helper to create rotate-and-insert operations (RLWIMI/RLDIMI).
11713 auto CreateRotateInsert =
11714 [&](unsigned Opcode, MVT VT, SDValue Dest, SDValue Src, unsigned RotAmt,
11715 unsigned MaskBegin,
11716 std::optional<unsigned> MaskEnd = std::nullopt) -> SDValue {
11717 SmallVector<SDValue, 5> Ops = {
11718 Dest, Src, DAG.getTargetConstant(Val: RotAmt, DL: dl, VT: MVT::i32),
11719 DAG.getTargetConstant(Val: MaskBegin, DL: dl, VT: MVT::i32)};
11720 if (MaskEnd.has_value())
11721 Ops.push_back(Elt: DAG.getTargetConstant(Val: *MaskEnd, DL: dl, VT: MVT::i32));
11722
11723 return SDValue(DAG.getMachineNode(Opcode, dl, VT, Ops), 0);
11724 };
11725
11726 // Helper to perform 32-bit byte swap using rotl(8) + 2x rlwimi.
11727 auto Swap32 = [&](SDValue Val32) -> SDValue {
11728 SDValue Rot = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Val32,
11729 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
11730 // Insert bits [24:31] from Val32 into Rot at position [0:7].
11731 SDValue Swap =
11732 CreateRotateInsert(PPC::RLWIMI, MVT::i32, Rot, Val32, 24, 0, 7);
11733 // Insert bits [16:23] from Val32 into Swap at position [16:23].
11734 return CreateRotateInsert(PPC::RLWIMI, MVT::i32, Swap, Val32, 24, 16, 23);
11735 };
11736 // Extract and swap high and low 32-bit halves independently for parallelism.
11737 SDValue Hi32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
11738 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Input,
11739 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64)));
11740 SDValue Lo32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Input);
11741
11742 // Combine swapped halves: rotate LoSwap left by 32 bits and insert into
11743 // HiSwap to swap their positions, completing the 64-bit byte reversal.
11744 SDValue HiSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Hi32));
11745 SDValue LoSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Lo32));
11746
11747 return CreateRotateInsert(PPC::RLDIMI, MVT::i64, HiSwap, LoSwap, 32, 0);
11748}
11749
11750// ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be
11751// compared to a value that is atomically loaded (atomic loads zero-extend).
11752SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
11753 SelectionDAG &DAG) const {
11754 assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP &&
11755 "Expecting an atomic compare-and-swap here.");
11756 SDLoc dl(Op);
11757 auto *AtomicNode = cast<AtomicSDNode>(Val: Op.getNode());
11758 EVT MemVT = AtomicNode->getMemoryVT();
11759 if (MemVT.getSizeInBits() >= 32)
11760 return Op;
11761
11762 SDValue CmpOp = Op.getOperand(i: 2);
11763 // If this is already correctly zero-extended, leave it alone.
11764 auto HighBits = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - MemVT.getSizeInBits());
11765 if (DAG.MaskedValueIsZero(Op: CmpOp, Mask: HighBits))
11766 return Op;
11767
11768 // Clear the high bits of the compare operand.
11769 unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1;
11770 SDValue NewCmpOp =
11771 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: CmpOp,
11772 N2: DAG.getConstant(Val: MaskVal, DL: dl, VT: MVT::i32));
11773
11774 // Replace the existing compare operand with the properly zero-extended one.
11775 SmallVector<SDValue, 4> Ops;
11776 for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++)
11777 Ops.push_back(Elt: AtomicNode->getOperand(Num: i));
11778 Ops[2] = NewCmpOp;
11779 MachineMemOperand *MMO = AtomicNode->getMemOperand();
11780 SDVTList Tys = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
11781 auto NodeTy =
11782 (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16;
11783 return DAG.getMemIntrinsicNode(Opcode: NodeTy, dl, VTList: Tys, Ops, MemVT, MMO);
11784}
11785
11786SDValue PPCTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
11787 SelectionDAG &DAG) const {
11788 AtomicSDNode *N = cast<AtomicSDNode>(Val: Op.getNode());
11789 EVT MemVT = N->getMemoryVT();
11790 assert(MemVT.getSimpleVT() == MVT::i128 &&
11791 "Expect quadword atomic operations");
11792 SDLoc dl(N);
11793 unsigned Opc = N->getOpcode();
11794 switch (Opc) {
11795 case ISD::ATOMIC_LOAD: {
11796 // Lower quadword atomic load to int_ppc_atomic_load_i128 which will be
11797 // lowered to ppc instructions by pattern matching instruction selector.
11798 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
11799 SmallVector<SDValue, 4> Ops{
11800 N->getOperand(Num: 0),
11801 DAG.getConstant(Val: Intrinsic::ppc_atomic_load_i128, DL: dl, VT: MVT::i32)};
11802 for (int I = 1, E = N->getNumOperands(); I < E; ++I)
11803 Ops.push_back(Elt: N->getOperand(Num: I));
11804 SDValue LoadedVal = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys,
11805 Ops, MemVT, MMO: N->getMemOperand());
11806 SDValue ValLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal);
11807 SDValue ValHi =
11808 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal.getValue(R: 1));
11809 ValHi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ValHi,
11810 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11811 SDValue Val =
11812 DAG.getNode(Opcode: ISD::OR, DL: dl, ResultTys: {MVT::i128, MVT::Other}, Ops: {ValLo, ValHi});
11813 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, ResultTys: {MVT::i128, MVT::Other},
11814 Ops: {Val, LoadedVal.getValue(R: 2)});
11815 }
11816 case ISD::ATOMIC_STORE: {
11817 // Lower quadword atomic store to int_ppc_atomic_store_i128 which will be
11818 // lowered to ppc instructions by pattern matching instruction selector.
11819 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
11820 SmallVector<SDValue, 4> Ops{
11821 N->getOperand(Num: 0),
11822 DAG.getConstant(Val: Intrinsic::ppc_atomic_store_i128, DL: dl, VT: MVT::i32)};
11823 SDValue Val = N->getOperand(Num: 1);
11824 SDValue ValLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: Val);
11825 SDValue ValHi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: Val,
11826 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11827 ValHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: ValHi);
11828 Ops.push_back(Elt: ValLo);
11829 Ops.push_back(Elt: ValHi);
11830 Ops.push_back(Elt: N->getOperand(Num: 2));
11831 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops, MemVT,
11832 MMO: N->getMemOperand());
11833 }
11834 default:
11835 llvm_unreachable("Unexpected atomic opcode");
11836 }
11837}
11838
11839static SDValue getDataClassTest(SDValue Op, FPClassTest Mask, const SDLoc &Dl,
11840 SelectionDAG &DAG,
11841 const PPCSubtarget &Subtarget) {
11842 assert(Mask <= fcAllFlags && "Invalid fp_class flags!");
11843
11844 enum DataClassMask {
11845 DC_NAN = 1 << 6,
11846 DC_NEG_INF = 1 << 4,
11847 DC_POS_INF = 1 << 5,
11848 DC_NEG_ZERO = 1 << 2,
11849 DC_POS_ZERO = 1 << 3,
11850 DC_NEG_SUBNORM = 1,
11851 DC_POS_SUBNORM = 1 << 1,
11852 };
11853
11854 EVT VT = Op.getValueType();
11855
11856 unsigned TestOp = VT == MVT::f128 ? PPC::XSTSTDCQP
11857 : VT == MVT::f64 ? PPC::XSTSTDCDP
11858 : PPC::XSTSTDCSP;
11859
11860 if (Mask == fcAllFlags)
11861 return DAG.getBoolConstant(V: true, DL: Dl, VT: MVT::i1, OpVT: VT);
11862 if (Mask == 0)
11863 return DAG.getBoolConstant(V: false, DL: Dl, VT: MVT::i1, OpVT: VT);
11864
11865 // When it's cheaper or necessary to test reverse flags.
11866 if ((Mask & fcNormal) == fcNormal || Mask == ~fcQNan || Mask == ~fcSNan) {
11867 SDValue Rev = getDataClassTest(Op, Mask: ~Mask, Dl, DAG, Subtarget);
11868 return DAG.getNOT(DL: Dl, Val: Rev, VT: MVT::i1);
11869 }
11870
11871 // Power doesn't support testing whether a value is 'normal'. Test the rest
11872 // first, and test if it's 'not not-normal' with expected sign.
11873 if (Mask & fcNormal) {
11874 SDValue Rev(DAG.getMachineNode(
11875 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11876 Op1: DAG.getTargetConstant(Val: DC_NAN | DC_NEG_INF | DC_POS_INF |
11877 DC_NEG_ZERO | DC_POS_ZERO |
11878 DC_NEG_SUBNORM | DC_POS_SUBNORM,
11879 DL: Dl, VT: MVT::i32),
11880 Op2: Op),
11881 0);
11882 // Sign are stored in CR bit 0, result are in CR bit 2.
11883 SDValue Sign(
11884 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11885 Op2: DAG.getTargetConstant(Val: PPC::sub_lt, DL: Dl, VT: MVT::i32)),
11886 0);
11887 SDValue Normal(DAG.getNOT(
11888 DL: Dl,
11889 Val: SDValue(DAG.getMachineNode(
11890 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11891 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11892 0),
11893 VT: MVT::i1));
11894 if (Mask & fcPosNormal)
11895 Sign = DAG.getNOT(DL: Dl, Val: Sign, VT: MVT::i1);
11896 SDValue Result = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: Sign, N2: Normal);
11897 if (Mask == fcPosNormal || Mask == fcNegNormal)
11898 return Result;
11899
11900 return DAG.getNode(
11901 Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11902 N1: getDataClassTest(Op, Mask: Mask & ~fcNormal, Dl, DAG, Subtarget), N2: Result);
11903 }
11904
11905 // The instruction doesn't differentiate between signaling or quiet NaN. Test
11906 // the rest first, and test if it 'is NaN and is signaling/quiet'.
11907 if ((Mask & fcNan) == fcQNan || (Mask & fcNan) == fcSNan) {
11908 bool IsQuiet = Mask & fcQNan;
11909 SDValue NanCheck = getDataClassTest(Op, Mask: fcNan, Dl, DAG, Subtarget);
11910
11911 // Quietness is determined by the first bit in fraction field.
11912 uint64_t QuietMask = 0;
11913 SDValue HighWord;
11914 if (VT == MVT::f128) {
11915 HighWord = DAG.getNode(
11916 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: DAG.getBitcast(VT: MVT::v4i32, V: Op),
11917 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 3 : 0, DL: Dl));
11918 QuietMask = 0x8000;
11919 } else if (VT == MVT::f64) {
11920 if (Subtarget.isPPC64()) {
11921 HighWord = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::i32,
11922 N1: DAG.getBitcast(VT: MVT::i64, V: Op),
11923 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11924 } else {
11925 SDValue Vec = DAG.getBitcast(
11926 VT: MVT::v4i32, V: DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: Dl, VT: MVT::v2f64, Operand: Op));
11927 HighWord = DAG.getNode(
11928 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: Vec,
11929 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 1 : 0, DL: Dl));
11930 }
11931 QuietMask = 0x80000;
11932 } else if (VT == MVT::f32) {
11933 HighWord = DAG.getBitcast(VT: MVT::i32, V: Op);
11934 QuietMask = 0x400000;
11935 }
11936 SDValue NanRes = DAG.getSetCC(
11937 DL: Dl, VT: MVT::i1,
11938 LHS: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: HighWord,
11939 N2: DAG.getConstant(Val: QuietMask, DL: Dl, VT: MVT::i32)),
11940 RHS: DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32), Cond: IsQuiet ? ISD::SETNE : ISD::SETEQ);
11941 NanRes = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: NanCheck, N2: NanRes);
11942 if (Mask == fcQNan || Mask == fcSNan)
11943 return NanRes;
11944
11945 return DAG.getNode(Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11946 N1: getDataClassTest(Op, Mask: Mask & ~fcNan, Dl, DAG, Subtarget),
11947 N2: NanRes);
11948 }
11949
11950 unsigned NativeMask = 0;
11951 if ((Mask & fcNan) == fcNan)
11952 NativeMask |= DC_NAN;
11953 if (Mask & fcNegInf)
11954 NativeMask |= DC_NEG_INF;
11955 if (Mask & fcPosInf)
11956 NativeMask |= DC_POS_INF;
11957 if (Mask & fcNegZero)
11958 NativeMask |= DC_NEG_ZERO;
11959 if (Mask & fcPosZero)
11960 NativeMask |= DC_POS_ZERO;
11961 if (Mask & fcNegSubnormal)
11962 NativeMask |= DC_NEG_SUBNORM;
11963 if (Mask & fcPosSubnormal)
11964 NativeMask |= DC_POS_SUBNORM;
11965 return SDValue(
11966 DAG.getMachineNode(
11967 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1,
11968 Op1: SDValue(DAG.getMachineNode(
11969 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11970 Op1: DAG.getTargetConstant(Val: NativeMask, DL: Dl, VT: MVT::i32), Op2: Op),
11971 0),
11972 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11973 0);
11974}
11975
11976SDValue PPCTargetLowering::LowerIS_FPCLASS(SDValue Op,
11977 SelectionDAG &DAG) const {
11978 assert(Subtarget.hasP9Vector() && "Test data class requires Power9");
11979 SDValue LHS = Op.getOperand(i: 0);
11980 uint64_t RHSC = Op.getConstantOperandVal(i: 1);
11981 SDLoc Dl(Op);
11982 FPClassTest Category = static_cast<FPClassTest>(RHSC);
11983 if (LHS.getValueType() == MVT::ppcf128) {
11984 // The higher part determines the value class.
11985 LHS = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::f64, N1: LHS,
11986 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11987 }
11988
11989 return getDataClassTest(Op: LHS, Mask: Category, Dl, DAG, Subtarget);
11990}
11991
11992// Adjust the length value for a load/store with length to account for the
11993// instructions requiring a left justified length, and for non-byte element
11994// types requiring scaling by element size.
11995static SDValue AdjustLength(SDValue Val, unsigned Bits, bool Left,
11996 SelectionDAG &DAG) {
11997 SDLoc dl(Val);
11998 EVT VT = Val->getValueType(ResNo: 0);
11999 unsigned LeftAdj = Left ? VT.getSizeInBits() - 8 : 0;
12000 unsigned TypeAdj = llvm::countr_zero<uint32_t>(Val: Bits / 8);
12001 SDValue SHLAmt = DAG.getConstant(Val: LeftAdj + TypeAdj, DL: dl, VT);
12002 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Val, N2: SHLAmt);
12003}
12004
12005SDValue PPCTargetLowering::LowerVP_LOAD(SDValue Op, SelectionDAG &DAG) const {
12006 auto VPLD = cast<VPLoadSDNode>(Val&: Op);
12007 bool Future = Subtarget.isISAFuture();
12008 SDLoc dl(Op);
12009 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(3).getNode(), true) &&
12010 "Mask predication not supported");
12011 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12012 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPLD->getOperand(Num: 4));
12013 unsigned IID = Future ? Intrinsic::ppc_vsx_lxvrl : Intrinsic::ppc_vsx_lxvl;
12014 unsigned EltBits = Op->getValueType(ResNo: 0).getScalarType().getSizeInBits();
12015 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
12016 SDValue Ops[] = {VPLD->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
12017 VPLD->getOperand(Num: 1), Len};
12018 SDVTList Tys = DAG.getVTList(VT1: Op->getValueType(ResNo: 0), VT2: MVT::Other);
12019 SDValue VPL =
12020 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys, Ops,
12021 MemVT: VPLD->getMemoryVT(), MMO: VPLD->getMemOperand());
12022 return VPL;
12023}
12024
12025SDValue PPCTargetLowering::LowerVP_STORE(SDValue Op, SelectionDAG &DAG) const {
12026 auto VPST = cast<VPStoreSDNode>(Val&: Op);
12027 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(4).getNode(), true) &&
12028 "Mask predication not supported");
12029 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12030 SDLoc dl(Op);
12031 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPST->getOperand(Num: 5));
12032 unsigned EltBits =
12033 Op->getOperand(Num: 1).getValueType().getScalarType().getSizeInBits();
12034 bool Future = Subtarget.isISAFuture();
12035 unsigned IID = Future ? Intrinsic::ppc_vsx_stxvrl : Intrinsic::ppc_vsx_stxvl;
12036 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
12037 SDValue Ops[] = {
12038 VPST->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
12039 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: VPST->getOperand(Num: 1)),
12040 VPST->getOperand(Num: 2), Len};
12041 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12042 SDValue VPS =
12043 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12044 MemVT: VPST->getMemoryVT(), MMO: VPST->getMemOperand());
12045 return VPS;
12046}
12047
12048SDValue PPCTargetLowering::LowerPartialReduce(SDValue Op,
12049 SelectionDAG &DAG) const {
12050 SDValue Acc = Op.getOperand(i: 0);
12051 SDValue Op1 = Op.getOperand(i: 1);
12052 SDValue Op2 = Op.getOperand(i: 2);
12053
12054 assert(Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA &&
12055 "Unexpected partial reduction");
12056
12057 if (Acc.getValueType() != MVT::v4i32)
12058 return SDValue();
12059 if (Op1.getValueType() != MVT::v16i32 || Op1.getOpcode() != ISD::SIGN_EXTEND)
12060 return SDValue();
12061 SDValue Op1Input = Op1.getOperand(i: 0);
12062 if (Op1Input.getValueType() != MVT::v16i8 || !llvm::isOneOrOneSplat(V: Op2))
12063 return SDValue();
12064
12065 SDLoc dl(Op);
12066 SDValue Ones = DAG.getConstant(Val: 1, DL: dl, VT: MVT::v16i8);
12067 return DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_SUMLA, DL: dl, VT: MVT::v4i32, N1: Acc, N2: Op1Input,
12068 N3: Ones);
12069}
12070
12071SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
12072 SelectionDAG &DAG) const {
12073 SDLoc dl(Op);
12074
12075 MachineFunction &MF = DAG.getMachineFunction();
12076 SDValue Op0 = Op.getOperand(i: 0);
12077 EVT ValVT = Op0.getValueType();
12078 unsigned EltSize = Op.getValueType().getScalarSizeInBits();
12079 if (isa<ConstantSDNode>(Val: Op0) && EltSize <= 32) {
12080 int64_t IntVal = Op.getConstantOperandVal(i: 0);
12081 if (IntVal >= -16 && IntVal <= 15)
12082 return getCanonicalConstSplat(Val: IntVal, SplatSize: EltSize / 8, VT: Op.getValueType(), DAG,
12083 dl);
12084 }
12085
12086 ReuseLoadInfo RLI;
12087 if (Subtarget.hasLFIWAX() && Subtarget.hasVSX() &&
12088 Op.getValueType() == MVT::v4i32 && Op0.getOpcode() == ISD::LOAD &&
12089 Op0.getValueType() == MVT::i32 && Op0.hasOneUse() &&
12090 canReuseLoadAddress(Op: Op0, MemVT: MVT::i32, RLI, DAG, ET: ISD::NON_EXTLOAD)) {
12091
12092 MachineMemOperand *MMO = MF.getMachineMemOperand(
12093 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
12094 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
12095 SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
12096 SDValue Bits = DAG.getMemIntrinsicNode(
12097 Opcode: PPCISD::LD_SPLAT, dl, VTList: DAG.getVTList(VT1: MVT::v4i32, VT2: MVT::Other), Ops,
12098 MemVT: MVT::i32, MMO);
12099 if (RLI.ResChain)
12100 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
12101 return Bits.getValue(R: 0);
12102 }
12103
12104 // Create a stack slot that is 16-byte aligned.
12105 MachineFrameInfo &MFI = MF.getFrameInfo();
12106 int FrameIdx = MFI.CreateStackObject(Size: 16, Alignment: Align(16), isSpillSlot: false);
12107 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12108 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
12109
12110 SDValue Val = Op0;
12111 // P10 hardware store forwarding requires that a single store contains all
12112 // the data for the load. P10 is able to merge a pair of adjacent stores. Try
12113 // to avoid load hit store on P10 when running binaries compiled for older
12114 // processors by generating two mergeable scalar stores to forward with the
12115 // vector load.
12116 if (EnableP10StoreForward && Subtarget.isPPC64() &&
12117 !Subtarget.isLittleEndian() && ValVT.isInteger() &&
12118 ValVT.getSizeInBits() <= 64) {
12119 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: Val);
12120 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: MVT::i64, DL: DAG.getDataLayout());
12121 SDValue ShiftBy = DAG.getConstant(
12122 Val: 64 - Op.getValueType().getScalarSizeInBits(), DL: dl, VT: ShiftAmountTy);
12123 Val = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: Val, N2: ShiftBy);
12124 SDValue Plus8 =
12125 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FIdx, N2: DAG.getConstant(Val: 8, DL: dl, VT: PtrVT));
12126 SDValue Store2 =
12127 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: Plus8, PtrInfo: MachinePointerInfo());
12128 SDValue Store = DAG.getStore(Chain: Store2, dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12129 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx,
12130 PtrInfo: MachinePointerInfo());
12131 }
12132
12133 // Store the input value into Value#0 of the stack slot.
12134 SDValue Store =
12135 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12136 // Load it out.
12137 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12138}
12139
12140SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
12141 SelectionDAG &DAG) const {
12142 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
12143 "Should only be called for ISD::INSERT_VECTOR_ELT");
12144
12145 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
12146
12147 EVT VT = Op.getValueType();
12148 SDLoc dl(Op);
12149 SDValue V1 = Op.getOperand(i: 0);
12150 SDValue V2 = Op.getOperand(i: 1);
12151
12152 if (VT == MVT::v2f64 && C)
12153 return Op;
12154
12155 if (Subtarget.hasP9Vector()) {
12156 // A f32 load feeding into a v4f32 insert_vector_elt is handled in this way
12157 // because on P10, it allows this specific insert_vector_elt load pattern to
12158 // utilize the refactored load and store infrastructure in order to exploit
12159 // prefixed loads.
12160 // On targets with inexpensive direct moves (Power9 and up), a
12161 // (insert_vector_elt v4f32:$vec, (f32 load)) is always better as an integer
12162 // load since a single precision load will involve conversion to double
12163 // precision on the load followed by another conversion to single precision.
12164 if ((VT == MVT::v4f32) && (V2.getValueType() == MVT::f32) &&
12165 (isa<LoadSDNode>(Val: V2))) {
12166 SDValue BitcastVector = DAG.getBitcast(VT: MVT::v4i32, V: V1);
12167 SDValue BitcastLoad = DAG.getBitcast(VT: MVT::i32, V: V2);
12168 SDValue InsVecElt =
12169 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v4i32, N1: BitcastVector,
12170 N2: BitcastLoad, N3: Op.getOperand(i: 2));
12171 return DAG.getBitcast(VT: MVT::v4f32, V: InsVecElt);
12172 }
12173 }
12174
12175 if (Subtarget.isISA3_1()) {
12176 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && !Subtarget.isPPC64())
12177 return SDValue();
12178 // On P10, we have legal lowering for constant and variable indices for
12179 // all vectors.
12180 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
12181 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64)
12182 return Op;
12183 }
12184
12185 // Before P10, we have legal lowering for constant indices but not for
12186 // variable ones.
12187 if (!C)
12188 return SDValue();
12189
12190 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types.
12191 if (VT == MVT::v8i16 || VT == MVT::v16i8) {
12192 SDValue Mtvsrz = DAG.getNode(Opcode: PPCISD::MTVSRZ, DL: dl, VT, Operand: V2);
12193 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8;
12194 unsigned InsertAtElement = C->getZExtValue();
12195 unsigned InsertAtByte = InsertAtElement * BytesInEachElement;
12196 if (Subtarget.isLittleEndian()) {
12197 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte;
12198 }
12199 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT, N1: V1, N2: Mtvsrz,
12200 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
12201 }
12202 return Op;
12203}
12204
12205SDValue PPCTargetLowering::LowerDMFVectorLoad(SDValue Op,
12206 SelectionDAG &DAG) const {
12207 SDLoc dl(Op);
12208 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12209 SDValue LoadChain = LN->getChain();
12210 SDValue BasePtr = LN->getBasePtr();
12211 EVT VT = Op.getValueType();
12212 bool IsV1024i1 = VT == MVT::v1024i1;
12213 bool IsV2048i1 = VT == MVT::v2048i1;
12214
12215 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12216 // Dense Math dmr pair registers, respectively.
12217 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12218 (void)IsV2048i1;
12219 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12220 "Dense Math support required.");
12221 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12222
12223 SmallVector<SDValue, 8> Loads;
12224 SmallVector<SDValue, 8> LoadChains;
12225
12226 SDValue IntrinID = DAG.getConstant(Val: Intrinsic::ppc_vsx_lxvp, DL: dl, VT: MVT::i32);
12227 SDValue LoadOps[] = {LoadChain, IntrinID, BasePtr};
12228 MachineMemOperand *MMO = LN->getMemOperand();
12229 unsigned NumVecs = VT.getSizeInBits() / 256;
12230 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12231 MachineMemOperand *NewMMO =
12232 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12233 if (Idx > 0) {
12234 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12235 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12236 LoadOps[2] = BasePtr;
12237 }
12238 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
12239 VTList: DAG.getVTList(VT1: MVT::v256i1, VT2: MVT::Other),
12240 Ops: LoadOps, MemVT: MVT::v256i1, MMO: NewMMO);
12241 LoadChains.push_back(Elt: Ld.getValue(R: 1));
12242 Loads.push_back(Elt: Ld);
12243 }
12244
12245 if (Subtarget.isLittleEndian()) {
12246 std::reverse(first: Loads.begin(), last: Loads.end());
12247 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12248 }
12249
12250 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12251 SDValue Value = DMFInsert1024(Pairs: Loads, dl, DAG);
12252
12253 if (IsV1024i1) {
12254 return DAG.getMergeValues(Ops: {Value, TF}, dl);
12255 }
12256
12257 // Handle Loads for V2048i1 which represents a dmr pair.
12258 SmallVector<SDValue, 4> MoreLoads{Loads[4], Loads[5], Loads[6], Loads[7]};
12259 SDValue Dmr1Value = DMFInsert1024(Pairs: MoreLoads, dl, DAG);
12260
12261 SDValue Dmr0Sub = DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32);
12262 SDValue Dmr1Sub = DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32);
12263
12264 SDValue DmrPRC = DAG.getTargetConstant(Val: PPC::DMRpRCRegClassID, DL: dl, VT: MVT::i32);
12265 const SDValue DmrPOps[] = {DmrPRC, Value, Dmr0Sub, Dmr1Value, Dmr1Sub};
12266
12267 SDValue DmrPValue = SDValue(
12268 DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v2048i1, Ops: DmrPOps), 0);
12269
12270 return DAG.getMergeValues(Ops: {DmrPValue, TF}, dl);
12271}
12272
12273SDValue PPCTargetLowering::DMFInsert1024(const SmallVectorImpl<SDValue> &Pairs,
12274 const SDLoc &dl,
12275 SelectionDAG &DAG) const {
12276 SDValue Lo =
12277 DAG.getNode(Opcode: PPCISD::INST512, DL: dl, VT: MVT::v512i1, N1: Pairs[0], N2: Pairs[1]);
12278 SDValue LoSub = DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32);
12279 SDValue Hi =
12280 DAG.getNode(Opcode: PPCISD::INST512HI, DL: dl, VT: MVT::v512i1, N1: Pairs[2], N2: Pairs[3]);
12281 SDValue HiSub = DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32);
12282 SDValue RC = DAG.getTargetConstant(Val: PPC::DMRRCRegClassID, DL: dl, VT: MVT::i32);
12283
12284 return SDValue(DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v1024i1,
12285 Ops: {RC, Lo, LoSub, Hi, HiSub}),
12286 0);
12287}
12288
12289SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
12290 SelectionDAG &DAG) const {
12291 SDLoc dl(Op);
12292 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12293 SDValue LoadChain = LN->getChain();
12294 SDValue BasePtr = LN->getBasePtr();
12295 EVT VT = Op.getValueType();
12296
12297 if (VT == MVT::v1024i1 || VT == MVT::v2048i1)
12298 return LowerDMFVectorLoad(Op, DAG);
12299
12300 if (VT != MVT::v256i1 && VT != MVT::v512i1)
12301 return Op;
12302
12303 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12304 assert((VT != MVT::v512i1 || Subtarget.hasMMA()) &&
12305 "Type unsupported without MMA");
12306 assert((VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12307 "Type unsupported without paired vector support");
12308
12309 // For v256i1 on ISA Future, let the load go through to instruction selection
12310 // where it will be matched to lxvp/plxvp by the instruction patterns.
12311 if (VT == MVT::v256i1 && Subtarget.isISAFuture())
12312 return Op;
12313
12314 // For other cases, create 2 or 4 v16i8 loads to load the pair or accumulator
12315 // value in 2 or 4 vsx registers.
12316 Align Alignment = LN->getAlign();
12317 SmallVector<SDValue, 4> Loads;
12318 SmallVector<SDValue, 4> LoadChains;
12319 unsigned NumVecs = VT.getSizeInBits() / 128;
12320 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12321 SDValue Load =
12322 DAG.getLoad(VT: MVT::v16i8, dl, Chain: LoadChain, Ptr: BasePtr,
12323 PtrInfo: LN->getPointerInfo().getWithOffset(O: Idx * 16),
12324 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12325 MMOFlags: LN->getMemOperand()->getFlags(), Metadata: LN->getAAInfo());
12326 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12327 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12328 Loads.push_back(Elt: Load);
12329 LoadChains.push_back(Elt: Load.getValue(R: 1));
12330 }
12331 if (Subtarget.isLittleEndian()) {
12332 std::reverse(first: Loads.begin(), last: Loads.end());
12333 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12334 }
12335 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12336 SDValue Value =
12337 DAG.getNode(Opcode: VT == MVT::v512i1 ? PPCISD::ACC_BUILD : PPCISD::PAIR_BUILD,
12338 DL: dl, VT, Ops: Loads);
12339 SDValue RetOps[] = {Value, TF};
12340 return DAG.getMergeValues(Ops: RetOps, dl);
12341}
12342
12343SDValue PPCTargetLowering::LowerDMFVectorStore(SDValue Op,
12344 SelectionDAG &DAG) const {
12345
12346 SDLoc dl(Op);
12347 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12348 SDValue StoreChain = SN->getChain();
12349 SDValue BasePtr = SN->getBasePtr();
12350 SmallVector<SDValue, 8> Values;
12351 SmallVector<SDValue, 8> Stores;
12352 EVT VT = SN->getValue().getValueType();
12353 bool IsV1024i1 = VT == MVT::v1024i1;
12354 bool IsV2048i1 = VT == MVT::v2048i1;
12355
12356 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12357 // Dense Math dmr pair registers, respectively.
12358 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12359 (void)IsV2048i1;
12360 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12361 "Dense Math support required.");
12362 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12363
12364 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12365 if (IsV1024i1) {
12366 SDValue Lo(DAG.getMachineNode(
12367 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12368 Op1: Op.getOperand(i: 1),
12369 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12370 0);
12371 SDValue Hi(DAG.getMachineNode(
12372 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12373 Op1: Op.getOperand(i: 1),
12374 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12375 0);
12376 MachineSDNode *ExtNode =
12377 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Lo);
12378 Values.push_back(Elt: SDValue(ExtNode, 0));
12379 Values.push_back(Elt: SDValue(ExtNode, 1));
12380 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Hi);
12381 Values.push_back(Elt: SDValue(ExtNode, 0));
12382 Values.push_back(Elt: SDValue(ExtNode, 1));
12383 } else {
12384 // This corresponds to v2048i1 which represents a dmr pair.
12385 SDValue Dmr0(
12386 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12387 Op1: Op.getOperand(i: 1),
12388 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32)),
12389 0);
12390
12391 SDValue Dmr1(
12392 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12393 Op1: Op.getOperand(i: 1),
12394 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32)),
12395 0);
12396
12397 SDValue Dmr0Lo(DAG.getMachineNode(
12398 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12399 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12400 0);
12401
12402 SDValue Dmr0Hi(DAG.getMachineNode(
12403 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12404 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12405 0);
12406
12407 SDValue Dmr1Lo(DAG.getMachineNode(
12408 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12409 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12410 0);
12411
12412 SDValue Dmr1Hi(DAG.getMachineNode(
12413 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12414 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12415 0);
12416
12417 MachineSDNode *ExtNode =
12418 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr0Lo);
12419 Values.push_back(Elt: SDValue(ExtNode, 0));
12420 Values.push_back(Elt: SDValue(ExtNode, 1));
12421 ExtNode =
12422 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr0Hi);
12423 Values.push_back(Elt: SDValue(ExtNode, 0));
12424 Values.push_back(Elt: SDValue(ExtNode, 1));
12425 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr1Lo);
12426 Values.push_back(Elt: SDValue(ExtNode, 0));
12427 Values.push_back(Elt: SDValue(ExtNode, 1));
12428 ExtNode =
12429 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr1Hi);
12430 Values.push_back(Elt: SDValue(ExtNode, 0));
12431 Values.push_back(Elt: SDValue(ExtNode, 1));
12432 }
12433
12434 if (Subtarget.isLittleEndian())
12435 std::reverse(first: Values.begin(), last: Values.end());
12436
12437 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12438 SmallVector<SDValue, 4> Ops{
12439 StoreChain, DAG.getConstant(Val: Intrinsic::ppc_vsx_stxvp, DL: dl, VT: MVT::i32),
12440 Values[0], BasePtr};
12441 MachineMemOperand *MMO = SN->getMemOperand();
12442 unsigned NumVecs = VT.getSizeInBits() / 256;
12443 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12444 MachineMemOperand *NewMMO =
12445 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12446 if (Idx > 0) {
12447 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12448 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12449 Ops[3] = BasePtr;
12450 }
12451 Ops[2] = Values[Idx];
12452 SDValue St = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12453 MemVT: MVT::v256i1, MMO: NewMMO);
12454 Stores.push_back(Elt: St);
12455 }
12456
12457 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12458 return TF;
12459}
12460
12461SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
12462 SelectionDAG &DAG) const {
12463 SDLoc dl(Op);
12464 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12465 SDValue StoreChain = SN->getChain();
12466 SDValue BasePtr = SN->getBasePtr();
12467 SDValue Value = SN->getValue();
12468 SDValue Value2 = SN->getValue();
12469 EVT StoreVT = Value.getValueType();
12470
12471 if (StoreVT == MVT::v1024i1 || StoreVT == MVT::v2048i1)
12472 return LowerDMFVectorStore(Op, DAG);
12473
12474 if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1)
12475 return Op;
12476
12477 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12478 assert((StoreVT != MVT::v512i1 || Subtarget.hasMMA()) &&
12479 "Type unsupported without MMA");
12480 assert((StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12481 "Type unsupported without paired vector support");
12482
12483 // For v256i1 on ISA Future, let the store go through to instruction selection
12484 // where it will be matched to stxvp/pstxvp by the instruction patterns.
12485 if (StoreVT == MVT::v256i1 && Subtarget.isISAFuture() &&
12486 EnableAutoPairedVecSt)
12487 return Op;
12488
12489 // For other cases, create 2 or 4 v16i8 stores to store the pair or
12490 // accumulator underlying registers individually.
12491 Align Alignment = SN->getAlign();
12492 SmallVector<SDValue, 4> Stores;
12493 unsigned NumVecs = 2;
12494 if (StoreVT == MVT::v512i1) {
12495 if (Subtarget.isISAFuture()) {
12496 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12497 MachineSDNode *ExtNode = DAG.getMachineNode(
12498 Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Op.getOperand(i: 1));
12499
12500 Value = SDValue(ExtNode, 0);
12501 Value2 = SDValue(ExtNode, 1);
12502 } else
12503 Value = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: Value);
12504 NumVecs = 4;
12505 }
12506 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12507 unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx;
12508 SDValue Elt;
12509 if (Subtarget.isISAFuture()) {
12510 VecNum = Subtarget.isLittleEndian() ? 1 - (Idx % 2) : (Idx % 2);
12511 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
12512 N1: Idx > 1 ? Value2 : Value,
12513 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12514 } else
12515 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: Value,
12516 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12517
12518 SDValue Store =
12519 DAG.getStore(Chain: StoreChain, dl, Val: Elt, Ptr: BasePtr,
12520 PtrInfo: SN->getPointerInfo().getWithOffset(O: Idx * 16),
12521 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12522 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
12523 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12524 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12525 Stores.push_back(Elt: Store);
12526 }
12527 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12528 return TF;
12529}
12530
12531SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
12532 SDLoc dl(Op);
12533 if (Op.getValueType() == MVT::v4i32) {
12534 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12535
12536 SDValue Zero = getCanonicalConstSplat(Val: 0, SplatSize: 1, VT: MVT::v4i32, DAG, dl);
12537 // +16 as shift amt.
12538 SDValue Neg16 = getCanonicalConstSplat(Val: -16, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
12539 SDValue RHSSwap = // = vrlw RHS, 16
12540 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vrlw, LHS: RHS, RHS: Neg16, DAG, dl);
12541
12542 // Shrinkify inputs to v8i16.
12543 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: LHS);
12544 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHS);
12545 RHSSwap = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHSSwap);
12546
12547 // Low parts multiplied together, generating 32-bit results (we ignore the
12548 // top parts).
12549 SDValue LoProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmulouh,
12550 LHS, RHS, DAG, dl, DestVT: MVT::v4i32);
12551
12552 SDValue HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmsumuhm,
12553 Op0: LHS, Op1: RHSSwap, Op2: Zero, DAG, dl, DestVT: MVT::v4i32);
12554 // Shift the high parts up 16 bits.
12555 HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: HiProd,
12556 RHS: Neg16, DAG, dl);
12557 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: LoProd, N2: HiProd);
12558 } else if (Op.getValueType() == MVT::v16i8) {
12559 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12560 bool isLittleEndian = Subtarget.isLittleEndian();
12561
12562 // Multiply the even 8-bit parts, producing 16-bit sums.
12563 SDValue EvenParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuleub,
12564 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12565 EvenParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: EvenParts);
12566
12567 // Multiply the odd 8-bit parts, producing 16-bit sums.
12568 SDValue OddParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuloub,
12569 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12570 OddParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OddParts);
12571
12572 // Merge the results together. Because vmuleub and vmuloub are
12573 // instructions with a big-endian bias, we must reverse the
12574 // element numbering and reverse the meaning of "odd" and "even"
12575 // when generating little endian code.
12576 int Ops[16];
12577 for (unsigned i = 0; i != 8; ++i) {
12578 if (isLittleEndian) {
12579 Ops[i*2 ] = 2*i;
12580 Ops[i*2+1] = 2*i+16;
12581 } else {
12582 Ops[i*2 ] = 2*i+1;
12583 Ops[i*2+1] = 2*i+1+16;
12584 }
12585 }
12586 if (isLittleEndian)
12587 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OddParts, N2: EvenParts, Mask: Ops);
12588 else
12589 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: EvenParts, N2: OddParts, Mask: Ops);
12590 } else {
12591 llvm_unreachable("Unknown mul to lower!");
12592 }
12593}
12594
12595SDValue PPCTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
12596 bool IsStrict = Op->isStrictFPOpcode();
12597 if (Op.getOperand(i: IsStrict ? 1 : 0).getValueType() == MVT::f128 &&
12598 !Subtarget.hasP9Vector())
12599 return SDValue();
12600
12601 return Op;
12602}
12603
12604// Custom lowering for fpext vf32 to v2f64
12605SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
12606
12607 assert(Op.getOpcode() == ISD::FP_EXTEND &&
12608 "Should only be called for ISD::FP_EXTEND");
12609
12610 // FIXME: handle extends from half precision float vectors on P9.
12611 // We only want to custom lower an extend from v2f32 to v2f64.
12612 if (Op.getValueType() != MVT::v2f64 ||
12613 Op.getOperand(i: 0).getValueType() != MVT::v2f32)
12614 return SDValue();
12615
12616 SDLoc dl(Op);
12617 SDValue Op0 = Op.getOperand(i: 0);
12618
12619 switch (Op0.getOpcode()) {
12620 default:
12621 return SDValue();
12622 case ISD::EXTRACT_SUBVECTOR: {
12623 assert(Op0.getNumOperands() == 2 &&
12624 isa<ConstantSDNode>(Op0->getOperand(1)) &&
12625 "Node should have 2 operands with second one being a constant!");
12626
12627 if (Op0.getOperand(i: 0).getValueType() != MVT::v4f32)
12628 return SDValue();
12629
12630 // Custom lower is only done for high or low doubleword.
12631 int Idx = Op0.getConstantOperandVal(i: 1);
12632 if (Idx % 2 != 0)
12633 return SDValue();
12634
12635 // Since input is v4f32, at this point Idx is either 0 or 2.
12636 // Shift to get the doubleword position we want.
12637 int DWord = Idx >> 1;
12638
12639 // High and low word positions are different on little endian.
12640 if (Subtarget.isLittleEndian())
12641 DWord ^= 0x1;
12642
12643 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64,
12644 N1: Op0.getOperand(i: 0), N2: DAG.getConstant(Val: DWord, DL: dl, VT: MVT::i32));
12645 }
12646 case ISD::FADD:
12647 case ISD::FMUL:
12648 case ISD::FSUB: {
12649 SDValue NewLoad[2];
12650 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) {
12651 // Ensure both input are loads.
12652 SDValue LdOp = Op0.getOperand(i);
12653 if (LdOp.getOpcode() != ISD::LOAD)
12654 return SDValue();
12655 // Generate new load node.
12656 LoadSDNode *LD = cast<LoadSDNode>(Val&: LdOp);
12657 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12658 NewLoad[i] = DAG.getMemIntrinsicNode(
12659 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12660 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12661 }
12662 SDValue NewOp =
12663 DAG.getNode(Opcode: Op0.getOpcode(), DL: SDLoc(Op0), VT: MVT::v4f32, N1: NewLoad[0],
12664 N2: NewLoad[1], Flags: Op0.getNode()->getFlags());
12665 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewOp,
12666 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12667 }
12668 case ISD::LOAD: {
12669 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op0);
12670 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12671 SDValue NewLd = DAG.getMemIntrinsicNode(
12672 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12673 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12674 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewLd,
12675 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12676 }
12677 }
12678 llvm_unreachable("ERROR:Should return for all cases within swtich.");
12679}
12680
12681static SDValue ConvertCarryValueToCarryFlag(EVT SumType, SDValue Value,
12682 SelectionDAG &DAG,
12683 const PPCSubtarget &STI) {
12684 SDLoc DL(Value);
12685 if (STI.useCRBits())
12686 Value = DAG.getNode(Opcode: ISD::SELECT, DL, VT: SumType, N1: Value,
12687 N2: DAG.getConstant(Val: 1, DL, VT: SumType),
12688 N3: DAG.getConstant(Val: 0, DL, VT: SumType));
12689 else
12690 Value = DAG.getZExtOrTrunc(Op: Value, DL, VT: SumType);
12691 SDValue Sum = DAG.getNode(Opcode: PPCISD::ADDC, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32),
12692 N1: Value, N2: DAG.getAllOnesConstant(DL, VT: SumType));
12693 return Sum.getValue(R: 1);
12694}
12695
12696static SDValue ConvertCarryFlagToCarryValue(EVT SumType, SDValue Flag,
12697 EVT CarryType, SelectionDAG &DAG,
12698 const PPCSubtarget &STI) {
12699 SDLoc DL(Flag);
12700 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SumType);
12701 SDValue Carry = DAG.getNode(
12702 Opcode: PPCISD::ADDE, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32), N1: Zero, N2: Zero, N3: Flag);
12703 if (STI.useCRBits())
12704 return DAG.getSetCC(DL, VT: CarryType, LHS: Carry, RHS: Zero, Cond: ISD::SETNE);
12705 return DAG.getZExtOrTrunc(Op: Carry, DL, VT: CarryType);
12706}
12707
12708SDValue PPCTargetLowering::LowerADDSUBO(SDValue Op, SelectionDAG &DAG) const {
12709
12710 SDLoc DL(Op);
12711 SDNode *N = Op.getNode();
12712 EVT VT = N->getValueType(ResNo: 0);
12713 EVT CarryType = N->getValueType(ResNo: 1);
12714 unsigned Opc = N->getOpcode();
12715 bool IsAdd = Opc == ISD::UADDO;
12716 Opc = IsAdd ? PPCISD::ADDC : PPCISD::SUBC;
12717 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12718 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
12719 SDValue Carry = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType,
12720 DAG, STI: Subtarget);
12721 if (!IsAdd)
12722 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Carry,
12723 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
12724 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: Carry);
12725}
12726
12727SDValue PPCTargetLowering::LowerADDSUBO_CARRY(SDValue Op,
12728 SelectionDAG &DAG) const {
12729 SDLoc DL(Op);
12730 SDNode *N = Op.getNode();
12731 unsigned Opc = N->getOpcode();
12732 EVT VT = N->getValueType(ResNo: 0);
12733 EVT CarryType = N->getValueType(ResNo: 1);
12734 SDValue CarryOp = N->getOperand(Num: 2);
12735 bool IsAdd = Opc == ISD::UADDO_CARRY;
12736 Opc = IsAdd ? PPCISD::ADDE : PPCISD::SUBE;
12737 if (!IsAdd)
12738 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12739 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12740 CarryOp = ConvertCarryValueToCarryFlag(SumType: VT, Value: CarryOp, DAG, STI: Subtarget);
12741 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12742 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: CarryOp);
12743 CarryOp = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType, DAG,
12744 STI: Subtarget);
12745 if (!IsAdd)
12746 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12747 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12748 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: CarryOp);
12749}
12750
12751SDValue PPCTargetLowering::LowerSSUBO(SDValue Op, SelectionDAG &DAG) const {
12752
12753 SDLoc dl(Op);
12754 SDValue LHS = Op.getOperand(i: 0);
12755 SDValue RHS = Op.getOperand(i: 1);
12756 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12757
12758 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
12759
12760 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: RHS, N2: LHS);
12761 SDValue Xor2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sub, N2: LHS);
12762
12763 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Xor1, N2: Xor2);
12764
12765 SDValue Overflow =
12766 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: And,
12767 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12768
12769 SDValue OverflowTrunc =
12770 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12771
12772 return DAG.getMergeValues(Ops: {Sub, OverflowTrunc}, dl);
12773}
12774
12775/// Implements signed add with overflow detection using the rule:
12776/// (x eqv y) & (sum xor x), where the overflow bit is extracted from the sign
12777SDValue PPCTargetLowering::LowerSADDO(SDValue Op, SelectionDAG &DAG) const {
12778
12779 SDLoc dl(Op);
12780 SDValue LHS = Op.getOperand(i: 0);
12781 SDValue RHS = Op.getOperand(i: 1);
12782 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12783
12784 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
12785
12786 // Compute ~(x xor y)
12787 SDValue XorXY = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
12788 SDValue EqvXY = DAG.getNOT(DL: dl, Val: XorXY, VT);
12789 // Compute (s xor x)
12790 SDValue SumXorX = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sum, N2: LHS);
12791
12792 // overflow = (x eqv y) & (s xor x)
12793 SDValue OverflowInSign = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: EqvXY, N2: SumXorX);
12794
12795 // Shift sign bit down to LSB
12796 SDValue Overflow =
12797 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: OverflowInSign,
12798 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12799 // Truncate to the overflow type (i1)
12800 SDValue OverflowTrunc =
12801 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12802
12803 return DAG.getMergeValues(Ops: {Sum, OverflowTrunc}, dl);
12804}
12805
12806/// Lower ABDU with negation pattern using branchless carry arithmetic.
12807/// Recognizes: abdu(a, sub(0, x)) and transforms to:
12808/// a - (0 - x) = a + x (mod 2^n)
12809/// Uses SUBC to compute result without branches.
12810SDValue PPCTargetLowering::LowerABDU(SDValue Op, SelectionDAG &DAG) const {
12811 SDLoc DL(Op);
12812 SDValue LHS = Op.getOperand(i: 0);
12813 SDValue RHS = Op.getOperand(i: 1);
12814 EVT OpVT = LHS.getValueType();
12815 EVT VT = Op.getValueType();
12816 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
12817
12818 // If the subtract doesn't overflow then just use abs(sub()).
12819 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: LHS, N1: RHS))
12820 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12821 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: RHS));
12822
12823 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: RHS, N1: LHS))
12824 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12825 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: RHS, N2: LHS));
12826
12827 // General path: use SUBC (or ADDC when RHS is 0-X) to get
12828 // subtract-with-flags, then CMOV to select a-b or b-a. ADDC/SUBC produce the
12829 // flags we need.
12830 unsigned Opcode = PPCISD::SUBC;
12831
12832 // Check if RHS is a negation (0 - X). If so, we can use ADDC instead of SUBC:
12833 // a - (0 - x) = a + x (mod 2^n)
12834 // Same semantics as in LowerCMP; apply same safety checks.
12835 if (RHS.getOpcode() == ISD::SUB) {
12836 SDValue SubLHS = RHS.getOperand(i: 0);
12837 SDValue SubRHS = RHS.getOperand(i: 1);
12838
12839 if (isNullConstant(V: SubLHS) && DAG.isKnownNeverZero(Op: SubRHS)) {
12840 Opcode = PPCISD::ADDC;
12841 RHS = SubRHS;
12842 }
12843 }
12844
12845 // On PPC64, carry ops use the full 64-bit register. Operands are type-legal
12846 // i32 here; widen only for the carry path (fast abs(sub) stays at VT).
12847 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12848 LHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: LHS);
12849 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: RHS);
12850 OpVT = MVT::i64;
12851 }
12852
12853 // Calculate LHS - RHS and capture the carry (CA)
12854 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12855 SDValue Res = DAG.getNode(Opcode, DL, VTList: VTs, N1: LHS, N2: RHS);
12856 SDValue CA0 = Res.getValue(R: 1);
12857
12858 // t2 = A - B + CA0 using SUBE.
12859 SDValue ZeroOrNeg1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Res, N2: Res, N3: CA0);
12860
12861 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: OpVT, N1: Res, N2: ZeroOrNeg1);
12862
12863 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: Xor, N2: ZeroOrNeg1);
12864
12865 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
12866 return Res;
12867}
12868
12869// Lower unsigned 3-way compare producing -1/0/1.
12870SDValue PPCTargetLowering::LowerUCMP(SDValue Op, SelectionDAG &DAG) const {
12871 SDLoc DL(Op);
12872 SDValue A = DAG.getFreeze(V: Op.getOperand(i: 0));
12873 SDValue B = DAG.getFreeze(V: Op.getOperand(i: 1));
12874 EVT OpVT = A.getValueType();
12875 EVT ResVT = Op.getValueType();
12876
12877 // On PPC64, i32 carries are affected by the upper 32 bits of the registers.
12878 // We must zero-extend to i64 to ensure the carry reflects the 32-bit unsigned
12879 // comparison.
12880 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12881 A = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: A);
12882 B = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: B);
12883 OpVT = MVT::i64;
12884 }
12885
12886 // First compute diff = A - B.
12887 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: A, N2: B);
12888
12889 // Generate B - A using SUBC to capture carry.
12890 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12891 SDValue SubC = DAG.getNode(Opcode: PPCISD::SUBC, DL, VTList: VTs, N1: B, N2: A);
12892 SDValue CA0 = SubC.getValue(R: 1);
12893
12894 // t2 = A - B + CA0 using SUBE.
12895 SDValue SubE1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: A, N2: B, N3: CA0);
12896 SDValue CA1 = SubE1.getValue(R: 1);
12897
12898 // res = diff - t2 + CA1 using SUBE (produces desired -1/0/1).
12899 SDValue ResPair = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Diff, N2: SubE1, N3: CA1);
12900
12901 // Extract the first result and truncate to result type if needed.
12902 return DAG.getSExtOrTrunc(Op: ResPair.getValue(R: 0), DL, VT: ResVT);
12903}
12904
12905// Lower signed 3-way compare producing -1/0/1.
12906// For PowerPC with SETB (P9+), use the default expansion.
12907// For pre-P9, we generate a custom sequence using comparison and bit
12908// extraction.
12909// cmpw cr7, LHS, RHS
12910// mfocrf r, CR7
12911// rlwinm LT, r, 29, 31, 31 (extract LT bit from CR7)
12912// rlwinm GT, r, 30, 31, 31 (extract GT bit from CR7)
12913// subf result, LT, GT (GT - LT = -1/0/1)
12914SDValue PPCTargetLowering::LowerSCMP(SDValue Op, SelectionDAG &DAG) const {
12915 SDLoc DL(Op);
12916 SDValue LHS = Op.getOperand(i: 0);
12917 SDValue RHS = Op.getOperand(i: 1);
12918 EVT VT = LHS.getValueType();
12919 EVT ResVT = Op.getValueType();
12920
12921 // For P9+, use the default expansion which will use SETB instruction
12922 if (Subtarget.isISA3_0())
12923 return SDValue();
12924
12925 // For pre-P9, generate custom sequence without SETB
12926 bool Is64BitCmp = (VT == MVT::i64);
12927
12928 // On PPC64, always use 64-bit operations to avoid extra sign-extension
12929 // Even if result type is i32, it will likely be extended to i64 by calling
12930 // convention
12931 bool Use64BitOps = Subtarget.isPPC64();
12932
12933 // Perform comparison
12934 unsigned CmpOpc = Is64BitCmp ? PPC::CMPD : PPC::CMPW;
12935 SDValue Cmp = SDValue(DAG.getMachineNode(Opcode: CmpOpc, dl: DL, VT: MVT::i32, Op1: LHS, Op2: RHS), 0);
12936
12937 // Move comparison result to CR7 to have known bit positions
12938 SDValue CR7Reg = DAG.getRegister(Reg: PPC::CR7, VT: MVT::i32);
12939 SDValue InGlue; // Null incoming glue
12940 SDValue CopyToReg =
12941 DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: DL, Reg: CR7Reg, N: Cmp, Glue: InGlue);
12942 SDValue Glue = CopyToReg.getValue(R: 1);
12943
12944 // Use MFOCRF to read CR7
12945 unsigned MFOCRFOpc = Use64BitOps ? PPC::MFOCRF8 : PPC::MFOCRF;
12946 EVT MFVT = Use64BitOps ? MVT::i64 : MVT::i32;
12947 SDValue MFOCRF =
12948 SDValue(DAG.getMachineNode(Opcode: MFOCRFOpc, dl: DL, VT: MFVT, Op1: CR7Reg, Op2: Glue), 0);
12949
12950 // CR7 bits in the result (after MFOCRF):
12951 // Bit 28: LT
12952 // Bit 29: GT
12953 // Bit 30: EQ
12954 // Bit 31: SO
12955
12956 // Extract LT and GT bits using RLWINM
12957 unsigned RLWinmOpc = Use64BitOps ? PPC::RLWINM8 : PPC::RLWINM;
12958
12959 // Extract LT bit (Bit 28 -> Rotate Left 29 to move to bit 31, then mask)
12960 SDValue LTOps[] = {MFOCRF, DAG.getTargetConstant(Val: 29, DL, VT: MVT::i32),
12961 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32),
12962 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32)};
12963 SDValue LTBit = SDValue(DAG.getMachineNode(Opcode: RLWinmOpc, dl: DL, VT: MFVT, Ops: LTOps), 0);
12964
12965 // Extract GT bit (Bit 29 -> Rotate Left 30 to move to bit 31, then mask)
12966 SDValue GTOps[] = {MFOCRF, DAG.getTargetConstant(Val: 30, DL, VT: MVT::i32),
12967 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32),
12968 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32)};
12969 SDValue GTBit = SDValue(DAG.getMachineNode(Opcode: RLWinmOpc, dl: DL, VT: MFVT, Ops: GTOps), 0);
12970
12971 // Compute result: GT - LT
12972 // If LT: 0 - 1 = -1
12973 // If GT: 1 - 0 = 1
12974 // If EQ: 0 - 0 = 0
12975 unsigned SubOpc = Use64BitOps ? PPC::SUBF8 : PPC::SUBF;
12976 SDValue Result =
12977 SDValue(DAG.getMachineNode(Opcode: SubOpc, dl: DL, VT: MFVT, Op1: LTBit, Op2: GTBit), 0);
12978
12979 if (ResVT == MFVT)
12980 return Result;
12981
12982 // Adjust result to match expected type (sign-extend or truncate as needed)
12983 return DAG.getSExtOrTrunc(Op: Result, DL, VT: ResVT);
12984}
12985
12986/// LowerOperation - Provide custom lowering hooks for some operations.
12987///
12988SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
12989 switch (Op.getOpcode()) {
12990 default:
12991 llvm_unreachable("Wasn't expecting to be able to lower this!");
12992 case ISD::FPOW: return lowerPow(Op, DAG);
12993 case ISD::FSIN: return lowerSin(Op, DAG);
12994 case ISD::FCOS: return lowerCos(Op, DAG);
12995 case ISD::FLOG: return lowerLog(Op, DAG);
12996 case ISD::FLOG10: return lowerLog10(Op, DAG);
12997 case ISD::FEXP: return lowerExp(Op, DAG);
12998 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
12999 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
13000 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
13001 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
13002 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
13003 case ISD::STRICT_FSETCC:
13004 case ISD::STRICT_FSETCCS:
13005 case ISD::SETCC: return LowerSETCC(Op, DAG);
13006 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
13007 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
13008 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
13009 case ISD::SSUBO:
13010 return LowerSSUBO(Op, DAG);
13011 case ISD::SADDO:
13012 return LowerSADDO(Op, DAG);
13013
13014 case ISD::INLINEASM:
13015 case ISD::INLINEASM_BR: return LowerINLINEASM(Op, DAG);
13016 // Variable argument lowering.
13017 case ISD::VASTART: return LowerVASTART(Op, DAG);
13018 case ISD::VAARG: return LowerVAARG(Op, DAG);
13019 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
13020
13021 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG);
13022 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
13023 case ISD::GET_DYNAMIC_AREA_OFFSET:
13024 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
13025
13026 // Exception handling lowering.
13027 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG);
13028 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
13029 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
13030
13031 case ISD::LOAD: return LowerLOAD(Op, DAG);
13032 case ISD::STORE: return LowerSTORE(Op, DAG);
13033 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
13034 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
13035 case ISD::STRICT_FP_TO_UINT:
13036 case ISD::STRICT_FP_TO_SINT:
13037 case ISD::FP_TO_UINT:
13038 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, dl: SDLoc(Op));
13039 case ISD::STRICT_UINT_TO_FP:
13040 case ISD::STRICT_SINT_TO_FP:
13041 case ISD::UINT_TO_FP:
13042 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
13043 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
13044 case ISD::SET_ROUNDING:
13045 return LowerSET_ROUNDING(Op, DAG);
13046
13047 // Lower 64-bit shifts.
13048 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG);
13049 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG);
13050 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG);
13051
13052 case ISD::FSHL: return LowerFunnelShift(Op, DAG);
13053 case ISD::FSHR: return LowerFunnelShift(Op, DAG);
13054
13055 // Vector-related lowering.
13056 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
13057 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
13058 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
13059 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
13060 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
13061 case ISD::MUL: return LowerMUL(Op, DAG);
13062 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
13063 case ISD::STRICT_FP_ROUND:
13064 case ISD::FP_ROUND:
13065 return LowerFP_ROUND(Op, DAG);
13066 case ISD::ROTL: return LowerROTL(Op, DAG);
13067
13068 // For counter-based loop handling.
13069 case ISD::INTRINSIC_W_CHAIN:
13070 return SDValue();
13071
13072 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
13073
13074 // Frame & Return address.
13075 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
13076 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
13077
13078 case ISD::INTRINSIC_VOID:
13079 return LowerINTRINSIC_VOID(Op, DAG);
13080 case ISD::BSWAP:
13081 return LowerBSWAP(Op, DAG);
13082 case ISD::ATOMIC_CMP_SWAP:
13083 return LowerATOMIC_CMP_SWAP(Op, DAG);
13084 case ISD::ATOMIC_STORE:
13085 return LowerATOMIC_LOAD_STORE(Op, DAG);
13086 case ISD::IS_FPCLASS:
13087 return LowerIS_FPCLASS(Op, DAG);
13088 case ISD::UADDO:
13089 case ISD::USUBO:
13090 return LowerADDSUBO(Op, DAG);
13091 case ISD::UADDO_CARRY:
13092 case ISD::USUBO_CARRY:
13093 return LowerADDSUBO_CARRY(Op, DAG);
13094 case ISD::UCMP:
13095 return LowerUCMP(Op, DAG);
13096 case ISD::ABDU:
13097 return LowerABDU(Op, DAG);
13098 case ISD::SCMP:
13099 return LowerSCMP(Op, DAG);
13100 case ISD::STRICT_LRINT:
13101 case ISD::STRICT_LLRINT:
13102 case ISD::STRICT_LROUND:
13103 case ISD::STRICT_LLROUND:
13104 case ISD::STRICT_FNEARBYINT:
13105 if (Op->getFlags().hasNoFPExcept())
13106 return Op;
13107 return SDValue();
13108 case ISD::VP_LOAD:
13109 return LowerVP_LOAD(Op, DAG);
13110 case ISD::VP_STORE:
13111 return LowerVP_STORE(Op, DAG);
13112 case ISD::PARTIAL_REDUCE_UMLA:
13113 return LowerPartialReduce(Op, DAG);
13114 }
13115}
13116
13117void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
13118 SmallVectorImpl<SDValue>&Results,
13119 SelectionDAG &DAG) const {
13120 SDLoc dl(N);
13121 switch (N->getOpcode()) {
13122 default:
13123 llvm_unreachable("Do not know how to custom type legalize this operation!");
13124 case ISD::ATOMIC_LOAD: {
13125 SDValue Res = LowerATOMIC_LOAD_STORE(Op: SDValue(N, 0), DAG);
13126 Results.push_back(Elt: Res);
13127 Results.push_back(Elt: Res.getValue(R: 1));
13128 break;
13129 }
13130 case ISD::READCYCLECOUNTER: {
13131 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other);
13132 SDValue RTB = DAG.getNode(Opcode: PPCISD::READ_TIME_BASE, DL: dl, VTList: VTs, N: N->getOperand(Num: 0));
13133
13134 Results.push_back(
13135 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: RTB, N2: RTB.getValue(R: 1)));
13136 Results.push_back(Elt: RTB.getValue(R: 2));
13137 break;
13138 }
13139 case ISD::INTRINSIC_W_CHAIN: {
13140 if (N->getConstantOperandVal(Num: 1) != Intrinsic::loop_decrement)
13141 break;
13142
13143 assert(N->getValueType(0) == MVT::i1 &&
13144 "Unexpected result type for CTR decrement intrinsic");
13145 EVT SVT = getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(),
13146 VT: N->getValueType(ResNo: 0));
13147 SDVTList VTs = DAG.getVTList(VT1: SVT, VT2: MVT::Other);
13148 SDValue NewInt = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: VTs, N1: N->getOperand(Num: 0),
13149 N2: N->getOperand(Num: 1));
13150
13151 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewInt));
13152 Results.push_back(Elt: NewInt.getValue(R: 1));
13153 break;
13154 }
13155 case ISD::INTRINSIC_WO_CHAIN: {
13156 switch (N->getConstantOperandVal(Num: 0)) {
13157 case Intrinsic::ppc_pack_longdouble:
13158 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::ppcf128,
13159 N1: N->getOperand(Num: 2), N2: N->getOperand(Num: 1)));
13160 break;
13161 case Intrinsic::ppc_maxfe:
13162 case Intrinsic::ppc_minfe:
13163 case Intrinsic::ppc_fnmsub:
13164 case Intrinsic::ppc_convert_f128_to_ppcf128:
13165 Results.push_back(Elt: LowerINTRINSIC_WO_CHAIN(Op: SDValue(N, 0), DAG));
13166 break;
13167 }
13168 break;
13169 }
13170 case ISD::VAARG: {
13171 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64())
13172 return;
13173
13174 EVT VT = N->getValueType(ResNo: 0);
13175
13176 if (VT == MVT::i64) {
13177 SDValue NewNode = LowerVAARG(Op: SDValue(N, 1), DAG);
13178
13179 Results.push_back(Elt: NewNode);
13180 Results.push_back(Elt: NewNode.getValue(R: 1));
13181 }
13182 return;
13183 }
13184 case ISD::STRICT_FP_TO_SINT:
13185 case ISD::STRICT_FP_TO_UINT:
13186 case ISD::FP_TO_SINT:
13187 case ISD::FP_TO_UINT: {
13188 // LowerFP_TO_INT() can only handle f32 and f64.
13189 if (N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType() ==
13190 MVT::ppcf128)
13191 return;
13192 SDValue LoweredValue = LowerFP_TO_INT(Op: SDValue(N, 0), DAG, dl);
13193 Results.push_back(Elt: LoweredValue);
13194 if (N->isStrictFPOpcode())
13195 Results.push_back(Elt: LoweredValue.getValue(R: 1));
13196 return;
13197 }
13198 case ISD::TRUNCATE: {
13199 if (!N->getValueType(ResNo: 0).isVector())
13200 return;
13201 SDValue Lowered = LowerTRUNCATEVector(Op: SDValue(N, 0), DAG);
13202 if (Lowered)
13203 Results.push_back(Elt: Lowered);
13204 return;
13205 }
13206 case ISD::SCALAR_TO_VECTOR: {
13207 SDValue Lowered = LowerSCALAR_TO_VECTOR(Op: SDValue(N, 0), DAG);
13208 if (Lowered)
13209 Results.push_back(Elt: Lowered);
13210 return;
13211 }
13212 case ISD::FSHL:
13213 case ISD::FSHR:
13214 // Don't handle funnel shifts here.
13215 return;
13216 case ISD::BITCAST:
13217 // Don't handle bitcast here.
13218 return;
13219 case ISD::FP_EXTEND:
13220 SDValue Lowered = LowerFP_EXTEND(Op: SDValue(N, 0), DAG);
13221 if (Lowered)
13222 Results.push_back(Elt: Lowered);
13223 return;
13224 }
13225}
13226
13227//===----------------------------------------------------------------------===//
13228// Other Lowering Code
13229//===----------------------------------------------------------------------===//
13230
13231static CallInst *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
13232 return Builder.CreateIntrinsicWithoutFolding(ID: Id, Args: {});
13233}
13234
13235Value *PPCTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
13236 Value *Addr,
13237 AtomicOrdering Ord) const {
13238 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
13239
13240 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13241 "Only 8/16/32/64-bit atomic loads supported");
13242 Intrinsic::ID IntID;
13243 switch (SZ) {
13244 default:
13245 llvm_unreachable("Unexpected PrimitiveSize");
13246 case 8:
13247 IntID = Intrinsic::ppc_lbarx;
13248 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13249 break;
13250 case 16:
13251 IntID = Intrinsic::ppc_lharx;
13252 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13253 break;
13254 case 32:
13255 IntID = Intrinsic::ppc_lwarx;
13256 break;
13257 case 64:
13258 IntID = Intrinsic::ppc_ldarx;
13259 break;
13260 }
13261 Value *Call =
13262 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
13263
13264 return Builder.CreateTruncOrBitCast(V: Call, DestTy: ValueTy);
13265}
13266
13267// Perform a store-conditional operation to Addr. Return the status of the
13268// store. This should be 0 if the store succeeded, non-zero otherwise.
13269Value *PPCTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
13270 Value *Val, Value *Addr,
13271 AtomicOrdering Ord) const {
13272 Type *Ty = Val->getType();
13273 unsigned SZ = Ty->getPrimitiveSizeInBits();
13274
13275 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13276 "Only 8/16/32/64-bit atomic loads supported");
13277 Intrinsic::ID IntID;
13278 switch (SZ) {
13279 default:
13280 llvm_unreachable("Unexpected PrimitiveSize");
13281 case 8:
13282 IntID = Intrinsic::ppc_stbcx;
13283 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13284 break;
13285 case 16:
13286 IntID = Intrinsic::ppc_sthcx;
13287 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13288 break;
13289 case 32:
13290 IntID = Intrinsic::ppc_stwcx;
13291 break;
13292 case 64:
13293 IntID = Intrinsic::ppc_stdcx;
13294 break;
13295 }
13296
13297 if (SZ == 8 || SZ == 16)
13298 Val = Builder.CreateZExt(V: Val, DestTy: Builder.getInt32Ty());
13299
13300 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
13301 /*FMFSource=*/nullptr, Name: "stcx");
13302 return Builder.CreateXor(LHS: Call, RHS: Builder.getInt32(C: 1));
13303}
13304
13305// The mappings for emitLeading/TrailingFence is taken from
13306// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
13307Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
13308 Instruction *Inst,
13309 AtomicOrdering Ord) const {
13310 if (Ord == AtomicOrdering::SequentiallyConsistent)
13311 return callIntrinsic(Builder, Id: Intrinsic::ppc_sync);
13312 if (isReleaseOrStronger(AO: Ord))
13313 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13314 return nullptr;
13315}
13316
13317Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
13318 Instruction *Inst,
13319 AtomicOrdering Ord) const {
13320 if (Inst->hasAtomicLoad() && isAcquireOrStronger(AO: Ord)) {
13321 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and
13322 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
13323 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
13324 if (isa<LoadInst>(Val: Inst))
13325 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::ppc_cfence,
13326 OverloadTypes: {Inst->getType()}, Args: {Inst});
13327 // FIXME: Can use isync for rmw operation.
13328 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13329 }
13330 return nullptr;
13331}
13332
13333MachineBasicBlock *PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI,
13334 MachineBasicBlock *BB,
13335 unsigned BinOpcode,
13336 unsigned CmpOpcode,
13337 unsigned CmpPred) const {
13338 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13339 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13340 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13341 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13342 unsigned AtomicSize = MI.getOperand(i: 3).getImm();
13343
13344 auto LoadMnemonic = PPC::LDARX;
13345 auto StoreMnemonic = PPC::STDCX;
13346 switch (AtomicSize) {
13347 default:
13348 llvm_unreachable("Unexpected size of atomic entity");
13349 case 1:
13350 LoadMnemonic = PPC::LBARX;
13351 StoreMnemonic = PPC::STBCX;
13352 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13353 break;
13354 case 2:
13355 LoadMnemonic = PPC::LHARX;
13356 StoreMnemonic = PPC::STHCX;
13357 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13358 break;
13359 case 4:
13360 LoadMnemonic = PPC::LWARX;
13361 StoreMnemonic = PPC::STWCX;
13362 break;
13363 case 8:
13364 LoadMnemonic = PPC::LDARX;
13365 StoreMnemonic = PPC::STDCX;
13366 break;
13367 }
13368
13369 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13370 MachineFunction *F = BB->getParent();
13371 MachineFunction::iterator It = ++BB->getIterator();
13372
13373 if (CmpOpcode == PPC::CMPW && (AtomicSize == 1 || AtomicSize == 2))
13374 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, /*IsByte=*/AtomicSize == 1, TII);
13375
13376 Register dest = MI.getOperand(i: 0).getReg();
13377 Register ptrA = MI.getOperand(i: 1).getReg();
13378 Register ptrB = MI.getOperand(i: 2).getReg();
13379 Register incr = MI.getOperand(i: 4).getReg();
13380 DebugLoc dl = MI.getDebugLoc();
13381
13382 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13383 MachineBasicBlock *loop2MBB =
13384 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13385 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13386 F->insert(MBBI: It, MBB: loopMBB);
13387 if (CmpOpcode)
13388 F->insert(MBBI: It, MBB: loop2MBB);
13389 F->insert(MBBI: It, MBB: exitMBB);
13390 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13391 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13392 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13393
13394 MachineRegisterInfo &RegInfo = F->getRegInfo();
13395 Register TmpReg = (!BinOpcode) ? incr :
13396 RegInfo.createVirtualRegister( RegClass: AtomicSize == 8 ? &PPC::G8RCRegClass
13397 : &PPC::GPRCRegClass);
13398
13399 // thisMBB:
13400 // ...
13401 // fallthrough --> loopMBB
13402 BB->addSuccessor(Succ: loopMBB);
13403
13404 // loopMBB:
13405 // l[wd]arx dest, ptr
13406 // add r0, dest, incr
13407 // st[wd]cx. r0, ptr
13408 // bne- loopMBB
13409 // fallthrough --> exitMBB
13410
13411 // For max/min...
13412 // loopMBB:
13413 // l[wd]arx dest, ptr
13414 // cmpl?[wd] dest, incr
13415 // bgt exitMBB
13416 // loop2MBB:
13417 // st[wd]cx. dest, ptr
13418 // bne- loopMBB
13419 // fallthrough --> exitMBB
13420
13421 BB = loopMBB;
13422 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest)
13423 .addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13424 if (BinOpcode)
13425 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg).addReg(RegNo: incr).addReg(RegNo: dest);
13426 if (CmpOpcode) {
13427 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13428 // Signed comparisons of byte or halfword values must be sign-extended.
13429 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) {
13430 Register ExtReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13431 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH),
13432 DestReg: ExtReg).addReg(RegNo: dest);
13433 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ExtReg).addReg(RegNo: incr);
13434 } else
13435 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: dest).addReg(RegNo: incr);
13436
13437 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13438 .addImm(Val: CmpPred)
13439 .addReg(RegNo: CrReg)
13440 .addMBB(MBB: exitMBB);
13441 BB->addSuccessor(Succ: loop2MBB);
13442 BB->addSuccessor(Succ: exitMBB);
13443 BB = loop2MBB;
13444 }
13445 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
13446 .addReg(RegNo: TmpReg).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13447 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13448 .addImm(Val: PPC::PRED_NE_MINUS)
13449 .addReg(RegNo: PPC::CR0)
13450 .addMBB(MBB: loopMBB);
13451 BB->addSuccessor(Succ: loopMBB);
13452 BB->addSuccessor(Succ: exitMBB);
13453
13454 // exitMBB:
13455 // ...
13456 BB = exitMBB;
13457 return BB;
13458}
13459
13460static bool isSignExtended(MachineInstr &MI, const PPCInstrInfo *TII) {
13461 switch(MI.getOpcode()) {
13462 default:
13463 return false;
13464 case PPC::COPY:
13465 return TII->isSignExtended(Reg: MI.getOperand(i: 1).getReg(),
13466 MRI: &MI.getMF()->getRegInfo());
13467 case PPC::LHA:
13468 case PPC::LHA8:
13469 case PPC::LHAU:
13470 case PPC::LHAU8:
13471 case PPC::LHAUX:
13472 case PPC::LHAUX8:
13473 case PPC::LHAX:
13474 case PPC::LHAX8:
13475 case PPC::LWA:
13476 case PPC::LWAUX:
13477 case PPC::LWAX:
13478 case PPC::LWAX_32:
13479 case PPC::LWA_32:
13480 case PPC::PLHA:
13481 case PPC::PLHA8:
13482 case PPC::PLHA8pc:
13483 case PPC::PLHApc:
13484 case PPC::PLWA:
13485 case PPC::PLWA8:
13486 case PPC::PLWA8pc:
13487 case PPC::PLWApc:
13488 case PPC::EXTSB:
13489 case PPC::EXTSB8:
13490 case PPC::EXTSB8_32_64:
13491 case PPC::EXTSB8_rec:
13492 case PPC::EXTSB_rec:
13493 case PPC::EXTSH:
13494 case PPC::EXTSH8:
13495 case PPC::EXTSH8_32_64:
13496 case PPC::EXTSH8_rec:
13497 case PPC::EXTSH_rec:
13498 case PPC::EXTSW:
13499 case PPC::EXTSWSLI:
13500 case PPC::EXTSWSLI_32_64:
13501 case PPC::EXTSWSLI_32_64_rec:
13502 case PPC::EXTSWSLI_rec:
13503 case PPC::EXTSW_32:
13504 case PPC::EXTSW_32_64:
13505 case PPC::EXTSW_32_64_rec:
13506 case PPC::EXTSW_rec:
13507 case PPC::SRAW:
13508 case PPC::SRAWI:
13509 case PPC::SRAWI_rec:
13510 case PPC::SRAW_rec:
13511 return true;
13512 }
13513 return false;
13514}
13515
13516// Sign extend operand OpIdx if the value is not known to be sign extended.
13517// Assumes the operand is a register. The flag IsByte controls which intruction
13518// is used for the sign extension.
13519static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
13520 unsigned OpIdx, bool IsByte,
13521 const PPCInstrInfo *TII) {
13522 MachineFunction *F = MI.getMF();
13523 MachineRegisterInfo &RegInfo = F->getRegInfo();
13524 Register Reg = MI.getOperand(i: OpIdx).getReg();
13525 bool IsSignExtended =
13526 Reg.isVirtual() && isSignExtended(MI&: *RegInfo.getVRegDef(Reg), TII);
13527
13528 if (!IsSignExtended) {
13529 Register ValueReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13530 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(),
13531 MCID: TII->get(Opcode: IsByte ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueReg)
13532 .addReg(RegNo: Reg);
13533 MI.getOperand(i: OpIdx).setReg(ValueReg);
13534 }
13535}
13536
13537MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
13538 MachineInstr &MI, MachineBasicBlock *BB, unsigned BinOpcode,
13539 unsigned CmpOpcode, unsigned CmpPred) const {
13540 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13541 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13542 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13543 assert(!Subtarget.hasPartwordAtomics() &&
13544 "Assumes that part-word atomics are not available");
13545 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13546
13547 // If this is a signed comparison and the value being compared is not known
13548 // to be sign extended, sign extend it here.
13549 DebugLoc dl = MI.getDebugLoc();
13550 MachineFunction *F = BB->getParent();
13551 MachineRegisterInfo &RegInfo = F->getRegInfo();
13552 const bool is8bit = MI.getOperand(i: 3).getImm() == 1;
13553 if (CmpOpcode == PPC::CMPW)
13554 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, IsByte: is8bit, TII);
13555 Register incr = MI.getOperand(i: 4).getReg();
13556
13557 // In 64 bit mode we have to use 64 bits for addresses, even though the
13558 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address
13559 // registers without caring whether they're 32 or 64, but here we're
13560 // doing actual arithmetic on the addresses.
13561 bool is64bit = Subtarget.isPPC64();
13562 bool isLittleEndian = Subtarget.isLittleEndian();
13563 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
13564
13565 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13566 MachineFunction::iterator It = ++BB->getIterator();
13567
13568 Register dest = MI.getOperand(i: 0).getReg();
13569 Register ptrA = MI.getOperand(i: 1).getReg();
13570 Register ptrB = MI.getOperand(i: 2).getReg();
13571
13572 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13573 MachineBasicBlock *loop2MBB =
13574 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13575 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13576 F->insert(MBBI: It, MBB: loopMBB);
13577 if (CmpOpcode)
13578 F->insert(MBBI: It, MBB: loop2MBB);
13579 F->insert(MBBI: It, MBB: exitMBB);
13580 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13581 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13582 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13583
13584 const TargetRegisterClass *RC =
13585 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13586 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
13587
13588 Register PtrReg = RegInfo.createVirtualRegister(RegClass: RC);
13589 Register Shift1Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13590 Register ShiftReg =
13591 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13592 Register Incr2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13593 Register MaskReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13594 Register Mask2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13595 Register Mask3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13596 Register Tmp2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13597 Register Tmp3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13598 Register Tmp4Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13599 Register TmpDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13600 Register SrwDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13601 Register Ptr1Reg;
13602 Register TmpReg =
13603 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13604
13605 // thisMBB:
13606 // ...
13607 // fallthrough --> loopMBB
13608 BB->addSuccessor(Succ: loopMBB);
13609
13610 // The 4-byte load must be aligned, while a char or short may be
13611 // anywhere in the word. Hence all this nasty bookkeeping code.
13612 // add ptr1, ptrA, ptrB [copy if ptrA==0]
13613 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
13614 // xori shift, shift1, 24 [16]
13615 // rlwinm ptr, ptr1, 0, 0, 29
13616 // slw incr2, incr, shift
13617 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
13618 // slw mask, mask2, shift
13619 // loopMBB:
13620 // lwarx tmpDest, ptr
13621 // add tmp, tmpDest, incr2
13622 // andc tmp2, tmpDest, mask
13623 // and tmp3, tmp, mask
13624 // or tmp4, tmp3, tmp2
13625 // stwcx. tmp4, ptr
13626 // bne- loopMBB
13627 // fallthrough --> exitMBB
13628 // srw SrwDest, tmpDest, shift
13629 // rlwinm SrwDest, SrwDest, 0, 24 [16], 31
13630 if (ptrA != ZeroReg) {
13631 Ptr1Reg = RegInfo.createVirtualRegister(RegClass: RC);
13632 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
13633 .addReg(RegNo: ptrA)
13634 .addReg(RegNo: ptrB);
13635 } else {
13636 Ptr1Reg = ptrB;
13637 }
13638 // We need use 32-bit subregister to avoid mismatch register class in 64-bit
13639 // mode.
13640 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
13641 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
13642 .addImm(Val: 3)
13643 .addImm(Val: 27)
13644 .addImm(Val: is8bit ? 28 : 27);
13645 if (!isLittleEndian)
13646 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
13647 .addReg(RegNo: Shift1Reg)
13648 .addImm(Val: is8bit ? 24 : 16);
13649 if (is64bit)
13650 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
13651 .addReg(RegNo: Ptr1Reg)
13652 .addImm(Val: 0)
13653 .addImm(Val: 61);
13654 else
13655 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
13656 .addReg(RegNo: Ptr1Reg)
13657 .addImm(Val: 0)
13658 .addImm(Val: 0)
13659 .addImm(Val: 29);
13660 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: Incr2Reg).addReg(RegNo: incr).addReg(RegNo: ShiftReg);
13661 if (is8bit)
13662 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
13663 else {
13664 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
13665 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
13666 .addReg(RegNo: Mask3Reg)
13667 .addImm(Val: 65535);
13668 }
13669 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
13670 .addReg(RegNo: Mask2Reg)
13671 .addReg(RegNo: ShiftReg);
13672
13673 BB = loopMBB;
13674 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
13675 .addReg(RegNo: ZeroReg)
13676 .addReg(RegNo: PtrReg);
13677 if (BinOpcode)
13678 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg)
13679 .addReg(RegNo: Incr2Reg)
13680 .addReg(RegNo: TmpDestReg);
13681 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
13682 .addReg(RegNo: TmpDestReg)
13683 .addReg(RegNo: MaskReg);
13684 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: Tmp3Reg).addReg(RegNo: TmpReg).addReg(RegNo: MaskReg);
13685 if (CmpOpcode) {
13686 // For unsigned comparisons, we can directly compare the shifted values.
13687 // For signed comparisons we shift and sign extend.
13688 Register SReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13689 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13690 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: SReg)
13691 .addReg(RegNo: TmpDestReg)
13692 .addReg(RegNo: MaskReg);
13693 unsigned ValueReg = SReg;
13694 unsigned CmpReg = Incr2Reg;
13695 if (CmpOpcode == PPC::CMPW) {
13696 ValueReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13697 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: ValueReg)
13698 .addReg(RegNo: SReg)
13699 .addReg(RegNo: ShiftReg);
13700 Register ValueSReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13701 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is8bit ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueSReg)
13702 .addReg(RegNo: ValueReg);
13703 ValueReg = ValueSReg;
13704 CmpReg = incr;
13705 }
13706 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ValueReg).addReg(RegNo: CmpReg);
13707 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13708 .addImm(Val: CmpPred)
13709 .addReg(RegNo: CrReg)
13710 .addMBB(MBB: exitMBB);
13711 BB->addSuccessor(Succ: loop2MBB);
13712 BB->addSuccessor(Succ: exitMBB);
13713 BB = loop2MBB;
13714 }
13715 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg).addReg(RegNo: Tmp3Reg).addReg(RegNo: Tmp2Reg);
13716 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
13717 .addReg(RegNo: Tmp4Reg)
13718 .addReg(RegNo: ZeroReg)
13719 .addReg(RegNo: PtrReg);
13720 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13721 .addImm(Val: PPC::PRED_NE_MINUS)
13722 .addReg(RegNo: PPC::CR0)
13723 .addMBB(MBB: loopMBB);
13724 BB->addSuccessor(Succ: loopMBB);
13725 BB->addSuccessor(Succ: exitMBB);
13726
13727 // exitMBB:
13728 // ...
13729 BB = exitMBB;
13730 // Since the shift amount is not a constant, we need to clear
13731 // the upper bits with a separate RLWINM.
13732 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: dest)
13733 .addReg(RegNo: SrwDestReg)
13734 .addImm(Val: 0)
13735 .addImm(Val: is8bit ? 24 : 16)
13736 .addImm(Val: 31);
13737 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: SrwDestReg)
13738 .addReg(RegNo: TmpDestReg)
13739 .addReg(RegNo: ShiftReg);
13740 return BB;
13741}
13742
13743llvm::MachineBasicBlock *
13744PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
13745 MachineBasicBlock *MBB) const {
13746 DebugLoc DL = MI.getDebugLoc();
13747 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13748 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo();
13749
13750 MachineFunction *MF = MBB->getParent();
13751 MachineRegisterInfo &MRI = MF->getRegInfo();
13752
13753 const BasicBlock *BB = MBB->getBasicBlock();
13754 MachineFunction::iterator I = ++MBB->getIterator();
13755
13756 Register DstReg = MI.getOperand(i: 0).getReg();
13757 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
13758 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
13759 Register mainDstReg = MRI.createVirtualRegister(RegClass: RC);
13760 Register restoreDstReg = MRI.createVirtualRegister(RegClass: RC);
13761
13762 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13763 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13764 "Invalid Pointer Size!");
13765 // For v = setjmp(buf), we generate
13766 //
13767 // thisMBB:
13768 // SjLjSetup mainMBB
13769 // bl mainMBB
13770 // v_restore = 1
13771 // b sinkMBB
13772 //
13773 // mainMBB:
13774 // buf[LabelOffset] = LR
13775 // v_main = 0
13776 //
13777 // sinkMBB:
13778 // v = phi(main, restore)
13779 //
13780
13781 MachineBasicBlock *thisMBB = MBB;
13782 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
13783 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
13784 MF->insert(MBBI: I, MBB: mainMBB);
13785 MF->insert(MBBI: I, MBB: sinkMBB);
13786
13787 MachineInstrBuilder MIB;
13788
13789 // Transfer the remainder of BB and its successor edges to sinkMBB.
13790 sinkMBB->splice(Where: sinkMBB->begin(), Other: MBB,
13791 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
13792 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
13793
13794 // Note that the structure of the jmp_buf used here is not compatible
13795 // with that used by libc, and is not designed to be. Specifically, it
13796 // stores only those 'reserved' registers that LLVM does not otherwise
13797 // understand how to spill. Also, by convention, by the time this
13798 // intrinsic is called, Clang has already stored the frame address in the
13799 // first slot of the buffer and stack address in the third. Following the
13800 // X86 target code, we'll store the jump address in the second slot. We also
13801 // need to save the TOC pointer (R2) to handle jumps between shared
13802 // libraries, and that will be stored in the fourth slot. The thread
13803 // identifier (R13) is not affected.
13804
13805 // thisMBB:
13806 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13807 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13808 const int64_t BPOffset = 4 * PVT.getStoreSize();
13809
13810 // Prepare IP either in reg.
13811 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
13812 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
13813 Register BufReg = MI.getOperand(i: 1).getReg();
13814
13815 if (Subtarget.is64BitELFABI()) {
13816 setUsesTOCBasePtr(*MBB->getParent());
13817 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13818 .addReg(RegNo: PPC::X2)
13819 .addImm(Val: TOCOffset)
13820 .addReg(RegNo: BufReg)
13821 .cloneMemRefs(OtherMI: MI);
13822 }
13823
13824 // Naked functions never have a base pointer, and so we use r1. For all
13825 // other functions, this decision must be delayed until during PEI.
13826 unsigned BaseReg;
13827 if (MF->getFunction().hasFnAttribute(Kind: Attribute::Naked))
13828 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
13829 else
13830 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP;
13831
13832 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL,
13833 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::STD : PPC::STW))
13834 .addReg(RegNo: BaseReg)
13835 .addImm(Val: BPOffset)
13836 .addReg(RegNo: BufReg)
13837 .cloneMemRefs(OtherMI: MI);
13838
13839 // Setup
13840 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::BCLalways)).addMBB(MBB: mainMBB);
13841 MIB.addRegMask(Mask: TRI->getNoPreservedMask());
13842
13843 BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: restoreDstReg).addImm(Val: 1);
13844
13845 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::EH_SjLj_Setup))
13846 .addMBB(MBB: mainMBB);
13847 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: sinkMBB);
13848
13849 thisMBB->addSuccessor(Succ: mainMBB, Prob: BranchProbability::getZero());
13850 thisMBB->addSuccessor(Succ: sinkMBB, Prob: BranchProbability::getOne());
13851
13852 // mainMBB:
13853 // mainDstReg = 0
13854 MIB =
13855 BuildMI(BB: mainMBB, MIMD: DL,
13856 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), DestReg: LabelReg);
13857
13858 // Store IP
13859 if (Subtarget.isPPC64()) {
13860 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13861 .addReg(RegNo: LabelReg)
13862 .addImm(Val: LabelOffset)
13863 .addReg(RegNo: BufReg);
13864 } else {
13865 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STW))
13866 .addReg(RegNo: LabelReg)
13867 .addImm(Val: LabelOffset)
13868 .addReg(RegNo: BufReg);
13869 }
13870 MIB.cloneMemRefs(OtherMI: MI);
13871
13872 BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: mainDstReg).addImm(Val: 0);
13873 mainMBB->addSuccessor(Succ: sinkMBB);
13874
13875 // sinkMBB:
13876 BuildMI(BB&: *sinkMBB, I: sinkMBB->begin(), MIMD: DL,
13877 MCID: TII->get(Opcode: PPC::PHI), DestReg: DstReg)
13878 .addReg(RegNo: mainDstReg).addMBB(MBB: mainMBB)
13879 .addReg(RegNo: restoreDstReg).addMBB(MBB: thisMBB);
13880
13881 MI.eraseFromParent();
13882 return sinkMBB;
13883}
13884
13885MachineBasicBlock *
13886PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
13887 MachineBasicBlock *MBB) const {
13888 DebugLoc DL = MI.getDebugLoc();
13889 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13890
13891 MachineFunction *MF = MBB->getParent();
13892 MachineRegisterInfo &MRI = MF->getRegInfo();
13893
13894 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13895 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13896 "Invalid Pointer Size!");
13897
13898 const TargetRegisterClass *RC =
13899 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13900 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
13901 // Since FP is only updated here but NOT referenced, it's treated as GPR.
13902 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31;
13903 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1;
13904 unsigned BP =
13905 (PVT == MVT::i64)
13906 ? PPC::X30
13907 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29
13908 : PPC::R30);
13909
13910 MachineInstrBuilder MIB;
13911
13912 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13913 const int64_t SPOffset = 2 * PVT.getStoreSize();
13914 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13915 const int64_t BPOffset = 4 * PVT.getStoreSize();
13916
13917 Register BufReg = MI.getOperand(i: 0).getReg();
13918
13919 // Reload FP (the jumped-to function may not have had a
13920 // frame pointer, and if so, then its r31 will be restored
13921 // as necessary).
13922 if (PVT == MVT::i64) {
13923 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: FP)
13924 .addImm(Val: 0)
13925 .addReg(RegNo: BufReg);
13926 } else {
13927 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: FP)
13928 .addImm(Val: 0)
13929 .addReg(RegNo: BufReg);
13930 }
13931 MIB.cloneMemRefs(OtherMI: MI);
13932
13933 // Reload IP
13934 if (PVT == MVT::i64) {
13935 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: Tmp)
13936 .addImm(Val: LabelOffset)
13937 .addReg(RegNo: BufReg);
13938 } else {
13939 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: Tmp)
13940 .addImm(Val: LabelOffset)
13941 .addReg(RegNo: BufReg);
13942 }
13943 MIB.cloneMemRefs(OtherMI: MI);
13944
13945 // Reload SP
13946 if (PVT == MVT::i64) {
13947 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: SP)
13948 .addImm(Val: SPOffset)
13949 .addReg(RegNo: BufReg);
13950 } else {
13951 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: SP)
13952 .addImm(Val: SPOffset)
13953 .addReg(RegNo: BufReg);
13954 }
13955 MIB.cloneMemRefs(OtherMI: MI);
13956
13957 // Reload BP
13958 if (PVT == MVT::i64) {
13959 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: BP)
13960 .addImm(Val: BPOffset)
13961 .addReg(RegNo: BufReg);
13962 } else {
13963 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: BP)
13964 .addImm(Val: BPOffset)
13965 .addReg(RegNo: BufReg);
13966 }
13967 MIB.cloneMemRefs(OtherMI: MI);
13968
13969 // Reload TOC
13970 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) {
13971 setUsesTOCBasePtr(*MBB->getParent());
13972 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: PPC::X2)
13973 .addImm(Val: TOCOffset)
13974 .addReg(RegNo: BufReg)
13975 .cloneMemRefs(OtherMI: MI);
13976 }
13977
13978 // Jump
13979 BuildMI(BB&: *MBB, I&: MI, MIMD: DL,
13980 MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(RegNo: Tmp);
13981 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR));
13982
13983 MI.eraseFromParent();
13984 return MBB;
13985}
13986
13987bool PPCTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
13988 // If the function specifically requests inline stack probes, emit them.
13989 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
13990 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
13991 "inline-asm";
13992 return false;
13993}
13994
13995unsigned PPCTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
13996 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
13997 unsigned StackAlign = TFI->getStackAlignment();
13998 assert(StackAlign >= 1 && isPowerOf2_32(StackAlign) &&
13999 "Unexpected stack alignment");
14000 // The default stack probe size is 4096 if the function has no
14001 // stack-probe-size attribute.
14002 const Function &Fn = MF.getFunction();
14003 unsigned StackProbeSize =
14004 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
14005 // Round down to the stack alignment.
14006 StackProbeSize &= ~(StackAlign - 1);
14007 return StackProbeSize ? StackProbeSize : StackAlign;
14008}
14009
14010// Lower dynamic stack allocation with probing. `emitProbedAlloca` is splitted
14011// into three phases. In the first phase, it uses pseudo instruction
14012// PREPARE_PROBED_ALLOCA to get the future result of actual FramePointer and
14013// FinalStackPtr. In the second phase, it generates a loop for probing blocks.
14014// At last, it uses pseudo instruction DYNAREAOFFSET to get the future result of
14015// MaxCallFrameSize so that it can calculate correct data area pointer.
14016MachineBasicBlock *
14017PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
14018 MachineBasicBlock *MBB) const {
14019 const bool isPPC64 = Subtarget.isPPC64();
14020 MachineFunction *MF = MBB->getParent();
14021 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
14022 DebugLoc DL = MI.getDebugLoc();
14023 const unsigned ProbeSize = getStackProbeSize(MF: *MF);
14024 const BasicBlock *ProbedBB = MBB->getBasicBlock();
14025 MachineRegisterInfo &MRI = MF->getRegInfo();
14026 // The CFG of probing stack looks as
14027 // +-----+
14028 // | MBB |
14029 // +--+--+
14030 // |
14031 // +----v----+
14032 // +--->+ TestMBB +---+
14033 // | +----+----+ |
14034 // | | |
14035 // | +-----v----+ |
14036 // +---+ BlockMBB | |
14037 // +----------+ |
14038 // |
14039 // +---------+ |
14040 // | TailMBB +<--+
14041 // +---------+
14042 // In MBB, calculate previous frame pointer and final stack pointer.
14043 // In TestMBB, test if sp is equal to final stack pointer, if so, jump to
14044 // TailMBB. In BlockMBB, update the sp atomically and jump back to TestMBB.
14045 // TailMBB is spliced via \p MI.
14046 MachineBasicBlock *TestMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14047 MachineBasicBlock *TailMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14048 MachineBasicBlock *BlockMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14049
14050 MachineFunction::iterator MBBIter = ++MBB->getIterator();
14051 MF->insert(MBBI: MBBIter, MBB: TestMBB);
14052 MF->insert(MBBI: MBBIter, MBB: BlockMBB);
14053 MF->insert(MBBI: MBBIter, MBB: TailMBB);
14054
14055 const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
14056 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
14057
14058 Register DstReg = MI.getOperand(i: 0).getReg();
14059 Register NegSizeReg = MI.getOperand(i: 1).getReg();
14060 Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
14061 Register FinalStackPtr = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14062 Register FramePointer = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14063 Register ActualNegSizeReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14064
14065 // Since value of NegSizeReg might be realigned in prologepilog, insert a
14066 // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and
14067 // NegSize.
14068 unsigned ProbeOpc;
14069 if (!MRI.hasOneNonDBGUse(RegNo: NegSizeReg))
14070 ProbeOpc =
14071 isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32;
14072 else
14073 // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg
14074 // and NegSizeReg will be allocated in the same phyreg to avoid
14075 // redundant copy when NegSizeReg has only one use which is current MI and
14076 // will be replaced by PREPARE_PROBED_ALLOCA then.
14077 ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64
14078 : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32;
14079 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: ProbeOpc), DestReg: FramePointer)
14080 .addDef(RegNo: ActualNegSizeReg)
14081 .addReg(RegNo: NegSizeReg)
14082 .add(MO: MI.getOperand(i: 2))
14083 .add(MO: MI.getOperand(i: 3));
14084
14085 // Calculate final stack pointer, which equals to SP + ActualNegSize.
14086 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4),
14087 DestReg: FinalStackPtr)
14088 .addReg(RegNo: SPReg)
14089 .addReg(RegNo: ActualNegSizeReg);
14090
14091 // Materialize a scratch register for update.
14092 int64_t NegProbeSize = -(int64_t)ProbeSize;
14093 assert(isInt<32>(NegProbeSize) && "Unhandled probe size!");
14094 Register ScratchReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14095 if (!isInt<16>(x: NegProbeSize)) {
14096 Register TempReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14097 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LIS8 : PPC::LIS), DestReg: TempReg)
14098 .addImm(Val: NegProbeSize >> 16);
14099 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ORI8 : PPC::ORI),
14100 DestReg: ScratchReg)
14101 .addReg(RegNo: TempReg)
14102 .addImm(Val: NegProbeSize & 0xFFFF);
14103 } else
14104 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LI8 : PPC::LI), DestReg: ScratchReg)
14105 .addImm(Val: NegProbeSize);
14106
14107 {
14108 // Probing leading residual part.
14109 Register Div = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14110 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::DIVD : PPC::DIVW), DestReg: Div)
14111 .addReg(RegNo: ActualNegSizeReg)
14112 .addReg(RegNo: ScratchReg);
14113 Register Mul = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14114 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::MULLD : PPC::MULLW), DestReg: Mul)
14115 .addReg(RegNo: Div)
14116 .addReg(RegNo: ScratchReg);
14117 Register NegMod = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14118 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::SUBF8 : PPC::SUBF), DestReg: NegMod)
14119 .addReg(RegNo: Mul)
14120 .addReg(RegNo: ActualNegSizeReg);
14121 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14122 .addReg(RegNo: FramePointer)
14123 .addReg(RegNo: SPReg)
14124 .addReg(RegNo: NegMod);
14125 }
14126
14127 {
14128 // Remaining part should be multiple of ProbeSize.
14129 Register CmpResult = MRI.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14130 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::CMPD : PPC::CMPW), DestReg: CmpResult)
14131 .addReg(RegNo: SPReg)
14132 .addReg(RegNo: FinalStackPtr);
14133 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::BCC))
14134 .addImm(Val: PPC::PRED_EQ)
14135 .addReg(RegNo: CmpResult)
14136 .addMBB(MBB: TailMBB);
14137 TestMBB->addSuccessor(Succ: BlockMBB);
14138 TestMBB->addSuccessor(Succ: TailMBB);
14139 }
14140
14141 {
14142 // Touch the block.
14143 // |P...|P...|P...
14144 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14145 .addReg(RegNo: FramePointer)
14146 .addReg(RegNo: SPReg)
14147 .addReg(RegNo: ScratchReg);
14148 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: TestMBB);
14149 BlockMBB->addSuccessor(Succ: TestMBB);
14150 }
14151
14152 // Calculation of MaxCallFrameSize is deferred to prologepilog, use
14153 // DYNAREAOFFSET pseudo instruction to get the future result.
14154 Register MaxCallFrameSizeReg =
14155 MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14156 BuildMI(BB: TailMBB, MIMD: DL,
14157 MCID: TII->get(Opcode: isPPC64 ? PPC::DYNAREAOFFSET8 : PPC::DYNAREAOFFSET),
14158 DestReg: MaxCallFrameSizeReg)
14159 .add(MO: MI.getOperand(i: 2))
14160 .add(MO: MI.getOperand(i: 3));
14161 BuildMI(BB: TailMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4), DestReg: DstReg)
14162 .addReg(RegNo: SPReg)
14163 .addReg(RegNo: MaxCallFrameSizeReg);
14164
14165 // Splice instructions after MI to TailMBB.
14166 TailMBB->splice(Where: TailMBB->end(), Other: MBB,
14167 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
14168 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
14169 MBB->addSuccessor(Succ: TestMBB);
14170
14171 // Delete the pseudo instruction.
14172 MI.eraseFromParent();
14173
14174 ++NumDynamicAllocaProbed;
14175 return TailMBB;
14176}
14177
14178/// Check if the opcode is a SELECT or SELECT_CC variant.
14179/// @param Opcode The opcode to check
14180/// @param CheckOnlyCC If true, only return true for SELECT_CC variants;
14181/// if false, return true for both SELECT and SELECT_CC
14182static bool IsSelect(unsigned Opcode, bool CheckOnlyCC = false) {
14183 switch (Opcode) {
14184 // SELECT_CC variants - always return true
14185 case PPC::SELECT_CC_I4:
14186 case PPC::SELECT_CC_I8:
14187 case PPC::SELECT_CC_F4:
14188 case PPC::SELECT_CC_F8:
14189 case PPC::SELECT_CC_F16:
14190 case PPC::SELECT_CC_VRRC:
14191 case PPC::SELECT_CC_VSFRC:
14192 case PPC::SELECT_CC_VSSRC:
14193 case PPC::SELECT_CC_VSRC:
14194 case PPC::SELECT_CC_SPE4:
14195 case PPC::SELECT_CC_SPE:
14196 return true;
14197 // SELECT variants - only return true if CheckOnlyCC is false
14198 case PPC::SELECT_I4:
14199 case PPC::SELECT_I8:
14200 case PPC::SELECT_F4:
14201 case PPC::SELECT_F8:
14202 case PPC::SELECT_F16:
14203 case PPC::SELECT_SPE:
14204 case PPC::SELECT_SPE4:
14205 case PPC::SELECT_VRRC:
14206 case PPC::SELECT_VSFRC:
14207 case PPC::SELECT_VSSRC:
14208 case PPC::SELECT_VSRC:
14209 return !CheckOnlyCC; // true if checking all SELECTs, false if only CC
14210 default:
14211 return false;
14212 }
14213}
14214static bool IsSelectCC(unsigned Opcode) { return IsSelect(Opcode, CheckOnlyCC: true); }
14215
14216/// Emit SELECT instruction, using ISEL if available, otherwise use
14217/// branch-based control flow.
14218///
14219/// For targets with ISEL support (SELECT_CC_I4/I8, SELECT_I4/I8), this
14220/// generates a single ISEL instruction. Otherwise, it creates a
14221/// branch-based control flow pattern with PHI nodes.
14222static MachineBasicBlock *emitSelect(MachineInstr &MI, MachineBasicBlock *BB,
14223 const TargetInstrInfo *TII,
14224 const PPCSubtarget &Subtarget) {
14225 assert(IsSelect(MI.getOpcode()) && "Instruction must be a SELECT variant");
14226
14227 // Check if we can use ISEL for this SELECT
14228 if (Subtarget.hasISEL() &&
14229 (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14230 MI.getOpcode() == PPC::SELECT_CC_I8 ||
14231 MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8)) {
14232 SmallVector<MachineOperand, 2> Cond;
14233 if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14234 MI.getOpcode() == PPC::SELECT_CC_I8)
14235 Cond.push_back(Elt: MI.getOperand(i: 4));
14236 else
14237 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
14238 Cond.push_back(Elt: MI.getOperand(i: 1));
14239
14240 DebugLoc dl = MI.getDebugLoc();
14241 TII->insertSelect(MBB&: *BB, I: MI, DL: dl, DstReg: MI.getOperand(i: 0).getReg(), Cond,
14242 TrueReg: MI.getOperand(i: 2).getReg(), FalseReg: MI.getOperand(i: 3).getReg());
14243 MI.eraseFromParent();
14244 return BB;
14245 }
14246
14247 // Fall back to branch-based SELECT implementation
14248 MachineFunction *F = BB->getParent();
14249 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14250 MachineFunction::iterator It = ++BB->getIterator();
14251 DebugLoc dl = MI.getDebugLoc();
14252
14253 MachineBasicBlock *thisMBB = BB;
14254 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14255 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14256 F->insert(MBBI: It, MBB: copy0MBB);
14257 F->insert(MBBI: It, MBB: sinkMBB);
14258
14259 if (isPhysRegUsedAfter(Reg: PPC::CARRY, MBI: MI.getIterator())) {
14260 copy0MBB->addLiveIn(PhysReg: PPC::CARRY);
14261 sinkMBB->addLiveIn(PhysReg: PPC::CARRY);
14262 }
14263
14264 // Set the call frame size on entry to the new basic blocks.
14265 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
14266 copy0MBB->setCallFrameSize(CallFrameSize);
14267 sinkMBB->setCallFrameSize(CallFrameSize);
14268
14269 // Transfer the remainder of BB and its successor edges to sinkMBB.
14270 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14271 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14272 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14273
14274 // Add successors
14275 BB->addSuccessor(Succ: copy0MBB);
14276 BB->addSuccessor(Succ: sinkMBB);
14277
14278 // Build branch instruction
14279 if (IsSelectCC(Opcode: MI.getOpcode()))
14280 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14281 .addImm(Val: MI.getOperand(i: 4).getImm())
14282 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14283 .addMBB(MBB: sinkMBB);
14284 else
14285 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BC))
14286 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14287 .addMBB(MBB: sinkMBB);
14288
14289 // copy0MBB: fallthrough to sinkMBB
14290 BB = copy0MBB;
14291 BB->addSuccessor(Succ: sinkMBB);
14292
14293 // sinkMBB: PHI instruction
14294 BB = sinkMBB;
14295 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::PHI), DestReg: MI.getOperand(i: 0).getReg())
14296 .addReg(RegNo: MI.getOperand(i: 3).getReg())
14297 .addMBB(MBB: copy0MBB)
14298 .addReg(RegNo: MI.getOperand(i: 2).getReg())
14299 .addMBB(MBB: thisMBB);
14300 MI.eraseFromParent();
14301 return BB;
14302}
14303
14304/// Helper function to create basic blocks for atomic compare-and-swap.
14305/// Creates three basic blocks (loop1MBB, loop2MBB, exitMBB) and sets up
14306/// the control flow structure common to both hardware and software
14307/// implementations of atomic compare-and-swap operations.
14308static void createAtomicLoopBlocks(MachineFunction *F, MachineBasicBlock *BB,
14309 MachineBasicBlock *&loop1MBB,
14310 MachineBasicBlock *&loop2MBB,
14311 MachineBasicBlock *&exitMBB,
14312 MachineInstr &MI,
14313 MachineFunction::iterator It) {
14314 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14315 loop1MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14316 loop2MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14317 exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14318 F->insert(MBBI: It, MBB: loop1MBB);
14319 F->insert(MBBI: It, MBB: loop2MBB);
14320 F->insert(MBBI: It, MBB: exitMBB);
14321 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
14322 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14323 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14324 BB->addSuccessor(Succ: loop1MBB);
14325}
14326
14327/// Emit hardware-supported atomic compare-and-swap for I32/I64 and I8/I16
14328/// with partword atomic support.
14329///
14330/// This uses native PowerPC atomic instructions (LBARX/LHARX/LWARX/LDARX for
14331/// load-and-reserve, STBCX/STHCX/STWCX/STDCX for store-conditional) to
14332/// implement atomic compare-and-swap at byte, halfword, word, or doubleword
14333/// granularity.
14334///
14335/// Control flow:
14336/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14337/// | |
14338/// +------------+
14339///
14340/// loop1MBB:
14341/// - Load-and-reserve from memory
14342/// - Compare loaded value with expected old value
14343/// - Branch to exitMBB if not equal (CAS failed)
14344/// loop2MBB:
14345/// - Store-conditional new value to memory
14346/// - Branch back to loop1MBB if store failed (retry)
14347/// - Fall through to exitMBB on success
14348static MachineBasicBlock *
14349emitAtomicCmpSwapHardware(MachineInstr &MI, MachineBasicBlock *BB,
14350 const TargetInstrInfo *TII,
14351 const PPCSubtarget &Subtarget) {
14352 MachineFunction *F = BB->getParent();
14353 MachineFunction::iterator It = ++BB->getIterator();
14354
14355 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64;
14356
14357 unsigned LoadMnemonic = PPC::LDARX;
14358 unsigned StoreMnemonic = PPC::STDCX;
14359 switch (MI.getOpcode()) {
14360 default:
14361 llvm_unreachable("Compare and swap of unknown size");
14362 case PPC::ATOMIC_CMP_SWAP_I8:
14363 LoadMnemonic = PPC::LBARX;
14364 StoreMnemonic = PPC::STBCX;
14365 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14366 break;
14367 case PPC::ATOMIC_CMP_SWAP_I16:
14368 LoadMnemonic = PPC::LHARX;
14369 StoreMnemonic = PPC::STHCX;
14370 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14371 break;
14372 case PPC::ATOMIC_CMP_SWAP_I32:
14373 LoadMnemonic = PPC::LWARX;
14374 StoreMnemonic = PPC::STWCX;
14375 break;
14376 case PPC::ATOMIC_CMP_SWAP_I64:
14377 LoadMnemonic = PPC::LDARX;
14378 StoreMnemonic = PPC::STDCX;
14379 break;
14380 }
14381
14382 MachineRegisterInfo &RegInfo = F->getRegInfo();
14383 Register dest = MI.getOperand(i: 0).getReg();
14384 Register ptrA = MI.getOperand(i: 1).getReg();
14385 Register ptrB = MI.getOperand(i: 2).getReg();
14386 Register oldval = MI.getOperand(i: 3).getReg();
14387 Register newval = MI.getOperand(i: 4).getReg();
14388 DebugLoc dl = MI.getDebugLoc();
14389
14390 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14391 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14392
14393 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14394
14395 // loop1MBB:
14396 // l[bhwd]arx dest, ptr
14397 // cmp[wd] dest, oldval
14398 // bne- exitBB
14399 BB = loop1MBB;
14400 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
14401 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::CMPD : PPC::CMPW), DestReg: CrReg)
14402 .addReg(RegNo: dest)
14403 .addReg(RegNo: oldval);
14404 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14405 .addImm(Val: PPC::PRED_NE_MINUS)
14406 .addReg(RegNo: CrReg)
14407 .addMBB(MBB: exitMBB);
14408 BB->addSuccessor(Succ: loop2MBB);
14409 BB->addSuccessor(Succ: exitMBB);
14410
14411 // loop2MBB:
14412 // st[bhwd]cx. newval, ptr
14413 // bne- loopMBB
14414 // b exitBB
14415 BB = loop2MBB;
14416 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
14417 .addReg(RegNo: newval)
14418 .addReg(RegNo: ptrA)
14419 .addReg(RegNo: ptrB);
14420 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14421 .addImm(Val: PPC::PRED_NE_MINUS)
14422 .addReg(RegNo: PPC::CR0)
14423 .addMBB(MBB: loop1MBB);
14424 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14425 BB->addSuccessor(Succ: loop1MBB);
14426 BB->addSuccessor(Succ: exitMBB);
14427
14428 return exitMBB;
14429}
14430
14431/// Emit software-emulated atomic compare-and-swap for I8/I16 without
14432/// hardware partword atomic support.
14433///
14434/// This emulates byte/halfword atomic operations using word (32-bit) atomic
14435/// instructions. Since PowerPC atomic instructions work at word granularity,
14436/// we must:
14437/// 1. Align the pointer to a word boundary
14438/// 2. Calculate the bit shift for the target byte/halfword within the word
14439/// 3. Create masks to isolate the target byte/halfword
14440/// 4. Shift old/new values into the correct bit position
14441/// 5. Use LWARX/STWCX on the full word
14442/// 6. Mask and merge to preserve other bytes in the word
14443/// 7. Extract and shift the result back
14444///
14445/// Control flow:
14446/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14447/// | |
14448/// +------------+
14449///
14450/// loop1MBB:
14451/// - LWARX: Load-and-reserve full word
14452/// - Mask to extract target byte/halfword
14453/// - Compare with expected old value
14454/// - Branch to exitMBB if not equal (CAS failed)
14455/// loop2MBB:
14456/// - Merge new value with other bytes in the word
14457/// - STWCX: Store-conditional full word
14458/// - Branch back to loop1MBB if store failed (retry)
14459/// - Fall through to exitMBB on success
14460/// exitMBB:
14461/// - Extract and return the loaded value
14462static MachineBasicBlock *
14463emitAtomicCmpSwapSoftware(MachineInstr &MI, MachineBasicBlock *BB,
14464 const TargetInstrInfo *TII,
14465 const PPCSubtarget &Subtarget) {
14466 MachineFunction *F = BB->getParent();
14467 MachineFunction::iterator It = ++BB->getIterator();
14468
14469 bool is64bit = Subtarget.isPPC64();
14470 bool isLittleEndian = Subtarget.isLittleEndian();
14471 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8;
14472
14473 Register dest = MI.getOperand(i: 0).getReg();
14474 Register ptrA = MI.getOperand(i: 1).getReg();
14475 Register ptrB = MI.getOperand(i: 2).getReg();
14476 Register oldval = MI.getOperand(i: 3).getReg();
14477 Register newval = MI.getOperand(i: 4).getReg();
14478 DebugLoc dl = MI.getDebugLoc();
14479
14480 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14481 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14482
14483 MachineRegisterInfo &RegInfo = F->getRegInfo();
14484 const TargetRegisterClass *RC =
14485 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
14486 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
14487
14488 // Lambda to create virtual registers
14489 auto createVReg = [&](const TargetRegisterClass *RC) {
14490 return RegInfo.createVirtualRegister(RegClass: RC);
14491 };
14492
14493 Register PtrReg = createVReg(RC);
14494 Register Shift1Reg = createVReg(GPRC);
14495 Register ShiftReg = isLittleEndian ? Shift1Reg : createVReg(GPRC);
14496 Register NewVal2Reg = createVReg(GPRC);
14497 Register NewVal3Reg = createVReg(GPRC);
14498 Register OldVal2Reg = createVReg(GPRC);
14499 Register OldVal3Reg = createVReg(GPRC);
14500 Register MaskReg = createVReg(GPRC);
14501 Register Mask2Reg = createVReg(GPRC);
14502 Register Mask3Reg = createVReg(GPRC);
14503 Register Tmp2Reg = createVReg(GPRC);
14504 Register Tmp4Reg = createVReg(GPRC);
14505 Register TmpDestReg = createVReg(GPRC);
14506 Register TmpReg = createVReg(GPRC);
14507 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
14508 Register CrReg = createVReg(&PPC::CRRCRegClass);
14509
14510 // Compute aligned pointer and shift amount
14511 Register Ptr1Reg;
14512 if (ptrA != ZeroReg) {
14513 Ptr1Reg = createVReg(RC);
14514 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
14515 .addReg(RegNo: ptrA)
14516 .addReg(RegNo: ptrB);
14517 } else {
14518 Ptr1Reg = ptrB;
14519 }
14520
14521 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
14522 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
14523 .addImm(Val: 3)
14524 .addImm(Val: 27)
14525 .addImm(Val: is8bit ? 28 : 27);
14526 if (!isLittleEndian)
14527 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
14528 .addReg(RegNo: Shift1Reg)
14529 .addImm(Val: is8bit ? 24 : 16);
14530 if (is64bit)
14531 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
14532 .addReg(RegNo: Ptr1Reg)
14533 .addImm(Val: 0)
14534 .addImm(Val: 61);
14535 else
14536 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
14537 .addReg(RegNo: Ptr1Reg)
14538 .addImm(Val: 0)
14539 .addImm(Val: 0)
14540 .addImm(Val: 29);
14541
14542 // Prepare masked values
14543 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: NewVal2Reg)
14544 .addReg(RegNo: newval)
14545 .addReg(RegNo: ShiftReg);
14546 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: OldVal2Reg)
14547 .addReg(RegNo: oldval)
14548 .addReg(RegNo: ShiftReg);
14549 if (is8bit)
14550 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
14551 else {
14552 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
14553 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
14554 .addReg(RegNo: Mask3Reg)
14555 .addImm(Val: 65535);
14556 }
14557 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
14558 .addReg(RegNo: Mask2Reg)
14559 .addReg(RegNo: ShiftReg);
14560 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: NewVal3Reg)
14561 .addReg(RegNo: NewVal2Reg)
14562 .addReg(RegNo: MaskReg);
14563 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: OldVal3Reg)
14564 .addReg(RegNo: OldVal2Reg)
14565 .addReg(RegNo: MaskReg);
14566
14567 // loop1MBB:
14568 // lwarx tmpDest, ptr
14569 // and tmp, tmpDest, mask
14570 // cmpw tmp, oldval3
14571 // bne- exitBB
14572 BB = loop1MBB;
14573 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
14574 .addReg(RegNo: ZeroReg)
14575 .addReg(RegNo: PtrReg);
14576 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: TmpReg)
14577 .addReg(RegNo: TmpDestReg)
14578 .addReg(RegNo: MaskReg);
14579 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CrReg).addReg(RegNo: TmpReg).addReg(RegNo: OldVal3Reg);
14580 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14581 .addImm(Val: PPC::PRED_NE)
14582 .addReg(RegNo: CrReg)
14583 .addMBB(MBB: exitMBB);
14584 BB->addSuccessor(Succ: loop2MBB);
14585 BB->addSuccessor(Succ: exitMBB);
14586
14587 // loop2MBB:
14588 // andc tmp2, tmpDest, mask
14589 // or tmp4, tmp2, newval3
14590 // stwcx. tmp4, ptr
14591 // bne- loop1MBB
14592 // b exitBB
14593 BB = loop2MBB;
14594 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
14595 .addReg(RegNo: TmpDestReg)
14596 .addReg(RegNo: MaskReg);
14597 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg)
14598 .addReg(RegNo: Tmp2Reg)
14599 .addReg(RegNo: NewVal3Reg);
14600 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
14601 .addReg(RegNo: Tmp4Reg)
14602 .addReg(RegNo: ZeroReg)
14603 .addReg(RegNo: PtrReg);
14604 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14605 .addImm(Val: PPC::PRED_NE)
14606 .addReg(RegNo: PPC::CR0)
14607 .addMBB(MBB: loop1MBB);
14608 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14609 BB->addSuccessor(Succ: loop1MBB);
14610 BB->addSuccessor(Succ: exitMBB);
14611
14612 // exitMBB:
14613 // srw dest, tmpDest, shift
14614 BB = exitMBB;
14615 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: dest)
14616 .addReg(RegNo: TmpReg)
14617 .addReg(RegNo: ShiftReg);
14618
14619 return BB;
14620}
14621
14622MachineBasicBlock *
14623PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
14624 MachineBasicBlock *BB) const {
14625 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
14626
14627 // To "insert" these instructions we actually have to insert their
14628 // control-flow patterns.
14629 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14630 MachineFunction::iterator It = ++BB->getIterator();
14631
14632 MachineFunction *F = BB->getParent();
14633 MachineRegisterInfo &MRI = F->getRegInfo();
14634
14635 // Handle SELECT with ISEL support first (before generic SELECT handling)
14636 if (IsSelect(Opcode: MI.getOpcode()))
14637 return emitSelect(MI, BB, TII, Subtarget);
14638
14639 switch (MI.getOpcode()) {
14640 case TargetOpcode::STACKMAP:
14641 return emitPatchPoint(MI, MBB: BB);
14642 case TargetOpcode::PATCHPOINT:
14643 // Call lowering should have added an r2 operand to indicate a dependence
14644 // on the TOC base pointer value. It can't however, because there is no
14645 // way to mark the dependence as implicit there, and so the stackmap code
14646 // will confuse it with a regular operand. Instead, add the dependence
14647 // here.
14648 if (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls())
14649 MI.addOperand(Op: MachineOperand::CreateReg(Reg: PPC::X2, isDef: false, isImp: true));
14650 return emitPatchPoint(MI, MBB: BB);
14651
14652 case PPC::EH_SjLj_SetJmp32:
14653 case PPC::EH_SjLj_SetJmp64:
14654 return emitEHSjLjSetJmp(MI, MBB: BB);
14655
14656 case PPC::EH_SjLj_LongJmp32:
14657 case PPC::EH_SjLj_LongJmp64:
14658 return emitEHSjLjLongJmp(MI, MBB: BB);
14659
14660 case PPC::ReadTB: {
14661 // To read the 64-bit time-base register on a 32-bit target, we read the
14662 // two halves. Should the counter have wrapped while it was being read, we
14663 // need to try again.
14664 // ...
14665 // readLoop:
14666 // mfspr Rx,TBU # load from TBU
14667 // mfspr Ry,TB # load from TB
14668 // mfspr Rz,TBU # load from TBU
14669 // cmpw crX,Rx,Rz # check if 'old'='new'
14670 // bne readLoop # branch if they're not equal
14671 // ...
14672
14673 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14674 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14675 DebugLoc dl = MI.getDebugLoc();
14676 F->insert(MBBI: It, MBB: readMBB);
14677 F->insert(MBBI: It, MBB: sinkMBB);
14678
14679 // Transfer the remainder of BB and its successor edges to sinkMBB.
14680 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14681 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14682 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14683
14684 BB->addSuccessor(Succ: readMBB);
14685 BB = readMBB;
14686
14687 MachineRegisterInfo &RegInfo = F->getRegInfo();
14688 Register ReadAgainReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
14689 Register LoReg = MI.getOperand(i: 0).getReg();
14690 Register HiReg = MI.getOperand(i: 1).getReg();
14691
14692 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: HiReg).addImm(Val: 269);
14693 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: LoReg).addImm(Val: 268);
14694 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: ReadAgainReg).addImm(Val: 269);
14695
14696 Register CmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14697
14698 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CmpReg)
14699 .addReg(RegNo: HiReg)
14700 .addReg(RegNo: ReadAgainReg);
14701 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14702 .addImm(Val: PPC::PRED_NE)
14703 .addReg(RegNo: CmpReg)
14704 .addMBB(MBB: readMBB);
14705
14706 BB->addSuccessor(Succ: readMBB);
14707 BB->addSuccessor(Succ: sinkMBB);
14708 break;
14709 }
14710 case PPC::ATOMIC_LOAD_ADD_NOWP:
14711 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14712 break;
14713 case PPC::ATOMIC_LOAD_ADD:
14714 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14715 break;
14716 case PPC::ATOMIC_LOAD_ADD_I64:
14717 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD8);
14718 break;
14719 case PPC::ATOMIC_LOAD_AND_NOWP:
14720 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14721 break;
14722 case PPC::ATOMIC_LOAD_AND:
14723 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14724 break;
14725 case PPC::ATOMIC_LOAD_AND_I64:
14726 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND8);
14727 break;
14728 case PPC::ATOMIC_LOAD_OR_NOWP:
14729 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14730 break;
14731 case PPC::ATOMIC_LOAD_OR:
14732 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14733 break;
14734 case PPC::ATOMIC_LOAD_OR_I64:
14735 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR8);
14736 break;
14737 case PPC::ATOMIC_LOAD_XOR_NOWP:
14738 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14739 break;
14740 case PPC::ATOMIC_LOAD_XOR:
14741 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14742 break;
14743 case PPC::ATOMIC_LOAD_XOR_I64:
14744 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR8);
14745 break;
14746 case PPC::ATOMIC_LOAD_NAND_NOWP:
14747 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14748 break;
14749 case PPC::ATOMIC_LOAD_NAND:
14750 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14751 break;
14752 case PPC::ATOMIC_LOAD_NAND_I64:
14753 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND8);
14754 break;
14755 case PPC::ATOMIC_LOAD_SUB_NOWP:
14756 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14757 break;
14758 case PPC::ATOMIC_LOAD_SUB:
14759 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14760 break;
14761 case PPC::ATOMIC_LOAD_SUB_I64:
14762 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF8);
14763 break;
14764 case PPC::ATOMIC_LOAD_MIN_NOWP:
14765 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14766 break;
14767 case PPC::ATOMIC_LOAD_MIN:
14768 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14769 break;
14770 case PPC::ATOMIC_LOAD_MIN_I64:
14771 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_LT);
14772 break;
14773 case PPC::ATOMIC_LOAD_MAX_NOWP:
14774 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14775 break;
14776 case PPC::ATOMIC_LOAD_MAX:
14777 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14778 break;
14779 case PPC::ATOMIC_LOAD_MAX_I64:
14780 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_GT);
14781 break;
14782 case PPC::ATOMIC_LOAD_UMIN_NOWP:
14783 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14784 break;
14785 case PPC::ATOMIC_LOAD_UMIN:
14786 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14787 break;
14788 case PPC::ATOMIC_LOAD_UMIN_I64:
14789 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_LT);
14790 break;
14791 case PPC::ATOMIC_LOAD_UMAX_NOWP:
14792 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14793 break;
14794 case PPC::ATOMIC_LOAD_UMAX:
14795 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14796 break;
14797 case PPC::ATOMIC_LOAD_UMAX_I64:
14798 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_GT);
14799 break;
14800 case PPC::ATOMIC_SWAP_NOWP:
14801 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0);
14802 break;
14803 case PPC::ATOMIC_SWAP:
14804 case PPC::ATOMIC_SWAP_I64:
14805 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0);
14806 break;
14807 case PPC::ATOMIC_CMP_SWAP_I32:
14808 case PPC::ATOMIC_CMP_SWAP_I64:
14809 case PPC::ATOMIC_CMP_SWAP_I8:
14810 case PPC::ATOMIC_CMP_SWAP_I16: {
14811 // Use hardware-supported atomic operations if available
14812 bool useHardware = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
14813 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 ||
14814 (Subtarget.hasPartwordAtomics() &&
14815 (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 ||
14816 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16));
14817
14818 if (useHardware)
14819 BB = emitAtomicCmpSwapHardware(MI, BB, TII, Subtarget);
14820 else
14821 BB = emitAtomicCmpSwapSoftware(MI, BB, TII, Subtarget);
14822 break;
14823 }
14824 case PPC::FADDrtz: {
14825 // This pseudo performs an FADD with rounding mode temporarily forced
14826 // to round-to-zero. We emit this via custom inserter since the FPSCR
14827 // is not modeled at the SelectionDAG level.
14828 Register Dest = MI.getOperand(i: 0).getReg();
14829 Register Src1 = MI.getOperand(i: 1).getReg();
14830 Register Src2 = MI.getOperand(i: 2).getReg();
14831 DebugLoc dl = MI.getDebugLoc();
14832
14833 MachineRegisterInfo &RegInfo = F->getRegInfo();
14834 Register MFFSReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
14835
14836 // Save FPSCR value.
14837 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: MFFSReg);
14838
14839 // Set rounding mode to round-to-zero.
14840 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB1))
14841 .addImm(Val: 31)
14842 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14843
14844 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB0))
14845 .addImm(Val: 30)
14846 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14847
14848 // Perform addition.
14849 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::FADD), DestReg: Dest)
14850 .addReg(RegNo: Src1)
14851 .addReg(RegNo: Src2);
14852 if (MI.getFlag(Flag: MachineInstr::NoFPExcept))
14853 MIB.setMIFlag(MachineInstr::NoFPExcept);
14854
14855 // Restore FPSCR value.
14856 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSFb)).addImm(Val: 1).addReg(RegNo: MFFSReg);
14857 break;
14858 }
14859 case PPC::ANDI_rec_1_EQ_BIT:
14860 case PPC::ANDI_rec_1_GT_BIT:
14861 case PPC::ANDI_rec_1_EQ_BIT8:
14862 case PPC::ANDI_rec_1_GT_BIT8: {
14863 unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 ||
14864 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8)
14865 ? PPC::ANDI8_rec
14866 : PPC::ANDI_rec;
14867 bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT ||
14868 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8);
14869
14870 MachineRegisterInfo &RegInfo = F->getRegInfo();
14871 Register Dest = RegInfo.createVirtualRegister(
14872 RegClass: Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass);
14873
14874 DebugLoc Dl = MI.getDebugLoc();
14875 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode), DestReg: Dest)
14876 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14877 .addImm(Val: 1);
14878 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14879 DestReg: MI.getOperand(i: 0).getReg())
14880 .addReg(RegNo: IsEQ ? PPC::CR0EQ : PPC::CR0GT);
14881 break;
14882 }
14883 case PPC::TCHECK_RET: {
14884 DebugLoc Dl = MI.getDebugLoc();
14885 MachineRegisterInfo &RegInfo = F->getRegInfo();
14886 Register CRReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14887 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TCHECK), DestReg: CRReg);
14888 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14889 DestReg: MI.getOperand(i: 0).getReg())
14890 .addReg(RegNo: CRReg);
14891 break;
14892 }
14893 case PPC::TBEGIN_RET: {
14894 DebugLoc Dl = MI.getDebugLoc();
14895 unsigned Imm = MI.getOperand(i: 1).getImm();
14896 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TBEGIN)).addImm(Val: Imm);
14897 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14898 DestReg: MI.getOperand(i: 0).getReg())
14899 .addReg(RegNo: PPC::CR0EQ);
14900 break;
14901 }
14902 case PPC::SETRNDi: {
14903 DebugLoc dl = MI.getDebugLoc();
14904 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14905
14906 // Save FPSCR value.
14907 if (MRI.use_empty(RegNo: OldFPSCRReg))
14908 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
14909 else
14910 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14911
14912 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has
14913 // the following settings:
14914 // 00 Round to nearest
14915 // 01 Round to 0
14916 // 10 Round to +inf
14917 // 11 Round to -inf
14918
14919 // When the operand is immediate, using the two least significant bits of
14920 // the immediate to set the bits 62:63 of FPSCR.
14921 unsigned Mode = MI.getOperand(i: 1).getImm();
14922 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0))
14923 .addImm(Val: 31)
14924 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14925
14926 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0))
14927 .addImm(Val: 30)
14928 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14929 break;
14930 }
14931 case PPC::SETRND: {
14932 DebugLoc dl = MI.getDebugLoc();
14933
14934 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg
14935 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg.
14936 // If the target doesn't have DirectMove, we should use stack to do the
14937 // conversion, because the target doesn't have the instructions like mtvsrd
14938 // or mfvsrd to do this conversion directly.
14939 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) {
14940 if (Subtarget.hasDirectMove()) {
14941 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg)
14942 .addReg(RegNo: SrcReg);
14943 } else {
14944 // Use stack to do the register copy.
14945 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD;
14946 MachineRegisterInfo &RegInfo = F->getRegInfo();
14947 const TargetRegisterClass *RC = RegInfo.getRegClass(Reg: SrcReg);
14948 if (RC == &PPC::F8RCRegClass) {
14949 // Copy register from F8RCRegClass to G8RCRegclass.
14950 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) &&
14951 "Unsupported RegClass.");
14952
14953 StoreOp = PPC::STFD;
14954 LoadOp = PPC::LD;
14955 } else {
14956 // Copy register from G8RCRegClass to F8RCRegclass.
14957 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) &&
14958 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) &&
14959 "Unsupported RegClass.");
14960 }
14961
14962 MachineFrameInfo &MFI = F->getFrameInfo();
14963 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
14964
14965 MachineMemOperand *MMOStore = F->getMachineMemOperand(
14966 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14967 F: MachineMemOperand::MOStore, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14968 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14969
14970 // Store the SrcReg into the stack.
14971 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: StoreOp))
14972 .addReg(RegNo: SrcReg)
14973 .addImm(Val: 0)
14974 .addFrameIndex(Idx: FrameIdx)
14975 .addMemOperand(MMO: MMOStore);
14976
14977 MachineMemOperand *MMOLoad = F->getMachineMemOperand(
14978 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14979 F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14980 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14981
14982 // Load from the stack where SrcReg is stored, and save to DestReg,
14983 // so we have done the RegClass conversion from RegClass::SrcReg to
14984 // RegClass::DestReg.
14985 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: LoadOp), DestReg)
14986 .addImm(Val: 0)
14987 .addFrameIndex(Idx: FrameIdx)
14988 .addMemOperand(MMO: MMOLoad);
14989 }
14990 };
14991
14992 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14993
14994 // Save FPSCR value.
14995 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14996
14997 // When the operand is gprc register, use two least significant bits of the
14998 // register and mtfsf instruction to set the bits 62:63 of FPSCR.
14999 //
15000 // copy OldFPSCRTmpReg, OldFPSCRReg
15001 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1)
15002 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62
15003 // copy NewFPSCRReg, NewFPSCRTmpReg
15004 // mtfsf 255, NewFPSCRReg
15005 MachineOperand SrcOp = MI.getOperand(i: 1);
15006 MachineRegisterInfo &RegInfo = F->getRegInfo();
15007 Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
15008
15009 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg);
15010
15011 Register ImDefReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
15012 Register ExtSrcReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
15013
15014 // The first operand of INSERT_SUBREG should be a register which has
15015 // subregisters, we only care about its RegClass, so we should use an
15016 // IMPLICIT_DEF register.
15017 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImDefReg);
15018 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::INSERT_SUBREG), DestReg: ExtSrcReg)
15019 .addReg(RegNo: ImDefReg)
15020 .add(MO: SrcOp)
15021 .addImm(Val: PPC::sub_32);
15022
15023 Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
15024 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDIMI), DestReg: NewFPSCRTmpReg)
15025 .addReg(RegNo: OldFPSCRTmpReg)
15026 .addReg(RegNo: ExtSrcReg)
15027 .addImm(Val: 0)
15028 .addImm(Val: 62);
15029
15030 Register NewFPSCRReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
15031 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg);
15032
15033 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63
15034 // bits of FPSCR.
15035 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSF))
15036 .addImm(Val: 255)
15037 .addReg(RegNo: NewFPSCRReg)
15038 .addImm(Val: 0)
15039 .addImm(Val: 0);
15040 break;
15041 }
15042 case PPC::SETFLM: {
15043 DebugLoc Dl = MI.getDebugLoc();
15044
15045 // Result of setflm is previous FPSCR content, so we need to save it first.
15046 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
15047 if (MRI.use_empty(RegNo: OldFPSCRReg))
15048 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
15049 else
15050 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
15051
15052 // Put bits in 32:63 to FPSCR.
15053 Register NewFPSCRReg = MI.getOperand(i: 1).getReg();
15054 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MTFSF))
15055 .addImm(Val: 255)
15056 .addReg(RegNo: NewFPSCRReg)
15057 .addImm(Val: 0)
15058 .addImm(Val: 0);
15059 break;
15060 }
15061 case PPC::PROBED_ALLOCA_32:
15062 case PPC::PROBED_ALLOCA_64:
15063 return emitProbedAlloca(MI, MBB: BB);
15064
15065 case PPC::SPLIT_QUADWORD: {
15066 DebugLoc DL = MI.getDebugLoc();
15067 Register Src = MI.getOperand(i: 2).getReg();
15068 Register Lo = MI.getOperand(i: 0).getReg();
15069 Register Hi = MI.getOperand(i: 1).getReg();
15070 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
15071 .addDef(RegNo: Lo)
15072 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x1);
15073 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
15074 .addDef(RegNo: Hi)
15075 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x0);
15076 break;
15077 }
15078 case PPC::LQX_PSEUDO:
15079 case PPC::STQX_PSEUDO: {
15080 DebugLoc DL = MI.getDebugLoc();
15081 // Ptr is used as the ptr_rc_no_r0 part
15082 // of LQ/STQ's memory operand and adding result of RA and RB,
15083 // so it has to be g8rc_and_g8rc_nox0.
15084 Register Ptr =
15085 F->getRegInfo().createVirtualRegister(RegClass: &PPC::G8RC_and_G8RC_NOX0RegClass);
15086 Register Val = MI.getOperand(i: 0).getReg();
15087 Register RA = MI.getOperand(i: 1).getReg();
15088 Register RB = MI.getOperand(i: 2).getReg();
15089 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::ADD8), DestReg: Ptr).addReg(RegNo: RA).addReg(RegNo: RB);
15090 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
15091 MCID: MI.getOpcode() == PPC::LQX_PSEUDO ? TII->get(Opcode: PPC::LQ)
15092 : TII->get(Opcode: PPC::STQ))
15093 .addReg(RegNo: Val, Flags: getDefRegState(B: MI.getOpcode() == PPC::LQX_PSEUDO))
15094 .addImm(Val: 0)
15095 .addReg(RegNo: Ptr);
15096 break;
15097 }
15098 default:
15099 llvm_unreachable("Unexpected instr type to insert");
15100 }
15101
15102 MI.eraseFromParent(); // The pseudo instruction is gone now.
15103 return BB;
15104}
15105
15106//===----------------------------------------------------------------------===//
15107// Target Optimization Hooks
15108//===----------------------------------------------------------------------===//
15109
15110static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
15111 // For the estimates, convergence is quadratic, so we essentially double the
15112 // number of digits correct after every iteration. For both FRE and FRSQRTE,
15113 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
15114 // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
15115 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
15116 if (VT.getScalarType() == MVT::f64)
15117 RefinementSteps++;
15118 return RefinementSteps;
15119}
15120
15121SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
15122 const DenormalMode &Mode,
15123 SDNodeFlags Flags) const {
15124 // We only have VSX Vector Test for software Square Root.
15125 EVT VT = Op.getValueType();
15126 if (!isTypeLegal(VT: MVT::i1) ||
15127 (VT != MVT::f64 &&
15128 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())))
15129 return TargetLowering::getSqrtInputTest(Operand: Op, DAG, Mode, Flags);
15130
15131 SDLoc DL(Op);
15132 // The output register of FTSQRT is CR field.
15133 SDValue FTSQRT = DAG.getNode(Opcode: PPCISD::FTSQRT, DL, VT: MVT::i32, Operand: Op, Flags);
15134 // ftsqrt BF,FRB
15135 // Let e_b be the unbiased exponent of the double-precision
15136 // floating-point operand in register FRB.
15137 // fe_flag is set to 1 if either of the following conditions occurs.
15138 // - The double-precision floating-point operand in register FRB is a zero,
15139 // a NaN, or an infinity, or a negative value.
15140 // - e_b is less than or equal to -970.
15141 // Otherwise fe_flag is set to 0.
15142 // Both VSX and non-VSX versions would set EQ bit in the CR if the number is
15143 // not eligible for iteration. (zero/negative/infinity/nan or unbiased
15144 // exponent is less than -970)
15145 SDValue SRIdxVal = DAG.getTargetConstant(Val: PPC::sub_eq, DL, VT: MVT::i32);
15146 return SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, VT: MVT::i1,
15147 Op1: FTSQRT, Op2: SRIdxVal),
15148 0);
15149}
15150
15151SDValue
15152PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op,
15153 SelectionDAG &DAG) const {
15154 // We only have VSX Vector Square Root.
15155 EVT VT = Op.getValueType();
15156 if (VT != MVT::f64 &&
15157 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))
15158 return TargetLowering::getSqrtResultForDenormInput(Operand: Op, DAG);
15159
15160 return DAG.getNode(Opcode: PPCISD::FSQRT, DL: SDLoc(Op), VT, Operand: Op);
15161}
15162
15163SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
15164 int Enabled, int &RefinementSteps,
15165 bool &UseOneConstNR,
15166 bool Reciprocal) const {
15167 EVT VT = Operand.getValueType();
15168 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
15169 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
15170 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15171 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15172 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15173 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15174
15175 // The Newton-Raphson computation with a single constant does not provide
15176 // enough accuracy on some CPUs.
15177 UseOneConstNR = !Subtarget.needsTwoConstNR();
15178 return DAG.getNode(Opcode: PPCISD::FRSQRTE, DL: SDLoc(Operand), VT, Operand);
15179 }
15180 return SDValue();
15181}
15182
15183SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
15184 int Enabled,
15185 int &RefinementSteps) const {
15186 EVT VT = Operand.getValueType();
15187 if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
15188 (VT == MVT::f64 && Subtarget.hasFRE()) ||
15189 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15190 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15191 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15192 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15193 return DAG.getNode(Opcode: PPCISD::FRE, DL: SDLoc(Operand), VT, Operand);
15194 }
15195 return SDValue();
15196}
15197
15198unsigned PPCTargetLowering::combineRepeatedFPDivisors() const {
15199 // Note: This functionality is used only when arcp is enabled, and
15200 // on cores with reciprocal estimates (which are used when arcp is
15201 // enabled for division), this functionality is redundant with the default
15202 // combiner logic (once the division -> reciprocal/multiply transformation
15203 // has taken place). As a result, this matters more for older cores than for
15204 // newer ones.
15205
15206 // Combine multiple FDIVs with the same divisor into multiple FMULs by the
15207 // reciprocal if there are two or more FDIVs (for embedded cores with only
15208 // one FP pipeline) for three or more FDIVs (for generic OOO cores).
15209 switch (Subtarget.getCPUDirective()) {
15210 default:
15211 return 3;
15212 case PPC::DIR_440:
15213 case PPC::DIR_A2:
15214 case PPC::DIR_E500:
15215 case PPC::DIR_E500mc:
15216 case PPC::DIR_E5500:
15217 return 2;
15218 }
15219}
15220
15221// isConsecutiveLSLoc needs to work even if all adds have not yet been
15222// collapsed, and so we need to look through chains of them.
15223static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base,
15224 int64_t& Offset, SelectionDAG &DAG) {
15225 if (DAG.isBaseWithConstantOffset(Op: Loc)) {
15226 Base = Loc.getOperand(i: 0);
15227 Offset += cast<ConstantSDNode>(Val: Loc.getOperand(i: 1))->getSExtValue();
15228
15229 // The base might itself be a base plus an offset, and if so, accumulate
15230 // that as well.
15231 getBaseWithConstantOffset(Loc: Loc.getOperand(i: 0), Base, Offset, DAG);
15232 }
15233}
15234
15235static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base,
15236 unsigned Bytes, int Dist,
15237 SelectionDAG &DAG) {
15238 if (VT.getSizeInBits() / 8 != Bytes)
15239 return false;
15240
15241 SDValue BaseLoc = Base->getBasePtr();
15242 if (Loc.getOpcode() == ISD::FrameIndex) {
15243 if (BaseLoc.getOpcode() != ISD::FrameIndex)
15244 return false;
15245 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
15246 int FI = cast<FrameIndexSDNode>(Val&: Loc)->getIndex();
15247 int BFI = cast<FrameIndexSDNode>(Val&: BaseLoc)->getIndex();
15248 int FS = MFI.getObjectSize(ObjectIdx: FI);
15249 int BFS = MFI.getObjectSize(ObjectIdx: BFI);
15250 if (FS != BFS || FS != (int)Bytes) return false;
15251 return MFI.getObjectOffset(ObjectIdx: FI) == (MFI.getObjectOffset(ObjectIdx: BFI) + Dist*Bytes);
15252 }
15253
15254 SDValue Base1 = Loc, Base2 = BaseLoc;
15255 int64_t Offset1 = 0, Offset2 = 0;
15256 getBaseWithConstantOffset(Loc, Base&: Base1, Offset&: Offset1, DAG);
15257 getBaseWithConstantOffset(Loc: BaseLoc, Base&: Base2, Offset&: Offset2, DAG);
15258 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes))
15259 return true;
15260
15261 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15262 const GlobalValue *GV1 = nullptr;
15263 const GlobalValue *GV2 = nullptr;
15264 Offset1 = 0;
15265 Offset2 = 0;
15266 bool isGA1 = TLI.isGAPlusOffset(N: Loc.getNode(), GA&: GV1, Offset&: Offset1);
15267 bool isGA2 = TLI.isGAPlusOffset(N: BaseLoc.getNode(), GA&: GV2, Offset&: Offset2);
15268 if (isGA1 && isGA2 && GV1 == GV2)
15269 return Offset1 == (Offset2 + Dist*Bytes);
15270 return false;
15271}
15272
15273// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does
15274// not enforce equality of the chain operands.
15275static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
15276 unsigned Bytes, int Dist,
15277 SelectionDAG &DAG) {
15278 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Val: N)) {
15279 EVT VT = LS->getMemoryVT();
15280 SDValue Loc = LS->getBasePtr();
15281 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG);
15282 }
15283
15284 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
15285 EVT VT;
15286 switch (N->getConstantOperandVal(Num: 1)) {
15287 default: return false;
15288 case Intrinsic::ppc_altivec_lvx:
15289 case Intrinsic::ppc_altivec_lvxl:
15290 case Intrinsic::ppc_vsx_lxvw4x:
15291 case Intrinsic::ppc_vsx_lxvw4x_be:
15292 VT = MVT::v4i32;
15293 break;
15294 case Intrinsic::ppc_vsx_lxvd2x:
15295 case Intrinsic::ppc_vsx_lxvd2x_be:
15296 VT = MVT::v2f64;
15297 break;
15298 case Intrinsic::ppc_altivec_lvebx:
15299 VT = MVT::i8;
15300 break;
15301 case Intrinsic::ppc_altivec_lvehx:
15302 VT = MVT::i16;
15303 break;
15304 case Intrinsic::ppc_altivec_lvewx:
15305 VT = MVT::i32;
15306 break;
15307 }
15308
15309 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 2), VT, Base, Bytes, Dist, DAG);
15310 }
15311
15312 if (N->getOpcode() == ISD::INTRINSIC_VOID) {
15313 EVT VT;
15314 switch (N->getConstantOperandVal(Num: 1)) {
15315 default: return false;
15316 case Intrinsic::ppc_altivec_stvx:
15317 case Intrinsic::ppc_altivec_stvxl:
15318 case Intrinsic::ppc_vsx_stxvw4x:
15319 VT = MVT::v4i32;
15320 break;
15321 case Intrinsic::ppc_vsx_stxvd2x:
15322 VT = MVT::v2f64;
15323 break;
15324 case Intrinsic::ppc_vsx_stxvw4x_be:
15325 VT = MVT::v4i32;
15326 break;
15327 case Intrinsic::ppc_vsx_stxvd2x_be:
15328 VT = MVT::v2f64;
15329 break;
15330 case Intrinsic::ppc_altivec_stvebx:
15331 VT = MVT::i8;
15332 break;
15333 case Intrinsic::ppc_altivec_stvehx:
15334 VT = MVT::i16;
15335 break;
15336 case Intrinsic::ppc_altivec_stvewx:
15337 VT = MVT::i32;
15338 break;
15339 }
15340
15341 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 3), VT, Base, Bytes, Dist, DAG);
15342 }
15343
15344 return false;
15345}
15346
15347// Return true is there is a nearyby consecutive load to the one provided
15348// (regardless of alignment). We search up and down the chain, looking though
15349// token factors and other loads (but nothing else). As a result, a true result
15350// indicates that it is safe to create a new consecutive load adjacent to the
15351// load provided.
15352static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) {
15353 SDValue Chain = LD->getChain();
15354 EVT VT = LD->getMemoryVT();
15355
15356 SmallPtrSet<SDNode *, 16> LoadRoots;
15357 SmallVector<SDNode *, 8> Queue(1, Chain.getNode());
15358 SmallPtrSet<SDNode *, 16> Visited;
15359
15360 // First, search up the chain, branching to follow all token-factor operands.
15361 // If we find a consecutive load, then we're done, otherwise, record all
15362 // nodes just above the top-level loads and token factors.
15363 while (!Queue.empty()) {
15364 SDNode *ChainNext = Queue.pop_back_val();
15365 if (!Visited.insert(Ptr: ChainNext).second)
15366 continue;
15367
15368 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: ChainNext)) {
15369 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15370 return true;
15371
15372 if (!Visited.count(Ptr: ChainLD->getChain().getNode()))
15373 Queue.push_back(Elt: ChainLD->getChain().getNode());
15374 } else if (ChainNext->getOpcode() == ISD::TokenFactor) {
15375 for (const SDUse &O : ChainNext->ops())
15376 if (!Visited.count(Ptr: O.getNode()))
15377 Queue.push_back(Elt: O.getNode());
15378 } else
15379 LoadRoots.insert(Ptr: ChainNext);
15380 }
15381
15382 // Second, search down the chain, starting from the top-level nodes recorded
15383 // in the first phase. These top-level nodes are the nodes just above all
15384 // loads and token factors. Starting with their uses, recursively look though
15385 // all loads (just the chain uses) and token factors to find a consecutive
15386 // load.
15387 Visited.clear();
15388 Queue.clear();
15389
15390 for (SDNode *I : LoadRoots) {
15391 Queue.push_back(Elt: I);
15392
15393 while (!Queue.empty()) {
15394 SDNode *LoadRoot = Queue.pop_back_val();
15395 if (!Visited.insert(Ptr: LoadRoot).second)
15396 continue;
15397
15398 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: LoadRoot))
15399 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15400 return true;
15401
15402 for (SDNode *U : LoadRoot->users())
15403 if (((isa<MemSDNode>(Val: U) &&
15404 cast<MemSDNode>(Val: U)->getChain().getNode() == LoadRoot) ||
15405 U->getOpcode() == ISD::TokenFactor) &&
15406 !Visited.count(Ptr: U))
15407 Queue.push_back(Elt: U);
15408 }
15409 }
15410
15411 return false;
15412}
15413
15414/// This function is called when we have proved that a SETCC node can be replaced
15415/// by subtraction (and other supporting instructions) so that the result of
15416/// comparison is kept in a GPR instead of CR. This function is purely for
15417/// codegen purposes and has some flags to guide the codegen process.
15418static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement,
15419 bool Swap, SDLoc &DL, SelectionDAG &DAG) {
15420 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15421
15422 // Zero extend the operands to the largest legal integer. Originally, they
15423 // must be of a strictly smaller size.
15424 auto Op0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15425 auto Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15426
15427 // Swap if needed. Depends on the condition code.
15428 if (Swap)
15429 std::swap(a&: Op0, b&: Op1);
15430
15431 // Subtract extended integers.
15432 auto SubNode = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: Op0, N2: Op1);
15433
15434 // Move the sign bit to the least significant position and zero out the rest.
15435 // Now the least significant bit carries the result of original comparison.
15436 auto Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: SubNode,
15437 N2: DAG.getConstant(Val: Size - 1, DL, VT: MVT::i32));
15438 auto Final = Shifted;
15439
15440 // Complement the result if needed. Based on the condition code.
15441 if (Complement)
15442 Final = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, N1: Shifted,
15443 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i64));
15444
15445 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Final);
15446}
15447
15448SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N,
15449 DAGCombinerInfo &DCI) const {
15450 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15451
15452 SelectionDAG &DAG = DCI.DAG;
15453 SDLoc DL(N);
15454
15455 // Size of integers being compared has a critical role in the following
15456 // analysis, so we prefer to do this when all types are legal.
15457 if (!DCI.isAfterLegalizeDAG())
15458 return SDValue();
15459
15460 // If all users of SETCC extend its value to a legal integer type
15461 // then we replace SETCC with a subtraction
15462 for (const SDNode *U : N->users())
15463 if (U->getOpcode() != ISD::ZERO_EXTEND)
15464 return SDValue();
15465
15466 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
15467 auto OpSize = N->getOperand(Num: 0).getValueSizeInBits();
15468
15469 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits();
15470
15471 if (OpSize < Size) {
15472 switch (CC) {
15473 default: break;
15474 case ISD::SETULT:
15475 return generateEquivalentSub(N, Size, Complement: false, Swap: false, DL, DAG);
15476 case ISD::SETULE:
15477 return generateEquivalentSub(N, Size, Complement: true, Swap: true, DL, DAG);
15478 case ISD::SETUGT:
15479 return generateEquivalentSub(N, Size, Complement: false, Swap: true, DL, DAG);
15480 case ISD::SETUGE:
15481 return generateEquivalentSub(N, Size, Complement: true, Swap: false, DL, DAG);
15482 }
15483 }
15484
15485 return SDValue();
15486}
15487
15488SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
15489 DAGCombinerInfo &DCI) const {
15490 SelectionDAG &DAG = DCI.DAG;
15491 SDLoc dl(N);
15492
15493 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits");
15494 // If we're tracking CR bits, we need to be careful that we don't have:
15495 // trunc(binary-ops(zext(x), zext(y)))
15496 // or
15497 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...)
15498 // such that we're unnecessarily moving things into GPRs when it would be
15499 // better to keep them in CR bits.
15500
15501 // Note that trunc here can be an actual i1 trunc, or can be the effective
15502 // truncation that comes from a setcc or select_cc.
15503 if (N->getOpcode() == ISD::TRUNCATE &&
15504 N->getValueType(ResNo: 0) != MVT::i1)
15505 return SDValue();
15506
15507 if (N->getOperand(Num: 0).getValueType() != MVT::i32 &&
15508 N->getOperand(Num: 0).getValueType() != MVT::i64)
15509 return SDValue();
15510
15511 if (N->getOpcode() == ISD::SETCC ||
15512 N->getOpcode() == ISD::SELECT_CC) {
15513 // If we're looking at a comparison, then we need to make sure that the
15514 // high bits (all except for the first) don't matter the result.
15515 ISD::CondCode CC =
15516 cast<CondCodeSDNode>(Val: N->getOperand(
15517 Num: N->getOpcode() == ISD::SETCC ? 2 : 4))->get();
15518 unsigned OpBits = N->getOperand(Num: 0).getValueSizeInBits();
15519
15520 if (ISD::isSignedIntSetCC(Code: CC)) {
15521 if (DAG.ComputeNumSignBits(Op: N->getOperand(Num: 0)) != OpBits ||
15522 DAG.ComputeNumSignBits(Op: N->getOperand(Num: 1)) != OpBits)
15523 return SDValue();
15524 } else if (ISD::isUnsignedIntSetCC(Code: CC)) {
15525 if (!DAG.MaskedValueIsZero(Op: N->getOperand(Num: 0),
15526 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)) ||
15527 !DAG.MaskedValueIsZero(Op: N->getOperand(Num: 1),
15528 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)))
15529 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI)
15530 : SDValue());
15531 } else {
15532 // This is neither a signed nor an unsigned comparison, just make sure
15533 // that the high bits are equal.
15534 KnownBits Op1Known = DAG.computeKnownBits(Op: N->getOperand(Num: 0));
15535 KnownBits Op2Known = DAG.computeKnownBits(Op: N->getOperand(Num: 1));
15536
15537 // We don't really care about what is known about the first bit (if
15538 // anything), so pretend that it is known zero for both to ensure they can
15539 // be compared as constants.
15540 Op1Known.Zero.setBit(0); Op1Known.One.clearBit(BitPosition: 0);
15541 Op2Known.Zero.setBit(0); Op2Known.One.clearBit(BitPosition: 0);
15542
15543 if (!Op1Known.isConstant() || !Op2Known.isConstant() ||
15544 Op1Known.getConstant() != Op2Known.getConstant())
15545 return SDValue();
15546 }
15547 }
15548
15549 // We now know that the higher-order bits are irrelevant, we just need to
15550 // make sure that all of the intermediate operations are bit operations, and
15551 // all inputs are extensions.
15552 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15553 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15554 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15555 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15556 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC &&
15557 N->getOperand(Num: 0).getOpcode() != ISD::TRUNCATE &&
15558 N->getOperand(Num: 0).getOpcode() != ISD::SIGN_EXTEND &&
15559 N->getOperand(Num: 0).getOpcode() != ISD::ZERO_EXTEND &&
15560 N->getOperand(Num: 0).getOpcode() != ISD::ANY_EXTEND)
15561 return SDValue();
15562
15563 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) &&
15564 N->getOperand(Num: 1).getOpcode() != ISD::AND &&
15565 N->getOperand(Num: 1).getOpcode() != ISD::OR &&
15566 N->getOperand(Num: 1).getOpcode() != ISD::XOR &&
15567 N->getOperand(Num: 1).getOpcode() != ISD::SELECT &&
15568 N->getOperand(Num: 1).getOpcode() != ISD::SELECT_CC &&
15569 N->getOperand(Num: 1).getOpcode() != ISD::TRUNCATE &&
15570 N->getOperand(Num: 1).getOpcode() != ISD::SIGN_EXTEND &&
15571 N->getOperand(Num: 1).getOpcode() != ISD::ZERO_EXTEND &&
15572 N->getOperand(Num: 1).getOpcode() != ISD::ANY_EXTEND)
15573 return SDValue();
15574
15575 SmallVector<SDValue, 4> Inputs;
15576 SmallVector<SDValue, 8> BinOps, PromOps;
15577 SmallPtrSet<SDNode *, 16> Visited;
15578
15579 for (unsigned i = 0; i < 2; ++i) {
15580 if (((N->getOperand(Num: i).getOpcode() == ISD::SIGN_EXTEND ||
15581 N->getOperand(Num: i).getOpcode() == ISD::ZERO_EXTEND ||
15582 N->getOperand(Num: i).getOpcode() == ISD::ANY_EXTEND) &&
15583 N->getOperand(Num: i).getOperand(i: 0).getValueType() == MVT::i1) ||
15584 isa<ConstantSDNode>(Val: N->getOperand(Num: i)))
15585 Inputs.push_back(Elt: N->getOperand(Num: i));
15586 else
15587 BinOps.push_back(Elt: N->getOperand(Num: i));
15588
15589 if (N->getOpcode() == ISD::TRUNCATE)
15590 break;
15591 }
15592
15593 // Visit all inputs, collect all binary operations (and, or, xor and
15594 // select) that are all fed by extensions.
15595 while (!BinOps.empty()) {
15596 SDValue BinOp = BinOps.pop_back_val();
15597
15598 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15599 continue;
15600
15601 PromOps.push_back(Elt: BinOp);
15602
15603 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15604 // The condition of the select is not promoted.
15605 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15606 continue;
15607 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15608 continue;
15609
15610 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15611 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15612 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
15613 BinOp.getOperand(i).getOperand(i: 0).getValueType() == MVT::i1) ||
15614 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15615 Inputs.push_back(Elt: BinOp.getOperand(i));
15616 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15617 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15618 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15619 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15620 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC ||
15621 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15622 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15623 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15624 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) {
15625 BinOps.push_back(Elt: BinOp.getOperand(i));
15626 } else {
15627 // We have an input that is not an extension or another binary
15628 // operation; we'll abort this transformation.
15629 return SDValue();
15630 }
15631 }
15632 }
15633
15634 // Make sure that this is a self-contained cluster of operations (which
15635 // is not quite the same thing as saying that everything has only one
15636 // use).
15637 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15638 if (isa<ConstantSDNode>(Val: Inputs[i]))
15639 continue;
15640
15641 for (const SDNode *User : Inputs[i].getNode()->users()) {
15642 if (User != N && !Visited.count(Ptr: User))
15643 return SDValue();
15644
15645 // Make sure that we're not going to promote the non-output-value
15646 // operand(s) or SELECT or SELECT_CC.
15647 // FIXME: Although we could sometimes handle this, and it does occur in
15648 // practice that one of the condition inputs to the select is also one of
15649 // the outputs, we currently can't deal with this.
15650 if (User->getOpcode() == ISD::SELECT) {
15651 if (User->getOperand(Num: 0) == Inputs[i])
15652 return SDValue();
15653 } else if (User->getOpcode() == ISD::SELECT_CC) {
15654 if (User->getOperand(Num: 0) == Inputs[i] ||
15655 User->getOperand(Num: 1) == Inputs[i])
15656 return SDValue();
15657 }
15658 }
15659 }
15660
15661 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15662 for (const SDNode *User : PromOps[i].getNode()->users()) {
15663 if (User != N && !Visited.count(Ptr: User))
15664 return SDValue();
15665
15666 // Make sure that we're not going to promote the non-output-value
15667 // operand(s) or SELECT or SELECT_CC.
15668 // FIXME: Although we could sometimes handle this, and it does occur in
15669 // practice that one of the condition inputs to the select is also one of
15670 // the outputs, we currently can't deal with this.
15671 if (User->getOpcode() == ISD::SELECT) {
15672 if (User->getOperand(Num: 0) == PromOps[i])
15673 return SDValue();
15674 } else if (User->getOpcode() == ISD::SELECT_CC) {
15675 if (User->getOperand(Num: 0) == PromOps[i] ||
15676 User->getOperand(Num: 1) == PromOps[i])
15677 return SDValue();
15678 }
15679 }
15680 }
15681
15682 // Replace all inputs with the extension operand.
15683 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15684 // Constants may have users outside the cluster of to-be-promoted nodes,
15685 // and so we need to replace those as we do the promotions.
15686 if (isa<ConstantSDNode>(Val: Inputs[i]))
15687 continue;
15688 else
15689 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: Inputs[i].getOperand(i: 0));
15690 }
15691
15692 std::list<HandleSDNode> PromOpHandles;
15693 for (auto &PromOp : PromOps)
15694 PromOpHandles.emplace_back(args&: PromOp);
15695
15696 // Replace all operations (these are all the same, but have a different
15697 // (i1) return type). DAG.getNode will validate that the types of
15698 // a binary operator match, so go through the list in reverse so that
15699 // we've likely promoted both operands first. Any intermediate truncations or
15700 // extensions disappear.
15701 while (!PromOpHandles.empty()) {
15702 SDValue PromOp = PromOpHandles.back().getValue();
15703 PromOpHandles.pop_back();
15704
15705 if (PromOp.getOpcode() == ISD::TRUNCATE ||
15706 PromOp.getOpcode() == ISD::SIGN_EXTEND ||
15707 PromOp.getOpcode() == ISD::ZERO_EXTEND ||
15708 PromOp.getOpcode() == ISD::ANY_EXTEND) {
15709 if (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: 0)) &&
15710 PromOp.getOperand(i: 0).getValueType() != MVT::i1) {
15711 // The operand is not yet ready (see comment below).
15712 PromOpHandles.emplace_front(args&: PromOp);
15713 continue;
15714 }
15715
15716 SDValue RepValue = PromOp.getOperand(i: 0);
15717 if (isa<ConstantSDNode>(Val: RepValue))
15718 RepValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: RepValue);
15719
15720 DAG.ReplaceAllUsesOfValueWith(From: PromOp, To: RepValue);
15721 continue;
15722 }
15723
15724 unsigned C;
15725 switch (PromOp.getOpcode()) {
15726 default: C = 0; break;
15727 case ISD::SELECT: C = 1; break;
15728 case ISD::SELECT_CC: C = 2; break;
15729 }
15730
15731 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15732 PromOp.getOperand(i: C).getValueType() != MVT::i1) ||
15733 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15734 PromOp.getOperand(i: C+1).getValueType() != MVT::i1)) {
15735 // The to-be-promoted operands of this node have not yet been
15736 // promoted (this should be rare because we're going through the
15737 // list backward, but if one of the operands has several users in
15738 // this cluster of to-be-promoted nodes, it is possible).
15739 PromOpHandles.emplace_front(args&: PromOp);
15740 continue;
15741 }
15742
15743 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15744
15745 // If there are any constant inputs, make sure they're replaced now.
15746 for (unsigned i = 0; i < 2; ++i)
15747 if (isa<ConstantSDNode>(Val: Ops[C+i]))
15748 Ops[C+i] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: Ops[C+i]);
15749
15750 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
15751 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: MVT::i1, Ops));
15752 }
15753
15754 // Now we're left with the initial truncation itself.
15755 if (N->getOpcode() == ISD::TRUNCATE)
15756 return N->getOperand(Num: 0);
15757
15758 // Otherwise, this is a comparison. The operands to be compared have just
15759 // changed type (to i1), but everything else is the same.
15760 return SDValue(N, 0);
15761}
15762
15763SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
15764 DAGCombinerInfo &DCI) const {
15765 SelectionDAG &DAG = DCI.DAG;
15766 SDLoc dl(N);
15767
15768 // If we're tracking CR bits, we need to be careful that we don't have:
15769 // zext(binary-ops(trunc(x), trunc(y)))
15770 // or
15771 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...)
15772 // such that we're unnecessarily moving things into CR bits that can more
15773 // efficiently stay in GPRs. Note that if we're not certain that the high
15774 // bits are set as required by the final extension, we still may need to do
15775 // some masking to get the proper behavior.
15776
15777 // This same functionality is important on PPC64 when dealing with
15778 // 32-to-64-bit extensions; these occur often when 32-bit values are used as
15779 // the return values of functions. Because it is so similar, it is handled
15780 // here as well.
15781
15782 if (N->getValueType(ResNo: 0) != MVT::i32 &&
15783 N->getValueType(ResNo: 0) != MVT::i64)
15784 return SDValue();
15785
15786 if (!((N->getOperand(Num: 0).getValueType() == MVT::i1 && Subtarget.useCRBits()) ||
15787 (N->getOperand(Num: 0).getValueType() == MVT::i32 && Subtarget.isPPC64())))
15788 return SDValue();
15789
15790 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15791 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15792 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15793 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15794 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC)
15795 return SDValue();
15796
15797 SmallVector<SDValue, 4> Inputs;
15798 SmallVector<SDValue, 8> BinOps(1, N->getOperand(Num: 0)), PromOps;
15799 SmallPtrSet<SDNode *, 16> Visited;
15800
15801 // Visit all inputs, collect all binary operations (and, or, xor and
15802 // select) that are all fed by truncations.
15803 while (!BinOps.empty()) {
15804 SDValue BinOp = BinOps.pop_back_val();
15805
15806 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15807 continue;
15808
15809 PromOps.push_back(Elt: BinOp);
15810
15811 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15812 // The condition of the select is not promoted.
15813 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15814 continue;
15815 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15816 continue;
15817
15818 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15819 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15820 Inputs.push_back(Elt: BinOp.getOperand(i));
15821 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15822 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15823 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15824 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15825 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) {
15826 BinOps.push_back(Elt: BinOp.getOperand(i));
15827 } else {
15828 // We have an input that is not a truncation or another binary
15829 // operation; we'll abort this transformation.
15830 return SDValue();
15831 }
15832 }
15833 }
15834
15835 // The operands of a select that must be truncated when the select is
15836 // promoted because the operand is actually part of the to-be-promoted set.
15837 DenseMap<SDNode *, EVT> SelectTruncOp[2];
15838
15839 // Make sure that this is a self-contained cluster of operations (which
15840 // is not quite the same thing as saying that everything has only one
15841 // use).
15842 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15843 if (isa<ConstantSDNode>(Val: Inputs[i]))
15844 continue;
15845
15846 for (SDNode *User : Inputs[i].getNode()->users()) {
15847 if (User != N && !Visited.count(Ptr: User))
15848 return SDValue();
15849
15850 // If we're going to promote the non-output-value operand(s) or SELECT or
15851 // SELECT_CC, record them for truncation.
15852 if (User->getOpcode() == ISD::SELECT) {
15853 if (User->getOperand(Num: 0) == Inputs[i])
15854 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15855 y: User->getOperand(Num: 0).getValueType()));
15856 } else if (User->getOpcode() == ISD::SELECT_CC) {
15857 if (User->getOperand(Num: 0) == Inputs[i])
15858 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15859 y: User->getOperand(Num: 0).getValueType()));
15860 if (User->getOperand(Num: 1) == Inputs[i])
15861 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15862 y: User->getOperand(Num: 1).getValueType()));
15863 }
15864 }
15865 }
15866
15867 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15868 for (SDNode *User : PromOps[i].getNode()->users()) {
15869 if (User != N && !Visited.count(Ptr: User))
15870 return SDValue();
15871
15872 // If we're going to promote the non-output-value operand(s) or SELECT or
15873 // SELECT_CC, record them for truncation.
15874 if (User->getOpcode() == ISD::SELECT) {
15875 if (User->getOperand(Num: 0) == PromOps[i])
15876 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15877 y: User->getOperand(Num: 0).getValueType()));
15878 } else if (User->getOpcode() == ISD::SELECT_CC) {
15879 if (User->getOperand(Num: 0) == PromOps[i])
15880 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15881 y: User->getOperand(Num: 0).getValueType()));
15882 if (User->getOperand(Num: 1) == PromOps[i])
15883 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15884 y: User->getOperand(Num: 1).getValueType()));
15885 }
15886 }
15887 }
15888
15889 unsigned PromBits = N->getOperand(Num: 0).getValueSizeInBits();
15890 bool ReallyNeedsExt = false;
15891 if (N->getOpcode() != ISD::ANY_EXTEND) {
15892 // If all of the inputs are not already sign/zero extended, then
15893 // we'll still need to do that at the end.
15894 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15895 if (isa<ConstantSDNode>(Val: Inputs[i]))
15896 continue;
15897
15898 unsigned OpBits =
15899 Inputs[i].getOperand(i: 0).getValueSizeInBits();
15900 assert(PromBits < OpBits && "Truncation not to a smaller bit count?");
15901
15902 if ((N->getOpcode() == ISD::ZERO_EXTEND &&
15903 !DAG.MaskedValueIsZero(Op: Inputs[i].getOperand(i: 0),
15904 Mask: APInt::getHighBitsSet(numBits: OpBits,
15905 hiBitsSet: OpBits-PromBits))) ||
15906 (N->getOpcode() == ISD::SIGN_EXTEND &&
15907 DAG.ComputeNumSignBits(Op: Inputs[i].getOperand(i: 0)) <
15908 (OpBits-(PromBits-1)))) {
15909 ReallyNeedsExt = true;
15910 break;
15911 }
15912 }
15913 }
15914
15915 // Convert PromOps to handles before doing any RAUW operations, as these
15916 // may CSE with existing nodes, deleting the originals.
15917 std::list<HandleSDNode> PromOpHandles;
15918 for (auto &PromOp : PromOps)
15919 PromOpHandles.emplace_back(args&: PromOp);
15920
15921 // Replace all inputs, either with the truncation operand, or a
15922 // truncation or extension to the final output type.
15923 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15924 // Constant inputs need to be replaced with the to-be-promoted nodes that
15925 // use them because they might have users outside of the cluster of
15926 // promoted nodes.
15927 if (isa<ConstantSDNode>(Val: Inputs[i]))
15928 continue;
15929
15930 SDValue InSrc = Inputs[i].getOperand(i: 0);
15931 if (Inputs[i].getValueType() == N->getValueType(ResNo: 0))
15932 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: InSrc);
15933 else if (N->getOpcode() == ISD::SIGN_EXTEND)
15934 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15935 To: DAG.getSExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15936 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15937 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15938 To: DAG.getZExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15939 else
15940 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15941 To: DAG.getAnyExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15942 }
15943
15944 // Replace all operations (these are all the same, but have a different
15945 // (promoted) return type). DAG.getNode will validate that the types of
15946 // a binary operator match, so go through the list in reverse so that
15947 // we've likely promoted both operands first.
15948 while (!PromOpHandles.empty()) {
15949 SDValue PromOp = PromOpHandles.back().getValue();
15950 PromOpHandles.pop_back();
15951
15952 unsigned C;
15953 switch (PromOp.getOpcode()) {
15954 default: C = 0; break;
15955 case ISD::SELECT: C = 1; break;
15956 case ISD::SELECT_CC: C = 2; break;
15957 }
15958
15959 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15960 PromOp.getOperand(i: C).getValueType() != N->getValueType(ResNo: 0)) ||
15961 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15962 PromOp.getOperand(i: C+1).getValueType() != N->getValueType(ResNo: 0))) {
15963 // The to-be-promoted operands of this node have not yet been
15964 // promoted (this should be rare because we're going through the
15965 // list backward, but if one of the operands has several users in
15966 // this cluster of to-be-promoted nodes, it is possible).
15967 PromOpHandles.emplace_front(args&: PromOp);
15968 continue;
15969 }
15970
15971 // For SELECT and SELECT_CC nodes, we do a similar check for any
15972 // to-be-promoted comparison inputs.
15973 if (PromOp.getOpcode() == ISD::SELECT ||
15974 PromOp.getOpcode() == ISD::SELECT_CC) {
15975 if ((SelectTruncOp[0].count(Val: PromOp.getNode()) &&
15976 PromOp.getOperand(i: 0).getValueType() != N->getValueType(ResNo: 0)) ||
15977 (SelectTruncOp[1].count(Val: PromOp.getNode()) &&
15978 PromOp.getOperand(i: 1).getValueType() != N->getValueType(ResNo: 0))) {
15979 PromOpHandles.emplace_front(args&: PromOp);
15980 continue;
15981 }
15982 }
15983
15984 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15985
15986 // If this node has constant inputs, then they'll need to be promoted here.
15987 for (unsigned i = 0; i < 2; ++i) {
15988 if (!isa<ConstantSDNode>(Val: Ops[C+i]))
15989 continue;
15990 if (Ops[C+i].getValueType() == N->getValueType(ResNo: 0))
15991 continue;
15992
15993 if (N->getOpcode() == ISD::SIGN_EXTEND)
15994 Ops[C+i] = DAG.getSExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15995 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15996 Ops[C+i] = DAG.getZExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15997 else
15998 Ops[C+i] = DAG.getAnyExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15999 }
16000
16001 // If we've promoted the comparison inputs of a SELECT or SELECT_CC,
16002 // truncate them again to the original value type.
16003 if (PromOp.getOpcode() == ISD::SELECT ||
16004 PromOp.getOpcode() == ISD::SELECT_CC) {
16005 auto SI0 = SelectTruncOp[0].find(Val: PromOp.getNode());
16006 if (SI0 != SelectTruncOp[0].end())
16007 Ops[0] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI0->second, Operand: Ops[0]);
16008 auto SI1 = SelectTruncOp[1].find(Val: PromOp.getNode());
16009 if (SI1 != SelectTruncOp[1].end())
16010 Ops[1] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI1->second, Operand: Ops[1]);
16011 }
16012
16013 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
16014 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Ops));
16015 }
16016
16017 // Now we're left with the initial extension itself.
16018 if (!ReallyNeedsExt)
16019 return N->getOperand(Num: 0);
16020
16021 // To zero extend, just mask off everything except for the first bit (in the
16022 // i1 case).
16023 if (N->getOpcode() == ISD::ZERO_EXTEND)
16024 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
16025 N2: DAG.getConstant(Val: APInt::getLowBitsSet(
16026 numBits: N->getValueSizeInBits(ResNo: 0), loBitsSet: PromBits),
16027 DL: dl, VT: N->getValueType(ResNo: 0)));
16028
16029 assert(N->getOpcode() == ISD::SIGN_EXTEND &&
16030 "Invalid extension type");
16031 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: N->getValueType(ResNo: 0), DL: DAG.getDataLayout());
16032 SDValue ShiftCst =
16033 DAG.getConstant(Val: N->getValueSizeInBits(ResNo: 0) - PromBits, DL: dl, VT: ShiftAmountTy);
16034 return DAG.getNode(
16035 Opcode: ISD::SRA, DL: dl, VT: N->getValueType(ResNo: 0),
16036 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: ShiftCst),
16037 N2: ShiftCst);
16038}
16039
16040// The function check a i128 load can convert to 16i8 load for Vcmpequb.
16041static bool canConvertToVcmpequb(SDValue &LHS, SDValue &RHS, bool IsPPC64) {
16042
16043 auto isValidForConvert = [IsPPC64](SDValue &Operand) {
16044 if (!Operand.hasOneUse())
16045 return false;
16046
16047 if (Operand.getValueType() != MVT::i128)
16048 return false;
16049
16050 if (Operand.getOpcode() == ISD::Constant) {
16051 auto *C = cast<ConstantSDNode>(Val&: Operand);
16052 const APInt &Val = C->getAPIntValue();
16053 // On PPC64, comparing an i128 value loaded from memory against a
16054 // constant smaller than 2^16 is usually better left to scalar lowering.
16055 // In that case, the compare can be lowered using xori (since xori has a
16056 // 16-bit immediate field), which is cheaper than materializing a vector
16057 // constant and using vcmpequb.
16058 if (IsPPC64 && Val.ult(RHS: 1ULL << 16))
16059 return false;
16060 return true;
16061 }
16062
16063 auto *LoadNode = dyn_cast<LoadSDNode>(Val&: Operand);
16064 if (!LoadNode)
16065 return false;
16066
16067 // If memory operation is volatile, do not perform any
16068 // optimization or transformation. Volatile operations must be preserved
16069 // as written to ensure correct program behavior, so we return an empty
16070 // SDValue to indicate no action.
16071
16072 if (LoadNode->isVolatile())
16073 return false;
16074
16075 // Only combine loads if both use the unindexed addressing mode.
16076 // PowerPC AltiVec/VMX does not support vector loads or stores with
16077 // pre/post-increment addressing. Indexed modes may imply implicit
16078 // pointer updates, which are not compatible with AltiVec vector
16079 // instructions.
16080 if (LoadNode->getAddressingMode() != ISD::UNINDEXED)
16081 return false;
16082
16083 // Only combine loads if both are non-extending loads
16084 // (ISD::NON_EXTLOAD). Extending loads (such as ISD::ZEXTLOAD or
16085 // ISD::SEXTLOAD) perform zero or sign extension, which may change the
16086 // loaded value's semantics and are not compatible with vector loads.
16087 if (LoadNode->getExtensionType() != ISD::NON_EXTLOAD)
16088 return false;
16089
16090 return true;
16091 };
16092
16093 return (isValidForConvert(LHS) && isValidForConvert(RHS));
16094}
16095
16096SDValue convertTwoLoadsAndCmpToVCMPEQUB(SelectionDAG &DAG, SDNode *N,
16097 const SDLoc &DL) {
16098
16099 assert(N->getOpcode() == ISD::SETCC && "Should be called with a SETCC node");
16100
16101 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16102 assert((CC == ISD::SETNE || CC == ISD::SETEQ) &&
16103 "CC mus be ISD::SETNE or ISD::SETEQ");
16104
16105 auto getV16i8Load = [&](const SDValue &Operand) {
16106 if (Operand.getOpcode() == ISD::Constant)
16107 return DAG.getBitcast(VT: MVT::v16i8, V: Operand);
16108
16109 assert(Operand.getOpcode() == ISD::LOAD && "Must be LoadSDNode here.");
16110
16111 auto *LoadNode = cast<LoadSDNode>(Val: Operand);
16112 // Create a new MachineMemOperand without range metadata.
16113 // Range metadata is only valid for integer scalar types, not vectors.
16114 // The original i128 load may have range metadata, but when we convert
16115 // to v16i8, that metadata is no longer semantically valid.
16116 MachineMemOperand *MMO = LoadNode->getMemOperand();
16117 MachineFunction &MF = DAG.getMachineFunction();
16118 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
16119 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: MMO->getSize(), BaseAlignment: MMO->getAlign(),
16120 Metadata: MMO->getAAInfo(), SSID: MMO->getSyncScopeID(), Ordering: MMO->getSuccessOrdering(),
16121 FailureOrdering: MMO->getFailureOrdering());
16122 SDValue NewLoad = DAG.getLoad(VT: MVT::v16i8, dl: DL, Chain: LoadNode->getChain(),
16123 Ptr: LoadNode->getBasePtr(), MMO: NewMMO);
16124 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadNode, 1), To: NewLoad.getValue(R: 1));
16125 return NewLoad;
16126 };
16127
16128 // Following code transforms the DAG
16129 // t0: ch,glue = EntryToken
16130 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16131 // t3: i128,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16132 // undef:i64
16133 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16134 // t5: i128,ch =
16135 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64 t6: i1 =
16136 // setcc t3, t5, setne:ch
16137 //
16138 // ---->
16139 //
16140 // t0: ch,glue = EntryToken
16141 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16142 // t3: v16i8,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16143 // undef:i64
16144 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16145 // t5: v16i8,ch =
16146 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64
16147 // t6: i32 =
16148 // llvm.ppc.altivec.vcmpequb.p TargetConstant:i32<10505>,
16149 // Constant:i32<2>, t3, t5
16150 // t7: i1 = setcc t6, Constant:i32<0>, seteq:ch
16151
16152 // Or transforms the DAG
16153 // t5: i128,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16154 // t8: i1 =
16155 // setcc Constant:i128<237684487579686500932345921536>, t5, setne:ch
16156 //
16157 // --->
16158 //
16159 // t5: v16i8,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16160 // t6: v16i8 = bitcast Constant:i128<237684487579686500932345921536>
16161 // t7: i32 =
16162 // llvm.ppc.altivec.vcmpequb.p Constant:i32<10962>, Constant:i32<2>, t5, t2
16163
16164 SDValue LHSVec = getV16i8Load(N->getOperand(Num: 0));
16165 SDValue RHSVec = getV16i8Load(N->getOperand(Num: 1));
16166
16167 SDValue IntrID =
16168 DAG.getConstant(Val: Intrinsic::ppc_altivec_vcmpequb_p, DL, VT: MVT::i32);
16169 SDValue CRSel = DAG.getConstant(Val: 2, DL, VT: MVT::i32); // which CR6 predicate field
16170 SDValue PredResult = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
16171 N1: IntrID, N2: CRSel, N3: LHSVec, N4: RHSVec);
16172 // ppc_altivec_vcmpequb_p returns 1 when two vectors are the same,
16173 // so we need to invert the CC opcode.
16174 return DAG.getSetCC(DL, VT: N->getValueType(ResNo: 0), LHS: PredResult,
16175 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32),
16176 Cond: CC == ISD::SETNE ? ISD::SETEQ : ISD::SETNE);
16177}
16178
16179// Detect whether there is a pattern like (setcc (and X, 1), 0, eq).
16180// If it is , return true; otherwise return false.
16181static bool canConvertSETCCToXori(SDNode *N) {
16182 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16183
16184 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16185 if (CC != ISD::SETEQ)
16186 return false;
16187
16188 SDValue LHS = N->getOperand(Num: 0);
16189 SDValue RHS = N->getOperand(Num: 1);
16190
16191 // Check the `SDValue &V` is from `and` with `1`.
16192 auto IsAndWithOne = [](SDValue &V) {
16193 if (V.getOpcode() == ISD::AND) {
16194 for (const SDValue &Op : V->ops())
16195 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op))
16196 if (C->isOne())
16197 return true;
16198 }
16199 return false;
16200 };
16201
16202 // Check whether the SETCC compare with zero.
16203 auto IsCompareWithZero = [](SDValue &V) {
16204 if (auto *C = dyn_cast<ConstantSDNode>(Val&: V))
16205 if (C->isZero())
16206 return true;
16207 return false;
16208 };
16209
16210 return (IsAndWithOne(LHS) && IsCompareWithZero(RHS)) ||
16211 (IsAndWithOne(RHS) && IsCompareWithZero(LHS));
16212}
16213
16214// You must check whether the `SDNode* N` can be converted to Xori using
16215// the function `static bool canConvertSETCCToXori(SDNode *N)`
16216// before calling the function; otherwise, it may produce incorrect results.
16217static SDValue ConvertSETCCToXori(SDNode *N, SelectionDAG &DAG) {
16218
16219 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16220 SDValue LHS = N->getOperand(Num: 0);
16221 SDValue RHS = N->getOperand(Num: 1);
16222 SDLoc DL(N);
16223
16224 [[maybe_unused]] ISD::CondCode CC =
16225 cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16226 assert((CC == ISD::SETEQ) && "CC must be ISD::SETEQ.");
16227 // Rewrite it as XORI (and X, 1), 1.
16228 auto MakeXor1 = [&](SDValue V) {
16229 EVT VT = V.getValueType();
16230 SDValue One = DAG.getConstant(Val: 1, DL, VT);
16231 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: V, N2: One);
16232 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Xor);
16233 };
16234
16235 if (LHS.getOpcode() == ISD::AND && RHS.getOpcode() != ISD::AND)
16236 return MakeXor1(LHS);
16237
16238 if (RHS.getOpcode() == ISD::AND && LHS.getOpcode() != ISD::AND)
16239 return MakeXor1(RHS);
16240
16241 llvm_unreachable("Should not reach here.");
16242}
16243
16244// Match `sext(setcc X, 0, eq)` and turn it into an ADDIC/SUBFE sequence.
16245//
16246// This generates code for:
16247// X == 0 ? -1 : 0
16248//
16249// On pre-ISA 3.1 targets, this is better than the longer CNTLZW/SRWI/NEG
16250// sequence. This is useful for cases like:
16251// uint8_t f(uint8_t x) { return (x == 0) ? -1 : 0; }
16252//
16253// ISA 3.1+ is skipped because those targets can use SETBC.
16254
16255SDValue PPCTargetLowering::combineSignExtendSetCC(SDNode *N,
16256 DAGCombinerInfo &DCI) const {
16257 if (Subtarget.isISA3_1())
16258 return SDValue();
16259
16260 EVT VT = N->getValueType(ResNo: 0);
16261 if (VT != MVT::i32 && (VT != MVT::i64 || !Subtarget.isPPC64()))
16262 return SDValue();
16263
16264 SDValue N0 = N->getOperand(Num: 0);
16265 if (N0.getOpcode() != ISD::SETCC)
16266 return SDValue();
16267
16268 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
16269 SDValue LHS = N0.getOperand(i: 0);
16270 SDValue RHS = N0.getOperand(i: 1);
16271
16272 // Not match: sext (setcc x, 0, eq) or sext (setcc 0, x, eq)
16273 if (CC != ISD::SETEQ || (!isNullConstant(V: LHS) && !isNullConstant(V: RHS)))
16274 return SDValue();
16275
16276 SDLoc dl(N);
16277 SelectionDAG &DAG = DCI.DAG;
16278 SDValue X = isNullConstant(V: LHS) ? RHS : LHS;
16279 EVT XVT = X.getValueType(); // The type of x in the setcc x, 0, eq.
16280
16281 // The type that ADDC/SUBE operate on. Reject larger types and zero-extend
16282 // smaller ones.
16283 MVT OpVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
16284 if (XVT.bitsGT(VT: OpVT))
16285 return SDValue();
16286
16287 if (XVT.bitsLT(VT: OpVT))
16288 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: OpVT, Operand: X);
16289
16290 // Generate: SUBFE(ADDC(X, -1)).
16291 SDValue MinusOne = DAG.getAllOnesConstant(DL: dl, VT: OpVT);
16292 SDValue Addc =
16293 DAG.getNode(Opcode: PPCISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32), N1: X, N2: MinusOne);
16294 SDValue Carry = Addc.getValue(R: 1);
16295 SDValue Sube = DAG.getNode(Opcode: PPCISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32),
16296 N1: Addc, N2: Addc, N3: Carry);
16297
16298 // Truncate back to i32 if we used i64 operations.
16299 if (OpVT == MVT::i64 && VT == MVT::i32)
16300 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Sube);
16301
16302 return Sube;
16303}
16304
16305SDValue PPCTargetLowering::combineSetCC(SDNode *N,
16306 DAGCombinerInfo &DCI) const {
16307 assert(N->getOpcode() == ISD::SETCC &&
16308 "Should be called with a SETCC node");
16309
16310 // Check if the pattern (setcc (and X, 1), 0, eq) is present.
16311 // If it is, rewrite it as XORI (and X, 1), 1.
16312 if (canConvertSETCCToXori(N))
16313 return ConvertSETCCToXori(N, DAG&: DCI.DAG);
16314
16315 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16316 if (CC == ISD::SETNE || CC == ISD::SETEQ) {
16317 SDValue LHS = N->getOperand(Num: 0);
16318 SDValue RHS = N->getOperand(Num: 1);
16319
16320 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS.
16321 if (LHS.getOpcode() == ISD::SUB && isNullConstant(V: LHS.getOperand(i: 0)) &&
16322 LHS.hasOneUse())
16323 std::swap(a&: LHS, b&: RHS);
16324
16325 // x == 0-y --> x+y == 0
16326 // x != 0-y --> x+y != 0
16327 if (RHS.getOpcode() == ISD::SUB && isNullConstant(V: RHS.getOperand(i: 0)) &&
16328 RHS.hasOneUse()) {
16329 SDLoc DL(N);
16330 SelectionDAG &DAG = DCI.DAG;
16331 EVT VT = N->getValueType(ResNo: 0);
16332 EVT OpVT = LHS.getValueType();
16333 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: OpVT, N1: LHS, N2: RHS.getOperand(i: 1));
16334 return DAG.getSetCC(DL, VT, LHS: Add, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond: CC);
16335 }
16336
16337 // Optimization: Fold i128 equality/inequality compares of two loads into a
16338 // vectorized compare using vcmpequb.p when Altivec is available.
16339 //
16340 // Rationale:
16341 // A scalar i128 SETCC (eq/ne) normally lowers to multiple scalar ops.
16342 // On VSX-capable subtargets, we can instead reinterpret the i128 loads
16343 // as v16i8 vectors and use the Altive vcmpequb.p instruction to
16344 // perform a full 128-bit equality check in a single vector compare.
16345 //
16346 // Example Result:
16347 // This transformation replaces memcmp(a, b, 16) with two vector loads
16348 // and one vector compare instruction.
16349
16350 if (Subtarget.hasAltivec() &&
16351 canConvertToVcmpequb(LHS, RHS, IsPPC64: Subtarget.isPPC64()))
16352 return convertTwoLoadsAndCmpToVCMPEQUB(DAG&: DCI.DAG, N, DL: SDLoc(N));
16353 }
16354
16355 return DAGCombineTruncBoolExt(N, DCI);
16356}
16357
16358// Is this an extending load from an f32 to an f64?
16359static bool isFPExtLoad(SDValue Op) {
16360 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Op.getNode()))
16361 return LD->getExtensionType() == ISD::EXTLOAD &&
16362 Op.getValueType() == MVT::f64;
16363 return false;
16364}
16365
16366/// Reduces the number of fp-to-int conversion when building a vector.
16367///
16368/// If this vector is built out of floating to integer conversions,
16369/// transform it to a vector built out of floating point values followed by a
16370/// single floating to integer conversion of the vector.
16371/// Namely (build_vector (fptosi $A), (fptosi $B), ...)
16372/// becomes (fptosi (build_vector ($A, $B, ...)))
16373SDValue PPCTargetLowering::
16374combineElementTruncationToVectorTruncation(SDNode *N,
16375 DAGCombinerInfo &DCI) const {
16376 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16377 "Should be called with a BUILD_VECTOR node");
16378
16379 SelectionDAG &DAG = DCI.DAG;
16380 SDLoc dl(N);
16381
16382 SDValue FirstInput = N->getOperand(Num: 0);
16383 assert(FirstInput.getOpcode() == PPCISD::MFVSR &&
16384 "The input operand must be an fp-to-int conversion.");
16385
16386 // This combine happens after legalization so the fp_to_[su]i nodes are
16387 // already converted to PPCSISD nodes.
16388 unsigned FirstConversion = FirstInput.getOperand(i: 0).getOpcode();
16389 if (FirstConversion == PPCISD::FCTIDZ ||
16390 FirstConversion == PPCISD::FCTIDUZ ||
16391 FirstConversion == PPCISD::FCTIWZ ||
16392 FirstConversion == PPCISD::FCTIWUZ) {
16393 bool IsSplat = true;
16394 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ ||
16395 FirstConversion == PPCISD::FCTIWUZ;
16396 EVT SrcVT = FirstInput.getOperand(i: 0).getValueType();
16397 SmallVector<SDValue, 4> Ops;
16398 EVT TargetVT = N->getValueType(ResNo: 0);
16399 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16400 SDValue NextOp = N->getOperand(Num: i);
16401 if (NextOp.getOpcode() != PPCISD::MFVSR)
16402 return SDValue();
16403 unsigned NextConversion = NextOp.getOperand(i: 0).getOpcode();
16404 if (NextConversion != FirstConversion)
16405 return SDValue();
16406 // If we are converting to 32-bit integers, we need to add an FP_ROUND.
16407 // This is not valid if the input was originally double precision. It is
16408 // also not profitable to do unless this is an extending load in which
16409 // case doing this combine will allow us to combine consecutive loads.
16410 if (Is32Bit && !isFPExtLoad(Op: NextOp.getOperand(i: 0).getOperand(i: 0)))
16411 return SDValue();
16412 if (N->getOperand(Num: i) != FirstInput)
16413 IsSplat = false;
16414 }
16415
16416 // If this is a splat, we leave it as-is since there will be only a single
16417 // fp-to-int conversion followed by a splat of the integer. This is better
16418 // for 32-bit and smaller ints and neutral for 64-bit ints.
16419 if (IsSplat)
16420 return SDValue();
16421
16422 // Now that we know we have the right type of node, get its operands
16423 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16424 SDValue In = N->getOperand(Num: i).getOperand(i: 0);
16425 if (Is32Bit) {
16426 // For 32-bit values, we need to add an FP_ROUND node (if we made it
16427 // here, we know that all inputs are extending loads so this is safe).
16428 if (In.isUndef())
16429 Ops.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
16430 else {
16431 SDValue Trunc =
16432 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: In.getOperand(i: 0),
16433 N2: DAG.getIntPtrConstant(Val: 1, DL: dl, /*isTarget=*/true));
16434 Ops.push_back(Elt: Trunc);
16435 }
16436 } else
16437 Ops.push_back(Elt: In.isUndef() ? DAG.getUNDEF(VT: SrcVT) : In.getOperand(i: 0));
16438 }
16439
16440 unsigned Opcode;
16441 if (FirstConversion == PPCISD::FCTIDZ ||
16442 FirstConversion == PPCISD::FCTIWZ)
16443 Opcode = ISD::FP_TO_SINT;
16444 else
16445 Opcode = ISD::FP_TO_UINT;
16446
16447 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32;
16448 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: dl, Ops);
16449 return DAG.getNode(Opcode, DL: dl, VT: TargetVT, Operand: BV);
16450 }
16451 return SDValue();
16452}
16453
16454// LXVKQ instruction load VSX vector with a special quadword value
16455// based on an immediate value. This helper method returns the details of the
16456// match as a tuple of {LXVKQ unsigned IMM Value, right_shift_amount}
16457// to help generate the LXVKQ instruction and the subsequent shift instruction
16458// required to match the original build vector pattern.
16459
16460// LXVKQPattern: {LXVKQ unsigned IMM Value, right_shift_amount}
16461using LXVKQPattern = std::tuple<uint32_t, uint8_t>;
16462
16463static std::optional<LXVKQPattern> getPatternInfo(const APInt &FullVal) {
16464
16465 // LXVKQ instruction loads the Quadword value:
16466 // 0x8000_0000_0000_0000_0000_0000_0000_0000 when imm = 0b10000
16467 static const APInt BasePattern = APInt(128, 0x8000000000000000ULL) << 64;
16468 static const uint32_t Uim = 16;
16469
16470 // Check for direct LXVKQ match (no shift needed)
16471 if (FullVal == BasePattern)
16472 return std::make_tuple(args: Uim, args: uint8_t{0});
16473
16474 // Check if FullValue is 1 (the result of the base pattern >> 127)
16475 if (FullVal == APInt(128, 1))
16476 return std::make_tuple(args: Uim, args: uint8_t{127});
16477
16478 return std::nullopt;
16479}
16480
16481/// Combine vector loads to a single load (using lxvkq) or splat with shift of a
16482/// constant (xxspltib + vsrq) by recognising patterns in the Build Vector.
16483/// LXVKQ instruction load VSX vector with a special quadword value based on an
16484/// immediate value. if UIM=0b10000 then LXVKQ loads VSR[32×TX+T] with value
16485/// 0x8000_0000_0000_0000_0000_0000_0000_0000.
16486/// This can be used to inline the build vector constants that have the
16487/// following patterns:
16488///
16489/// 0x8000_0000_0000_0000_0000_0000_0000_0000 (MSB set pattern)
16490/// 0x0000_0000_0000_0000_0000_0000_0000_0001 (LSB set pattern)
16491/// MSB pattern can directly loaded using LXVKQ while LSB is loaded using a
16492/// combination of splatting and right shift instructions.
16493
16494SDValue PPCTargetLowering::combineBVLoadsSpecialValue(SDValue Op,
16495 SelectionDAG &DAG) const {
16496
16497 assert((Op.getNode() && Op.getOpcode() == ISD::BUILD_VECTOR) &&
16498 "Expected a BuildVectorSDNode in combineBVLoadsSpecialValue");
16499
16500 // This transformation is only supported if we are loading either a byte,
16501 // halfword, word, or doubleword.
16502 EVT VT = Op.getValueType();
16503 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
16504 VT == MVT::v2i64))
16505 return SDValue();
16506
16507 LLVM_DEBUG(llvm::dbgs() << "\ncombineBVLoadsSpecialValue: Build vector ("
16508 << VT.getEVTString() << "): ";
16509 Op->dump());
16510
16511 unsigned NumElems = VT.getVectorNumElements();
16512 unsigned ElemBits = VT.getScalarSizeInBits();
16513
16514 bool IsLittleEndian = DAG.getDataLayout().isLittleEndian();
16515
16516 // Check for Non-constant operand in the build vector.
16517 for (const SDValue &Operand : Op.getNode()->op_values()) {
16518 if (!isa<ConstantSDNode>(Val: Operand))
16519 return SDValue();
16520 }
16521
16522 // Assemble build vector operands as a 128-bit register value
16523 // We need to reconstruct what the 128-bit register pattern would be
16524 // that produces this vector when interpreted with the current endianness
16525 APInt FullVal = APInt::getZero(numBits: 128);
16526
16527 for (unsigned Index = 0; Index < NumElems; ++Index) {
16528 auto *C = cast<ConstantSDNode>(Val: Op.getOperand(i: Index));
16529
16530 // Get element value as raw bits (zero-extended)
16531 uint64_t ElemValue = C->getZExtValue();
16532
16533 // Mask to element size to ensure we only get the relevant bits
16534 if (ElemBits < 64)
16535 ElemValue &= ((1ULL << ElemBits) - 1);
16536
16537 // Calculate bit position for this element in the 128-bit register
16538 unsigned BitPos =
16539 (IsLittleEndian) ? (Index * ElemBits) : (128 - (Index + 1) * ElemBits);
16540
16541 // Create APInt for the element value and shift it to correct position
16542 APInt ElemAPInt(128, ElemValue);
16543 ElemAPInt <<= BitPos;
16544
16545 // Place the element value at the correct bit position
16546 FullVal |= ElemAPInt;
16547 }
16548
16549 if (FullVal.isZero() || FullVal.isAllOnes())
16550 return SDValue();
16551
16552 if (auto UIMOpt = getPatternInfo(FullVal)) {
16553 const auto &[Uim, ShiftAmount] = *UIMOpt;
16554 SDLoc Dl(Op);
16555
16556 // Generate LXVKQ instruction if the shift amount is zero.
16557 if (ShiftAmount == 0) {
16558 SDValue UimVal = DAG.getTargetConstant(Val: Uim, DL: Dl, VT: MVT::i32);
16559 SDValue LxvkqInstr =
16560 SDValue(DAG.getMachineNode(Opcode: PPC::LXVKQ, dl: Dl, VT, Op1: UimVal), 0);
16561 LLVM_DEBUG(llvm::dbgs()
16562 << "combineBVLoadsSpecialValue: Instruction Emitted ";
16563 LxvkqInstr.dump());
16564 return LxvkqInstr;
16565 }
16566
16567 assert(ShiftAmount == 127 && "Unexpected lxvkq shift amount value");
16568
16569 // The right shifted pattern can be constructed using a combination of
16570 // XXSPLTIB and VSRQ instruction. VSRQ uses the shift amount from the lower
16571 // 7 bits of byte 15. This can be specified using XXSPLTIB with immediate
16572 // value 255.
16573 SDValue ShiftAmountVec =
16574 SDValue(DAG.getMachineNode(Opcode: PPC::XXSPLTIB, dl: Dl, VT: MVT::v4i32,
16575 Op1: DAG.getTargetConstant(Val: 255, DL: Dl, VT: MVT::i32)),
16576 0);
16577 // Generate appropriate right shift instruction
16578 SDValue ShiftVec = SDValue(
16579 DAG.getMachineNode(Opcode: PPC::VSRQ, dl: Dl, VT, Op1: ShiftAmountVec, Op2: ShiftAmountVec),
16580 0);
16581 LLVM_DEBUG(llvm::dbgs()
16582 << "\n combineBVLoadsSpecialValue: Instruction Emitted ";
16583 ShiftVec.dump());
16584 return ShiftVec;
16585 }
16586 // No patterns matched for build vectors.
16587 return SDValue();
16588}
16589
16590/// Reduce the number of loads when building a vector.
16591///
16592/// Building a vector out of multiple loads can be converted to a load
16593/// of the vector type if the loads are consecutive. If the loads are
16594/// consecutive but in descending order, a shuffle is added at the end
16595/// to reorder the vector.
16596static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) {
16597 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16598 "Should be called with a BUILD_VECTOR node");
16599
16600 SDLoc dl(N);
16601
16602 // Return early for non byte-sized type, as they can't be consecutive.
16603 if (!N->getValueType(ResNo: 0).getVectorElementType().isByteSized())
16604 return SDValue();
16605
16606 bool InputsAreConsecutiveLoads = true;
16607 bool InputsAreReverseConsecutive = true;
16608 unsigned ElemSize = N->getValueType(ResNo: 0).getScalarType().getStoreSize();
16609 SDValue FirstInput = N->getOperand(Num: 0);
16610 bool IsRoundOfExtLoad = false;
16611 LoadSDNode *FirstLoad = nullptr;
16612
16613 if (FirstInput.getOpcode() == ISD::FP_ROUND &&
16614 FirstInput.getOperand(i: 0).getOpcode() == ISD::LOAD) {
16615 FirstLoad = cast<LoadSDNode>(Val: FirstInput.getOperand(i: 0));
16616 IsRoundOfExtLoad = FirstLoad->getExtensionType() == ISD::EXTLOAD;
16617 }
16618 // Not a build vector of (possibly fp_rounded) loads.
16619 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) ||
16620 N->getNumOperands() == 1)
16621 return SDValue();
16622
16623 if (!IsRoundOfExtLoad)
16624 FirstLoad = cast<LoadSDNode>(Val&: FirstInput);
16625
16626 SmallVector<LoadSDNode *, 4> InputLoads;
16627 InputLoads.push_back(Elt: FirstLoad);
16628 for (int i = 1, e = N->getNumOperands(); i < e; ++i) {
16629 // If any inputs are fp_round(extload), they all must be.
16630 if (IsRoundOfExtLoad && N->getOperand(Num: i).getOpcode() != ISD::FP_ROUND)
16631 return SDValue();
16632
16633 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(Num: i).getOperand(i: 0) :
16634 N->getOperand(Num: i);
16635 if (NextInput.getOpcode() != ISD::LOAD)
16636 return SDValue();
16637
16638 SDValue PreviousInput =
16639 IsRoundOfExtLoad ? N->getOperand(Num: i-1).getOperand(i: 0) : N->getOperand(Num: i-1);
16640 LoadSDNode *LD1 = cast<LoadSDNode>(Val&: PreviousInput);
16641 LoadSDNode *LD2 = cast<LoadSDNode>(Val&: NextInput);
16642
16643 // If any inputs are fp_round(extload), they all must be.
16644 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD)
16645 return SDValue();
16646
16647 // We only care about regular loads. The PPC-specific load intrinsics
16648 // will not lead to a merge opportunity.
16649 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD2, Base: LD1, Bytes: ElemSize, Dist: 1))
16650 InputsAreConsecutiveLoads = false;
16651 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD1, Base: LD2, Bytes: ElemSize, Dist: 1))
16652 InputsAreReverseConsecutive = false;
16653
16654 // Exit early if the loads are neither consecutive nor reverse consecutive.
16655 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive)
16656 return SDValue();
16657 InputLoads.push_back(Elt: LD2);
16658 }
16659
16660 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) &&
16661 "The loads cannot be both consecutive and reverse consecutive.");
16662
16663 SDValue WideLoad;
16664 SDValue ReturnSDVal;
16665 if (InputsAreConsecutiveLoads) {
16666 assert(FirstLoad && "Input needs to be a LoadSDNode.");
16667 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: FirstLoad->getChain(),
16668 Ptr: FirstLoad->getBasePtr(), PtrInfo: FirstLoad->getPointerInfo(),
16669 Alignment: FirstLoad->getAlign());
16670 ReturnSDVal = WideLoad;
16671 } else if (InputsAreReverseConsecutive) {
16672 LoadSDNode *LastLoad = InputLoads.back();
16673 assert(LastLoad && "Input needs to be a LoadSDNode.");
16674 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: LastLoad->getChain(),
16675 Ptr: LastLoad->getBasePtr(), PtrInfo: LastLoad->getPointerInfo(),
16676 Alignment: LastLoad->getAlign());
16677 SmallVector<int, 16> Ops;
16678 for (int i = N->getNumOperands() - 1; i >= 0; i--)
16679 Ops.push_back(Elt: i);
16680
16681 ReturnSDVal = DAG.getVectorShuffle(VT: N->getValueType(ResNo: 0), dl, N1: WideLoad,
16682 N2: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)), Mask: Ops);
16683 } else
16684 return SDValue();
16685
16686 for (auto *LD : InputLoads)
16687 DAG.makeEquivalentMemoryOrdering(OldLoad: LD, NewMemOp: WideLoad);
16688 return ReturnSDVal;
16689}
16690
16691// This function adds the required vector_shuffle needed to get
16692// the elements of the vector extract in the correct position
16693// as specified by the CorrectElems encoding.
16694static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG,
16695 SDValue Input, uint64_t Elems,
16696 uint64_t CorrectElems) {
16697 SDLoc dl(N);
16698
16699 unsigned NumElems = Input.getValueType().getVectorNumElements();
16700 SmallVector<int, 16> ShuffleMask(NumElems, -1);
16701
16702 // Knowing the element indices being extracted from the original
16703 // vector and the order in which they're being inserted, just put
16704 // them at element indices required for the instruction.
16705 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16706 if (DAG.getDataLayout().isLittleEndian())
16707 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF;
16708 else
16709 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4;
16710 CorrectElems = CorrectElems >> 8;
16711 Elems = Elems >> 8;
16712 }
16713
16714 SDValue Shuffle =
16715 DAG.getVectorShuffle(VT: Input.getValueType(), dl, N1: Input,
16716 N2: DAG.getUNDEF(VT: Input.getValueType()), Mask: ShuffleMask);
16717
16718 EVT VT = N->getValueType(ResNo: 0);
16719 SDValue Conv = DAG.getBitcast(VT, V: Shuffle);
16720
16721 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
16722 VT: Input.getValueType().getVectorElementType(),
16723 NumElements: VT.getVectorNumElements());
16724 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT, N1: Conv,
16725 N2: DAG.getValueType(ExtVT));
16726}
16727
16728// Look for build vector patterns where input operands come from sign
16729// extended vector_extract elements of specific indices. If the correct indices
16730// aren't used, add a vector shuffle to fix up the indices and create
16731// SIGN_EXTEND_INREG node which selects the vector sign extend instructions
16732// during instruction selection.
16733static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) {
16734 // This array encodes the indices that the vector sign extend instructions
16735 // extract from when extending from one type to another for both BE and LE.
16736 // The right nibble of each byte corresponds to the LE incides.
16737 // and the left nibble of each byte corresponds to the BE incides.
16738 // For example: 0x3074B8FC byte->word
16739 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC
16740 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF
16741 // For example: 0x000070F8 byte->double word
16742 // For LE: the allowed indices are: 0x0,0x8
16743 // For BE: the allowed indices are: 0x7,0xF
16744 uint64_t TargetElems[] = {
16745 0x3074B8FC, // b->w
16746 0x000070F8, // b->d
16747 0x10325476, // h->w
16748 0x00003074, // h->d
16749 0x00001032, // w->d
16750 };
16751
16752 uint64_t Elems = 0;
16753 int Index;
16754 SDValue Input;
16755
16756 auto isSExtOfVecExtract = [&](SDValue Op) -> bool {
16757 if (!Op)
16758 return false;
16759 if (Op.getOpcode() != ISD::SIGN_EXTEND &&
16760 Op.getOpcode() != ISD::SIGN_EXTEND_INREG)
16761 return false;
16762
16763 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value
16764 // of the right width.
16765 SDValue Extract = Op.getOperand(i: 0);
16766 if (Extract.getOpcode() == ISD::ANY_EXTEND)
16767 Extract = Extract.getOperand(i: 0);
16768 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16769 return false;
16770
16771 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Val: Extract.getOperand(i: 1));
16772 if (!ExtOp)
16773 return false;
16774
16775 Index = ExtOp->getZExtValue();
16776 if (Input && Input != Extract.getOperand(i: 0))
16777 return false;
16778
16779 if (!Input)
16780 Input = Extract.getOperand(i: 0);
16781
16782 Elems = Elems << 8;
16783 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4;
16784 Elems |= Index;
16785
16786 return true;
16787 };
16788
16789 // If the build vector operands aren't sign extended vector extracts,
16790 // of the same input vector, then return.
16791 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16792 if (!isSExtOfVecExtract(N->getOperand(Num: i))) {
16793 return SDValue();
16794 }
16795 }
16796
16797 // If the vector extract indices are not correct, add the appropriate
16798 // vector_shuffle.
16799 int TgtElemArrayIdx;
16800 int InputSize = Input.getValueType().getScalarSizeInBits();
16801 int OutputSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
16802 if (InputSize + OutputSize == 40)
16803 TgtElemArrayIdx = 0;
16804 else if (InputSize + OutputSize == 72)
16805 TgtElemArrayIdx = 1;
16806 else if (InputSize + OutputSize == 48)
16807 TgtElemArrayIdx = 2;
16808 else if (InputSize + OutputSize == 80)
16809 TgtElemArrayIdx = 3;
16810 else if (InputSize + OutputSize == 96)
16811 TgtElemArrayIdx = 4;
16812 else
16813 return SDValue();
16814
16815 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx];
16816 CorrectElems = DAG.getDataLayout().isLittleEndian()
16817 ? CorrectElems & 0x0F0F0F0F0F0F0F0F
16818 : CorrectElems & 0xF0F0F0F0F0F0F0F0;
16819 if (Elems != CorrectElems) {
16820 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems);
16821 }
16822
16823 // Regular lowering will catch cases where a shuffle is not needed.
16824 return SDValue();
16825}
16826
16827// Look for the pattern of a load from a narrow width to i128, feeding
16828// into a BUILD_VECTOR of v1i128. Replace this sequence with a PPCISD node
16829// (LXVRZX). This node represents a zero extending load that will be matched
16830// to the Load VSX Vector Rightmost instructions.
16831static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) {
16832 SDLoc DL(N);
16833
16834 // This combine is only eligible for a BUILD_VECTOR of v1i128.
16835 if (N->getValueType(ResNo: 0) != MVT::v1i128)
16836 return SDValue();
16837
16838 SDValue Operand = N->getOperand(Num: 0);
16839 // Proceed with the transformation if the operand to the BUILD_VECTOR
16840 // is a load instruction.
16841 if (Operand.getOpcode() != ISD::LOAD)
16842 return SDValue();
16843
16844 auto *LD = cast<LoadSDNode>(Val&: Operand);
16845 EVT MemoryType = LD->getMemoryVT();
16846
16847 // This transformation is only valid if the we are loading either a byte,
16848 // halfword, word, or doubleword.
16849 bool ValidLDType = MemoryType == MVT::i8 || MemoryType == MVT::i16 ||
16850 MemoryType == MVT::i32 || MemoryType == MVT::i64;
16851
16852 // Ensure that the load from the narrow width is being zero extended to i128.
16853 if (!ValidLDType ||
16854 (LD->getExtensionType() != ISD::ZEXTLOAD &&
16855 LD->getExtensionType() != ISD::EXTLOAD))
16856 return SDValue();
16857
16858 SDValue LoadOps[] = {
16859 LD->getChain(), LD->getBasePtr(),
16860 DAG.getIntPtrConstant(Val: MemoryType.getScalarSizeInBits(), DL)};
16861
16862 return DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVRZX, dl: DL,
16863 VTList: DAG.getVTList(VT1: MVT::v1i128, VT2: MVT::Other),
16864 Ops: LoadOps, MemVT: MemoryType, MMO: LD->getMemOperand());
16865}
16866
16867SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N,
16868 DAGCombinerInfo &DCI) const {
16869 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16870 "Should be called with a BUILD_VECTOR node");
16871
16872 SelectionDAG &DAG = DCI.DAG;
16873 SDLoc dl(N);
16874
16875 if (!Subtarget.hasVSX())
16876 return SDValue();
16877
16878 // The target independent DAG combiner will leave a build_vector of
16879 // float-to-int conversions intact. We can generate MUCH better code for
16880 // a float-to-int conversion of a vector of floats.
16881 SDValue FirstInput = N->getOperand(Num: 0);
16882 if (FirstInput.getOpcode() == PPCISD::MFVSR) {
16883 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI);
16884 if (Reduced)
16885 return Reduced;
16886 }
16887
16888 // If we're building a vector out of consecutive loads, just load that
16889 // vector type.
16890 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG);
16891 if (Reduced)
16892 return Reduced;
16893
16894 // If we're building a vector out of extended elements from another vector
16895 // we have P9 vector integer extend instructions. The code assumes legal
16896 // input types (i.e. it can't handle things like v4i16) so do not run before
16897 // legalization.
16898 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) {
16899 Reduced = combineBVOfVecSExt(N, DAG);
16900 if (Reduced)
16901 return Reduced;
16902 }
16903
16904 // On Power10, the Load VSX Vector Rightmost instructions can be utilized
16905 // if this is a BUILD_VECTOR of v1i128, and if the operand to the BUILD_VECTOR
16906 // is a load from <valid narrow width> to i128.
16907 if (Subtarget.isISA3_1()) {
16908 SDValue BVOfZLoad = combineBVZEXTLOAD(N, DAG);
16909 if (BVOfZLoad)
16910 return BVOfZLoad;
16911 }
16912
16913 if (N->getValueType(ResNo: 0) != MVT::v2f64)
16914 return SDValue();
16915
16916 // Looking for:
16917 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1))
16918 if (FirstInput.getOpcode() != ISD::SINT_TO_FP &&
16919 FirstInput.getOpcode() != ISD::UINT_TO_FP)
16920 return SDValue();
16921 if (N->getOperand(Num: 1).getOpcode() != ISD::SINT_TO_FP &&
16922 N->getOperand(Num: 1).getOpcode() != ISD::UINT_TO_FP)
16923 return SDValue();
16924 if (FirstInput.getOpcode() != N->getOperand(Num: 1).getOpcode())
16925 return SDValue();
16926
16927 SDValue Ext1 = FirstInput.getOperand(i: 0);
16928 SDValue Ext2 = N->getOperand(Num: 1).getOperand(i: 0);
16929 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
16930 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16931 return SDValue();
16932
16933 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Val: Ext1.getOperand(i: 1));
16934 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Val: Ext2.getOperand(i: 1));
16935 if (!Ext1Op || !Ext2Op)
16936 return SDValue();
16937 if (Ext1.getOperand(i: 0).getValueType() != MVT::v4i32 ||
16938 Ext1.getOperand(i: 0) != Ext2.getOperand(i: 0))
16939 return SDValue();
16940
16941 int FirstElem = Ext1Op->getZExtValue();
16942 int SecondElem = Ext2Op->getZExtValue();
16943 int SubvecIdx;
16944 if (FirstElem == 0 && SecondElem == 1)
16945 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0;
16946 else if (FirstElem == 2 && SecondElem == 3)
16947 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1;
16948 else
16949 return SDValue();
16950
16951 SDValue SrcVec = Ext1.getOperand(i: 0);
16952 auto NodeType = (N->getOperand(Num: 1).getOpcode() == ISD::SINT_TO_FP) ?
16953 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP;
16954 return DAG.getNode(Opcode: NodeType, DL: dl, VT: MVT::v2f64,
16955 N1: SrcVec, N2: DAG.getIntPtrConstant(Val: SubvecIdx, DL: dl));
16956}
16957
16958SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
16959 DAGCombinerInfo &DCI) const {
16960 assert((N->getOpcode() == ISD::SINT_TO_FP ||
16961 N->getOpcode() == ISD::UINT_TO_FP) &&
16962 "Need an int -> FP conversion node here");
16963
16964 if (useSoftFloat() || !Subtarget.has64BitSupport())
16965 return SDValue();
16966
16967 SelectionDAG &DAG = DCI.DAG;
16968 SDLoc dl(N);
16969 SDValue Op(N, 0);
16970
16971 // Don't handle ppc_fp128 here or conversions that are out-of-range capable
16972 // from the hardware.
16973 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
16974 return SDValue();
16975 if (!Op.getOperand(i: 0).getValueType().isSimple())
16976 return SDValue();
16977 if (Op.getOperand(i: 0).getValueType().getSimpleVT() <= MVT(MVT::i1) ||
16978 Op.getOperand(i: 0).getValueType().getSimpleVT() > MVT(MVT::i64))
16979 return SDValue();
16980
16981 SDValue FirstOperand(Op.getOperand(i: 0));
16982 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD &&
16983 (FirstOperand.getValueType() == MVT::i8 ||
16984 FirstOperand.getValueType() == MVT::i16);
16985 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) {
16986 bool Signed = N->getOpcode() == ISD::SINT_TO_FP;
16987 bool DstDouble = Op.getValueType() == MVT::f64;
16988 unsigned ConvOp = Signed ?
16989 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) :
16990 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS);
16991 SDValue WidthConst =
16992 DAG.getIntPtrConstant(Val: FirstOperand.getValueType() == MVT::i8 ? 1 : 2,
16993 DL: dl, isTarget: false);
16994 LoadSDNode *LDN = cast<LoadSDNode>(Val: FirstOperand.getNode());
16995 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst };
16996 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXSIZX, dl,
16997 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
16998 Ops, MemVT: MVT::i8, MMO: LDN->getMemOperand());
16999 DAG.makeEquivalentMemoryOrdering(OldLoad: LDN, NewMemOp: Ld);
17000
17001 // For signed conversion, we need to sign-extend the value in the VSR
17002 if (Signed) {
17003 SDValue ExtOps[] = { Ld, WidthConst };
17004 SDValue Ext = DAG.getNode(Opcode: PPCISD::VEXTS, DL: dl, VT: MVT::f64, Ops: ExtOps);
17005 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ext);
17006 } else
17007 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ld);
17008 }
17009
17010
17011 // For i32 intermediate values, unfortunately, the conversion functions
17012 // leave the upper 32 bits of the value are undefined. Within the set of
17013 // scalar instructions, we have no method for zero- or sign-extending the
17014 // value. Thus, we cannot handle i32 intermediate values here.
17015 if (Op.getOperand(i: 0).getValueType() == MVT::i32)
17016 return SDValue();
17017
17018 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&
17019 "UINT_TO_FP is supported only with FPCVT");
17020
17021 // If we have FCFIDS, then use it when converting to single-precision.
17022 // Otherwise, convert to double-precision and then round.
17023 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
17024 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
17025 : PPCISD::FCFIDS)
17026 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
17027 : PPCISD::FCFID);
17028 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
17029 ? MVT::f32
17030 : MVT::f64;
17031
17032 // If we're converting from a float, to an int, and back to a float again,
17033 // then we don't need the store/load pair at all.
17034 if ((Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_UINT &&
17035 Subtarget.hasFPCVT()) ||
17036 (Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT)) {
17037 SDValue Src = Op.getOperand(i: 0).getOperand(i: 0);
17038 if (Src.getValueType() == MVT::f32) {
17039 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
17040 DCI.AddToWorklist(N: Src.getNode());
17041 } else if (Src.getValueType() != MVT::f64) {
17042 // Make sure that we don't pick up a ppc_fp128 source value.
17043 return SDValue();
17044 }
17045
17046 unsigned FCTOp =
17047 Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
17048 PPCISD::FCTIDUZ;
17049
17050 SDValue Tmp = DAG.getNode(Opcode: FCTOp, DL: dl, VT: MVT::f64, Operand: Src);
17051 SDValue FP = DAG.getNode(Opcode: FCFOp, DL: dl, VT: FCFTy, Operand: Tmp);
17052
17053 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
17054 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
17055 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
17056 DCI.AddToWorklist(N: FP.getNode());
17057 }
17058
17059 return FP;
17060 }
17061
17062 return SDValue();
17063}
17064
17065// expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for
17066// builtins) into loads with swaps.
17067SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
17068 DAGCombinerInfo &DCI) const {
17069 // Delay VSX load for LE combine until after LegalizeOps to prioritize other
17070 // load combines.
17071 if (DCI.isBeforeLegalizeOps())
17072 return SDValue();
17073
17074 SelectionDAG &DAG = DCI.DAG;
17075 SDLoc dl(N);
17076 SDValue Chain;
17077 SDValue Base;
17078 MachineMemOperand *MMO;
17079
17080 switch (N->getOpcode()) {
17081 default:
17082 llvm_unreachable("Unexpected opcode for little endian VSX load");
17083 case ISD::LOAD: {
17084 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
17085 Chain = LD->getChain();
17086 Base = LD->getBasePtr();
17087 MMO = LD->getMemOperand();
17088 // If the MMO suggests this isn't a load of a full vector, leave
17089 // things alone. For a built-in, we have to make the change for
17090 // correctness, so if there is a size problem that will be a bug.
17091 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17092 return SDValue();
17093 break;
17094 }
17095 case ISD::INTRINSIC_W_CHAIN: {
17096 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17097 Chain = Intrin->getChain();
17098 // Similarly to the store case below, Intrin->getBasePtr() doesn't get
17099 // us what we want. Get operand 2 instead.
17100 Base = Intrin->getOperand(Num: 2);
17101 MMO = Intrin->getMemOperand();
17102 break;
17103 }
17104 }
17105
17106 MVT VecTy = N->getValueType(ResNo: 0).getSimpleVT();
17107
17108 SDValue LoadOps[] = { Chain, Base };
17109 SDValue Load = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVD2X, dl,
17110 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other),
17111 Ops: LoadOps, MemVT: MVT::v2f64, MMO);
17112
17113 DCI.AddToWorklist(N: Load.getNode());
17114 Chain = Load.getValue(R: 1);
17115 SDValue Swap = DAG.getNode(
17116 Opcode: PPCISD::XXSWAPD, DL: dl, VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Load);
17117 DCI.AddToWorklist(N: Swap.getNode());
17118
17119 // Add a bitcast if the resulting load type doesn't match v2f64.
17120 if (VecTy != MVT::v2f64) {
17121 SDValue N = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: Swap);
17122 DCI.AddToWorklist(N: N.getNode());
17123 // Package {bitcast value, swap's chain} to match Load's shape.
17124 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VT1: VecTy, VT2: MVT::Other),
17125 N1: N, N2: Swap.getValue(R: 1));
17126 }
17127
17128 return Swap;
17129}
17130
17131// expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for
17132// builtins) into stores with swaps.
17133SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
17134 DAGCombinerInfo &DCI) const {
17135 // Delay VSX store for LE combine until after LegalizeOps to prioritize other
17136 // store combines.
17137 if (DCI.isBeforeLegalizeOps())
17138 return SDValue();
17139
17140 SelectionDAG &DAG = DCI.DAG;
17141 SDLoc dl(N);
17142 SDValue Chain;
17143 SDValue Base;
17144 unsigned SrcOpnd;
17145 MachineMemOperand *MMO;
17146
17147 switch (N->getOpcode()) {
17148 default:
17149 llvm_unreachable("Unexpected opcode for little endian VSX store");
17150 case ISD::STORE: {
17151 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
17152 Chain = ST->getChain();
17153 Base = ST->getBasePtr();
17154 MMO = ST->getMemOperand();
17155 SrcOpnd = 1;
17156 // If the MMO suggests this isn't a store of a full vector, leave
17157 // things alone. For a built-in, we have to make the change for
17158 // correctness, so if there is a size problem that will be a bug.
17159 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17160 return SDValue();
17161 break;
17162 }
17163 case ISD::INTRINSIC_VOID: {
17164 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17165 Chain = Intrin->getChain();
17166 // Intrin->getBasePtr() oddly does not get what we want.
17167 Base = Intrin->getOperand(Num: 3);
17168 MMO = Intrin->getMemOperand();
17169 SrcOpnd = 2;
17170 break;
17171 }
17172 }
17173
17174 SDValue Src = N->getOperand(Num: SrcOpnd);
17175 MVT VecTy = Src.getValueType().getSimpleVT();
17176
17177 // All stores are done as v2f64 and possible bit cast.
17178 if (VecTy != MVT::v2f64) {
17179 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: Src);
17180 DCI.AddToWorklist(N: Src.getNode());
17181 }
17182
17183 SDValue Swap = DAG.getNode(Opcode: PPCISD::XXSWAPD, DL: dl,
17184 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Src);
17185 DCI.AddToWorklist(N: Swap.getNode());
17186 Chain = Swap.getValue(R: 1);
17187 SDValue StoreOps[] = { Chain, Swap, Base };
17188 SDValue Store = DAG.getMemIntrinsicNode(Opcode: PPCISD::STXVD2X, dl,
17189 VTList: DAG.getVTList(VT: MVT::Other),
17190 Ops: StoreOps, MemVT: VecTy, MMO);
17191 DCI.AddToWorklist(N: Store.getNode());
17192 return Store;
17193}
17194
17195// Handle DAG combine for STORE (FP_TO_INT F).
17196SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N,
17197 DAGCombinerInfo &DCI) const {
17198 SelectionDAG &DAG = DCI.DAG;
17199 SDLoc dl(N);
17200 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
17201 (void)Opcode;
17202 bool Strict = N->getOperand(Num: 1)->isStrictFPOpcode();
17203
17204 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
17205 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT)
17206 && "Not a FP_TO_INT Instruction!");
17207
17208 SDValue Val = N->getOperand(Num: 1).getOperand(i: Strict ? 1 : 0);
17209 EVT Op1VT = N->getOperand(Num: 1).getValueType();
17210 EVT ResVT = Val.getValueType();
17211
17212 if (!Subtarget.hasVSX() || !Subtarget.hasFPCVT() || !isTypeLegal(VT: ResVT))
17213 return SDValue();
17214
17215 // Only perform combine for conversion to i64/i32 or power9 i16/i8.
17216 bool ValidTypeForStoreFltAsInt =
17217 (Op1VT == MVT::i32 || (Op1VT == MVT::i64 && Subtarget.isPPC64()) ||
17218 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8)));
17219
17220 // TODO: Lower conversion from f128 on all VSX targets
17221 if (ResVT == MVT::ppcf128 || (ResVT == MVT::f128 && !Subtarget.hasP9Vector()))
17222 return SDValue();
17223
17224 if ((Op1VT != MVT::i64 && !Subtarget.hasP8Vector()) ||
17225 cast<StoreSDNode>(Val: N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt)
17226 return SDValue();
17227
17228 Val = convertFPToInt(Op: N->getOperand(Num: 1), DAG, Subtarget);
17229
17230 // Set number of bytes being converted.
17231 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8;
17232 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2),
17233 DAG.getIntPtrConstant(Val: ByteSize, DL: dl, isTarget: false),
17234 DAG.getValueType(Op1VT)};
17235
17236 Val = DAG.getMemIntrinsicNode(Opcode: PPCISD::ST_VSR_SCAL_INT, dl,
17237 VTList: DAG.getVTList(VT: MVT::Other), Ops,
17238 MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
17239 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
17240
17241 return Val;
17242}
17243
17244static bool isAlternatingShuffMask(const ArrayRef<int> &Mask, int NumElts) {
17245 // Check that the source of the element keeps flipping
17246 // (i.e. Mask[i] < NumElts -> Mask[i+i] >= NumElts).
17247 bool PrevElemFromFirstVec = Mask[0] < NumElts;
17248 for (int i = 1, e = Mask.size(); i < e; i++) {
17249 if (PrevElemFromFirstVec && Mask[i] < NumElts)
17250 return false;
17251 if (!PrevElemFromFirstVec && Mask[i] >= NumElts)
17252 return false;
17253 PrevElemFromFirstVec = !PrevElemFromFirstVec;
17254 }
17255 return true;
17256}
17257
17258static bool isSplatBV(SDValue Op) {
17259 if (Op.getOpcode() != ISD::BUILD_VECTOR)
17260 return false;
17261 SDValue FirstOp;
17262
17263 // Find first non-undef input.
17264 for (int i = 0, e = Op.getNumOperands(); i < e; i++) {
17265 FirstOp = Op.getOperand(i);
17266 if (!FirstOp.isUndef())
17267 break;
17268 }
17269
17270 // All inputs are undef or the same as the first non-undef input.
17271 for (int i = 1, e = Op.getNumOperands(); i < e; i++)
17272 if (Op.getOperand(i) != FirstOp && !Op.getOperand(i).isUndef())
17273 return false;
17274 return true;
17275}
17276
17277static SDValue isScalarToVec(SDValue Op) {
17278 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17279 return Op;
17280 if (Op.getOpcode() != ISD::BITCAST)
17281 return SDValue();
17282 Op = Op.getOperand(i: 0);
17283 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17284 return Op;
17285 return SDValue();
17286}
17287
17288// Fix up the shuffle mask to account for the fact that the result of
17289// scalar_to_vector is not in lane zero. This just takes all values in
17290// the ranges specified by the min/max indices and adds the number of
17291// elements required to ensure each element comes from the respective
17292// position in the valid lane.
17293// On little endian, that's just the corresponding element in the other
17294// half of the vector. On big endian, it is in the same half but right
17295// justified rather than left justified in that half.
17296static void fixupShuffleMaskForPermutedSToV(
17297 SmallVectorImpl<int> &ShuffV, int LHSFirstElt, int LHSLastElt,
17298 int RHSFirstElt, int RHSLastElt, int HalfVec, unsigned LHSNumValidElts,
17299 unsigned RHSNumValidElts, const PPCSubtarget &Subtarget) {
17300 int LHSEltFixup =
17301 Subtarget.isLittleEndian() ? HalfVec : HalfVec - LHSNumValidElts;
17302 int RHSEltFixup =
17303 Subtarget.isLittleEndian() ? HalfVec : HalfVec - RHSNumValidElts;
17304 for (int I = 0, E = ShuffV.size(); I < E; ++I) {
17305 int Idx = ShuffV[I];
17306 if (Idx >= LHSFirstElt && Idx <= LHSLastElt)
17307 ShuffV[I] += LHSEltFixup;
17308 else if (Idx >= RHSFirstElt && Idx <= RHSLastElt)
17309 ShuffV[I] += RHSEltFixup;
17310 }
17311}
17312
17313// Replace a SCALAR_TO_VECTOR with a SCALAR_TO_VECTOR_PERMUTED except if
17314// the original is:
17315// (<n x Ty> (scalar_to_vector (Ty (extract_elt <n x Ty> %a, C))))
17316// In such a case, just change the shuffle mask to extract the element
17317// from the permuted index.
17318static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG,
17319 const PPCSubtarget &Subtarget) {
17320 SDLoc dl(OrigSToV);
17321 EVT VT = OrigSToV.getValueType();
17322 assert(OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR &&
17323 "Expecting a SCALAR_TO_VECTOR here");
17324 SDValue Input = OrigSToV.getOperand(i: 0);
17325
17326 if (Input.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17327 ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Val: Input.getOperand(i: 1));
17328 SDValue OrigVector = Input.getOperand(i: 0);
17329
17330 // Can't handle non-const element indices or different vector types
17331 // for the input to the extract and the output of the scalar_to_vector.
17332 if (Idx && VT == OrigVector.getValueType()) {
17333 unsigned NumElts = VT.getVectorNumElements();
17334 assert(
17335 NumElts > 1 &&
17336 "Cannot produce a permuted scalar_to_vector for one element vector");
17337 SmallVector<int, 16> NewMask(NumElts, -1);
17338 unsigned ResultInElt = NumElts / 2;
17339 ResultInElt -= Subtarget.isLittleEndian() ? 0 : 1;
17340 NewMask[ResultInElt] = Idx->getZExtValue();
17341 return DAG.getVectorShuffle(VT, dl, N1: OrigVector, N2: OrigVector, Mask: NewMask);
17342 }
17343 }
17344 return DAG.getNode(Opcode: PPCISD::SCALAR_TO_VECTOR_PERMUTED, DL: dl, VT,
17345 Operand: OrigSToV.getOperand(i: 0));
17346}
17347
17348static bool isShuffleMaskInRange(const SmallVectorImpl<int> &ShuffV,
17349 int HalfVec, int LHSLastElementDefined,
17350 int RHSLastElementDefined) {
17351 for (int Index : ShuffV) {
17352 if (Index < 0) // Skip explicitly undefined mask indices.
17353 continue;
17354 // Handle first input vector of the vector_shuffle.
17355 if ((LHSLastElementDefined >= 0) && (Index < HalfVec) &&
17356 (Index > LHSLastElementDefined))
17357 return false;
17358 // Handle second input vector of the vector_shuffle.
17359 if ((RHSLastElementDefined >= 0) &&
17360 (Index > HalfVec + RHSLastElementDefined))
17361 return false;
17362 }
17363 return true;
17364}
17365
17366static SDValue generateSToVPermutedForVecShuffle(
17367 int ScalarSize, uint64_t ShuffleEltWidth, unsigned &NumValidElts,
17368 int FirstElt, int &LastElt, SDValue VecShuffOperand, SDValue SToVNode,
17369 SelectionDAG &DAG, const PPCSubtarget &Subtarget) {
17370 EVT VecShuffOperandType = VecShuffOperand.getValueType();
17371 // Set up the values for the shuffle vector fixup.
17372 NumValidElts = ScalarSize / VecShuffOperandType.getScalarSizeInBits();
17373 // The last element depends on if the input comes from the LHS or RHS.
17374 //
17375 // For example:
17376 // (shuff (s_to_v i32), (bitcast (s_to_v i64), v4i32), ...)
17377 //
17378 // For the LHS: The last element that comes from the LHS is actually 0, not 3
17379 // because elements 1 and higher of a scalar_to_vector are undefined.
17380 // For the RHS: The last element that comes from the RHS is actually 5, not 7
17381 // because elements 1 and higher of a scalar_to_vector are undefined.
17382 // It is also not 4 because the original scalar_to_vector is wider and
17383 // actually contains two i32 elements.
17384 LastElt = (uint64_t)ScalarSize > ShuffleEltWidth
17385 ? ScalarSize / ShuffleEltWidth - 1 + FirstElt
17386 : FirstElt;
17387 SDValue SToVPermuted = getSToVPermuted(OrigSToV: SToVNode, DAG, Subtarget);
17388 if (SToVPermuted.getValueType() != VecShuffOperandType)
17389 SToVPermuted = DAG.getBitcast(VT: VecShuffOperandType, V: SToVPermuted);
17390 return SToVPermuted;
17391}
17392
17393// On little endian subtargets, combine shuffles such as:
17394// vector_shuffle<16,1,17,3,18,5,19,7,20,9,21,11,22,13,23,15>, <zero>, %b
17395// into:
17396// vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7>, <zero>, %b
17397// because the latter can be matched to a single instruction merge.
17398// Furthermore, SCALAR_TO_VECTOR on little endian always involves a permute
17399// to put the value into element zero. Adjust the shuffle mask so that the
17400// vector can remain in permuted form (to prevent a swap prior to a shuffle).
17401// On big endian targets, this is still useful for SCALAR_TO_VECTOR
17402// nodes with elements smaller than doubleword because all the ways
17403// of getting scalar data into a vector register put the value in the
17404// rightmost element of the left half of the vector.
17405SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN,
17406 SelectionDAG &DAG) const {
17407 SDValue LHS = SVN->getOperand(Num: 0);
17408 SDValue RHS = SVN->getOperand(Num: 1);
17409 auto Mask = SVN->getMask();
17410 int NumElts = LHS.getValueType().getVectorNumElements();
17411 SDValue Res(SVN, 0);
17412 SDLoc dl(SVN);
17413 bool IsLittleEndian = Subtarget.isLittleEndian();
17414
17415 // On big endian targets this is only useful for subtargets with direct moves.
17416 // On little endian targets it would be useful for all subtargets with VSX.
17417 // However adding special handling for LE subtargets without direct moves
17418 // would be wasted effort since the minimum arch for LE is ISA 2.07 (Power8)
17419 // which includes direct moves.
17420 if (!Subtarget.hasDirectMove())
17421 return Res;
17422
17423 // If this is not a shuffle of a shuffle and the first element comes from
17424 // the second vector, canonicalize to the commuted form. This will make it
17425 // more likely to match one of the single instruction patterns.
17426 if (Mask[0] >= NumElts && LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
17427 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) {
17428 std::swap(a&: LHS, b&: RHS);
17429 Res = DAG.getCommutedVectorShuffle(SV: *SVN);
17430
17431 if (!isa<ShuffleVectorSDNode>(Val: Res))
17432 return Res;
17433
17434 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17435 }
17436
17437 // Adjust the shuffle mask if either input vector comes from a
17438 // SCALAR_TO_VECTOR and keep the respective input vector in permuted
17439 // form (to prevent the need for a swap).
17440 SmallVector<int, 16> ShuffV(Mask);
17441 SDValue SToVLHS = isScalarToVec(Op: LHS);
17442 SDValue SToVRHS = isScalarToVec(Op: RHS);
17443 if (SToVLHS || SToVRHS) {
17444 EVT VT = SVN->getValueType(ResNo: 0);
17445 uint64_t ShuffleEltWidth = VT.getVectorElementType().getSizeInBits();
17446 int ShuffleNumElts = ShuffV.size();
17447 int HalfVec = ShuffleNumElts / 2;
17448 // The width of the "valid lane" (i.e. the lane that contains the value that
17449 // is vectorized) needs to be expressed in terms of the number of elements
17450 // of the shuffle. It is thereby the ratio of the values before and after
17451 // any bitcast, which will be set later on if the LHS or RHS are
17452 // SCALAR_TO_VECTOR nodes.
17453 unsigned LHSNumValidElts = HalfVec;
17454 unsigned RHSNumValidElts = HalfVec;
17455
17456 // Initially assume that neither input is permuted. These will be adjusted
17457 // accordingly if either input is. Note, that -1 means that all elements
17458 // are undefined.
17459 int LHSFirstElt = 0;
17460 int RHSFirstElt = ShuffleNumElts;
17461 int LHSLastElt = -1;
17462 int RHSLastElt = -1;
17463
17464 // Get the permuted scalar to vector nodes for the source(s) that come from
17465 // ISD::SCALAR_TO_VECTOR.
17466 // On big endian systems, this only makes sense for element sizes smaller
17467 // than 64 bits since for 64-bit elements, all instructions already put
17468 // the value into element zero. Since scalar size of LHS and RHS may differ
17469 // after isScalarToVec, this should be checked using their own sizes.
17470 int LHSScalarSize = 0;
17471 int RHSScalarSize = 0;
17472 if (SToVLHS) {
17473 LHSScalarSize = SToVLHS.getValueType().getScalarSizeInBits();
17474 if (!IsLittleEndian && LHSScalarSize >= 64)
17475 return Res;
17476 }
17477 if (SToVRHS) {
17478 RHSScalarSize = SToVRHS.getValueType().getScalarSizeInBits();
17479 if (!IsLittleEndian && RHSScalarSize >= 64)
17480 return Res;
17481 }
17482 if (LHSScalarSize != 0)
17483 LHS = generateSToVPermutedForVecShuffle(
17484 ScalarSize: LHSScalarSize, ShuffleEltWidth, NumValidElts&: LHSNumValidElts, FirstElt: LHSFirstElt,
17485 LastElt&: LHSLastElt, VecShuffOperand: LHS, SToVNode: SToVLHS, DAG, Subtarget);
17486 if (RHSScalarSize != 0)
17487 RHS = generateSToVPermutedForVecShuffle(
17488 ScalarSize: RHSScalarSize, ShuffleEltWidth, NumValidElts&: RHSNumValidElts, FirstElt: RHSFirstElt,
17489 LastElt&: RHSLastElt, VecShuffOperand: RHS, SToVNode: SToVRHS, DAG, Subtarget);
17490
17491 if (!isShuffleMaskInRange(ShuffV, HalfVec, LHSLastElementDefined: LHSLastElt, RHSLastElementDefined: RHSLastElt))
17492 return Res;
17493
17494 // Fix up the shuffle mask to reflect where the desired element actually is.
17495 // The minimum and maximum indices that correspond to element zero for both
17496 // the LHS and RHS are computed and will control which shuffle mask entries
17497 // are to be changed. For example, if the RHS is permuted, any shuffle mask
17498 // entries in the range [RHSFirstElt,RHSLastElt] will be adjusted.
17499 fixupShuffleMaskForPermutedSToV(
17500 ShuffV, LHSFirstElt, LHSLastElt, RHSFirstElt, RHSLastElt, HalfVec,
17501 LHSNumValidElts, RHSNumValidElts, Subtarget);
17502 Res = DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17503
17504 // We may have simplified away the shuffle. We won't be able to do anything
17505 // further with it here.
17506 if (!isa<ShuffleVectorSDNode>(Val: Res))
17507 return Res;
17508 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17509 }
17510
17511 SDValue TheSplat = IsLittleEndian ? RHS : LHS;
17512 // The common case after we commuted the shuffle is that the RHS is a splat
17513 // and we have elements coming in from the splat at indices that are not
17514 // conducive to using a merge.
17515 // Example:
17516 // vector_shuffle<0,17,1,19,2,21,3,23,4,25,5,27,6,29,7,31> t1, <zero>
17517 if (!isSplatBV(Op: TheSplat))
17518 return Res;
17519
17520 // We are looking for a mask such that all even elements are from
17521 // one vector and all odd elements from the other.
17522 if (!isAlternatingShuffMask(Mask, NumElts))
17523 return Res;
17524
17525 // Adjust the mask so we are pulling in the same index from the splat
17526 // as the index from the interesting vector in consecutive elements.
17527 if (IsLittleEndian) {
17528 // Example (even elements from first vector):
17529 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero>
17530 if (Mask[0] < NumElts)
17531 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17532 if (ShuffV[i] < 0)
17533 continue;
17534 // If element from non-splat is undef, pick first element from splat.
17535 ShuffV[i] = (ShuffV[i - 1] >= 0 ? ShuffV[i - 1] : 0) + NumElts;
17536 }
17537 // Example (odd elements from first vector):
17538 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero>
17539 else
17540 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17541 if (ShuffV[i] < 0)
17542 continue;
17543 // If element from non-splat is undef, pick first element from splat.
17544 ShuffV[i] = (ShuffV[i + 1] >= 0 ? ShuffV[i + 1] : 0) + NumElts;
17545 }
17546 } else {
17547 // Example (even elements from first vector):
17548 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> <zero>, t1
17549 if (Mask[0] < NumElts)
17550 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17551 if (ShuffV[i] < 0)
17552 continue;
17553 // If element from non-splat is undef, pick first element from splat.
17554 ShuffV[i] = ShuffV[i + 1] >= 0 ? ShuffV[i + 1] - NumElts : 0;
17555 }
17556 // Example (odd elements from first vector):
17557 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> <zero>, t1
17558 else
17559 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17560 if (ShuffV[i] < 0)
17561 continue;
17562 // If element from non-splat is undef, pick first element from splat.
17563 ShuffV[i] = ShuffV[i - 1] >= 0 ? ShuffV[i - 1] - NumElts : 0;
17564 }
17565 }
17566
17567 // If the RHS has undefs, we need to remove them since we may have created
17568 // a shuffle that adds those instead of the splat value.
17569 SDValue SplatVal =
17570 cast<BuildVectorSDNode>(Val: TheSplat.getNode())->getSplatValue();
17571 TheSplat = DAG.getSplatBuildVector(VT: TheSplat.getValueType(), DL: dl, Op: SplatVal);
17572
17573 if (IsLittleEndian)
17574 RHS = TheSplat;
17575 else
17576 LHS = TheSplat;
17577 return DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17578}
17579
17580SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN,
17581 LSBaseSDNode *LSBase,
17582 DAGCombinerInfo &DCI) const {
17583 assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) &&
17584 "Not a reverse memop pattern!");
17585
17586 auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool {
17587 auto Mask = SVN->getMask();
17588 int i = 0;
17589 auto I = Mask.rbegin();
17590 auto E = Mask.rend();
17591
17592 for (; I != E; ++I) {
17593 if (*I != i)
17594 return false;
17595 i++;
17596 }
17597 return true;
17598 };
17599
17600 SelectionDAG &DAG = DCI.DAG;
17601 EVT VT = SVN->getValueType(ResNo: 0);
17602
17603 if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX())
17604 return SDValue();
17605
17606 // Before P9, we have PPCVSXSwapRemoval pass to hack the element order.
17607 // See comment in PPCVSXSwapRemoval.cpp.
17608 // It is conflict with PPCVSXSwapRemoval opt. So we don't do it.
17609 if (!Subtarget.hasP9Vector())
17610 return SDValue();
17611
17612 if(!IsElementReverse(SVN))
17613 return SDValue();
17614
17615 if (LSBase->getOpcode() == ISD::LOAD) {
17616 // If the load return value 0 has more than one user except the
17617 // shufflevector instruction, it is not profitable to replace the
17618 // shufflevector with a reverse load.
17619 for (SDUse &Use : LSBase->uses())
17620 if (Use.getResNo() == 0 &&
17621 Use.getUser()->getOpcode() != ISD::VECTOR_SHUFFLE)
17622 return SDValue();
17623
17624 SDLoc dl(LSBase);
17625 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()};
17626 return DAG.getMemIntrinsicNode(
17627 Opcode: PPCISD::LOAD_VEC_BE, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops: LoadOps,
17628 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17629 }
17630
17631 if (LSBase->getOpcode() == ISD::STORE) {
17632 // If there are other uses of the shuffle, the swap cannot be avoided.
17633 // Forcing the use of an X-Form (since swapped stores only have
17634 // X-Forms) without removing the swap is unprofitable.
17635 if (!SVN->hasOneUse())
17636 return SDValue();
17637
17638 SDLoc dl(LSBase);
17639 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(Num: 0),
17640 LSBase->getBasePtr()};
17641 return DAG.getMemIntrinsicNode(
17642 Opcode: PPCISD::STORE_VEC_BE, dl, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps,
17643 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17644 }
17645
17646 llvm_unreachable("Expected a load or store node here");
17647}
17648
17649static bool isStoreConditional(SDValue Intrin, unsigned &StoreWidth) {
17650 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 1);
17651 if (IntrinsicID == Intrinsic::ppc_stdcx)
17652 StoreWidth = 8;
17653 else if (IntrinsicID == Intrinsic::ppc_stwcx)
17654 StoreWidth = 4;
17655 else if (IntrinsicID == Intrinsic::ppc_sthcx)
17656 StoreWidth = 2;
17657 else if (IntrinsicID == Intrinsic::ppc_stbcx)
17658 StoreWidth = 1;
17659 else
17660 return false;
17661 return true;
17662}
17663
17664static SDValue DAGCombineAddc(SDNode *N,
17665 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17666 if (N->getOpcode() == PPCISD::ADDC && N->hasAnyUseOfValue(Value: 1)) {
17667 // (ADDC (ADDE 0, 0, C), -1) -> C
17668 SDValue LHS = N->getOperand(Num: 0);
17669 SDValue RHS = N->getOperand(Num: 1);
17670 if (LHS->getOpcode() == PPCISD::ADDE &&
17671 isNullConstant(V: LHS->getOperand(Num: 0)) &&
17672 isNullConstant(V: LHS->getOperand(Num: 1)) && isAllOnesConstant(V: RHS)) {
17673 return DCI.CombineTo(N, Res0: SDValue(N, 0), Res1: LHS->getOperand(Num: 2));
17674 }
17675 }
17676 return SDValue();
17677}
17678
17679static SDValue DAGCombineSube(SDNode *N,
17680 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17681 if (N->getOpcode() == PPCISD::SUBE) {
17682 SDValue LHS = N->getOperand(Num: 0);
17683 SDValue RHS = N->getOperand(Num: 1);
17684 SDValue Carry = N->getOperand(Num: 2);
17685
17686 // SUBE(ADDC(ADDE(0, 0, C), -1), same, C) -> ADDC(ADDE(0, 0, C), -1)
17687 if (LHS == RHS && LHS.getOpcode() == PPCISD::ADDC) {
17688 SDValue AddcLHS = LHS.getOperand(i: 0);
17689 SDValue AddcRHS = LHS.getOperand(i: 1);
17690 if (AddcLHS.getOpcode() == PPCISD::ADDE &&
17691 isNullConstant(V: AddcLHS.getOperand(i: 0)) &&
17692 isNullConstant(V: AddcLHS.getOperand(i: 1)) && isAllOnesConstant(V: AddcRHS) &&
17693 Carry == AddcLHS.getOperand(i: 2)) {
17694 return LHS;
17695 }
17696 }
17697 }
17698 return SDValue();
17699}
17700
17701/// Optimize the bitfloor(X) pattern for PowerPC.
17702/// Transforms: select_cc X, 0, 0, (srl MinSignedValue, (ctlz X)), seteq
17703/// Into: srl MinSignedValue, (ctlz X)
17704///
17705/// This is safe on PowerPC because the srw instruction returns 0 when the
17706/// shift amount is == bitwidth, which matches the behavior we need for X=0.
17707static SDValue combineSELECT_CCBitFloor(SDNode *N, SelectionDAG &DAG) {
17708 if (N->getOpcode() != ISD::SELECT_CC)
17709 return SDValue();
17710
17711 // SELECT_CC operands: LHS, RHS, TrueVal, FalseVal, CC
17712 SDValue CmpLHS = N->getOperand(Num: 0);
17713 SDValue CmpRHS = N->getOperand(Num: 1);
17714 SDValue TrueVal = N->getOperand(Num: 2);
17715 SDValue FalseVal = N->getOperand(Num: 3);
17716 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
17717
17718 // Check if condition is (X == 0)
17719 if (CC != ISD::SETEQ || !isNullConstant(V: CmpRHS))
17720 return SDValue();
17721
17722 // Check if TrueVal is constant 0
17723 if (!isNullConstant(V: TrueVal))
17724 return SDValue();
17725
17726 // This combine is replacing a select_cc with a PPC srl, not an srl with a
17727 // PPC srl. If the original srl had multiple uses it would just remain in the
17728 // code. This is at most a performance consideration.
17729 if (FalseVal.getOpcode() != ISD::SRL || !FalseVal.hasOneUse())
17730 return SDValue();
17731
17732 SDValue ShiftVal = FalseVal.getOperand(i: 0);
17733 SDValue ShiftAmt = FalseVal.getOperand(i: 1);
17734
17735 // Check if ShiftVal is MinSignedValue
17736 auto *ShiftConst = dyn_cast<ConstantSDNode>(Val&: ShiftVal);
17737 if (!ShiftConst || !ShiftConst->getAPIntValue().isMinSignedValue())
17738 return SDValue();
17739
17740 SDValue CtlzArg;
17741 // Check if ShiftAmt is (ctlz CmpLHS) or (truncate (ctlz ...))
17742 if (ShiftAmt.getOpcode() != ISD::CTLZ) {
17743 // Look through truncate if present (for i64 ctlz truncated to i32 shift
17744 // amount)
17745 if (ShiftAmt.getOpcode() != ISD::TRUNCATE)
17746 return SDValue();
17747
17748 // Verify the truncate target type is appropriate for shift amount (i32, not
17749 // i1 or other)
17750 if (ShiftAmt.getValueType() != MVT::i32)
17751 return SDValue();
17752
17753 SDValue CtlzNode = ShiftAmt.getOperand(i: 0);
17754
17755 if (CtlzNode.getOpcode() != ISD::CTLZ)
17756 return SDValue();
17757
17758 CtlzArg = CtlzNode.getOperand(i: 0);
17759 } else {
17760 CtlzArg = ShiftAmt.getOperand(i: 0);
17761 }
17762
17763 // Check if ctlz operates on the same value as the comparison
17764 if (CtlzArg != CmpLHS)
17765 return SDValue();
17766
17767 // Using PPCISD::SRL to ensure well-defined behavior.
17768 // On PowerPC, PPCISD::SRL guarantees that shift by bitwidth returns 0,
17769 // which is exactly what we need for the bitfloor(0) case.
17770 SDLoc DL(N);
17771 SDValue PPCSrl =
17772 DAG.getNode(Opcode: PPCISD::SRL, DL, VT: FalseVal.getValueType(), N1: ShiftVal, N2: ShiftAmt);
17773 return PPCSrl;
17774}
17775
17776// Optimize zero-extension of setcc when the compared value is known to be 0
17777// or 1.
17778//
17779// Pattern: zext(setcc(Value, 0, seteq/setne)) where Value is 0 or 1
17780// -> zext(xor(Value, 1)) for seteq
17781// -> zext(Value) for setne
17782//
17783// This optimization avoids the i32 -> i1 -> i32/i64 conversion sequence
17784// by keeping the value in its original i32 type throughout.
17785//
17786// Example:
17787// Before: zext(setcc(test_data_class(...), 0, seteq))
17788// // test_data_class returns 0 or 1 in i32
17789// // setcc converts i32 -> i1
17790// // zext converts i1 -> i64
17791// After: zext(xor(test_data_class(...), 1))
17792// // Stays in i32, then extends to i64
17793//
17794// This is beneficial because:
17795// 1. Eliminates the setcc instruction
17796// 2. Avoids i32 -> i1 truncation
17797// 3. Keeps computation in native integer width
17798
17799static SDValue combineZextSetccWithZero(SDNode *N, SelectionDAG &DAG) {
17800 // Check if this is a zero_extend
17801 if (N->getOpcode() != ISD::ZERO_EXTEND)
17802 return SDValue();
17803
17804 SDValue Src = N->getOperand(Num: 0);
17805
17806 // Check if the source is a setcc
17807 if (Src.getOpcode() != ISD::SETCC)
17808 return SDValue();
17809
17810 SDValue LHS = Src.getOperand(i: 0);
17811 SDValue RHS = Src.getOperand(i: 1);
17812 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Src.getOperand(i: 2))->get();
17813
17814 if (!isNullConstant(V: RHS) && !isNullConstant(V: LHS))
17815 return SDValue();
17816
17817 SDValue NonNullConstant = isNullConstant(V: RHS) ? LHS : RHS;
17818
17819 auto isZeroOrOne = [=](SDValue &V) {
17820 if (V.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
17821 V.getConstantOperandVal(i: 0) == Intrinsic::ppc_test_data_class)
17822 return true;
17823 return false;
17824 };
17825
17826 if (!isZeroOrOne(NonNullConstant))
17827 return SDValue();
17828
17829 // Check for pattern: zext(setcc (Value), 0, seteq)) or
17830 // zext(setcc (Value), 0, setne))
17831 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
17832 // Replace with: zext(xor(Value, 1)) for seteq
17833 // or: zext(Value) for setne
17834 // This keeps the value in i32 instead of converting to i1
17835 SDLoc DL(N);
17836 EVT VType = N->getValueType(ResNo: 0);
17837 SDValue NewNonNullConstant = DAG.getZExtOrTrunc(Op: NonNullConstant, DL, VT: VType);
17838
17839 if (CC == ISD::SETNE)
17840 return NewNonNullConstant;
17841
17842 SDValue One = DAG.getConstant(Val: 1, DL, VT: VType);
17843 return DAG.getNode(Opcode: ISD::XOR, DL, VT: VType, N1: NewNonNullConstant, N2: One);
17844 }
17845
17846 return SDValue();
17847}
17848
17849// Combine XOR patterns with SELECT_CC_I4/I8, for Example:
17850// 1. XOR(SELECT_CC_I4(cond, 1, 0, cc), 1) -> SELECT_CC_I4(cond, 0, 1, cc)
17851// 2. XOR(ZEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond, 0,
17852// 1, cc))
17853// 3. XOR(ANYEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond,
17854// 0, 1, cc))
17855// 4. etc
17856static SDValue combineXorSelectCC(SDNode *N, SelectionDAG &DAG) {
17857 assert(N->getOpcode() == ISD::XOR && "Expected XOR node");
17858
17859 EVT XorVT = N->getValueType(ResNo: 0);
17860 if ((XorVT != MVT::i32 && XorVT != MVT::i64))
17861 return SDValue();
17862
17863 SDValue LHS = N->getOperand(Num: 0);
17864 SDValue RHS = N->getOperand(Num: 1);
17865
17866 // Check for XOR with constant 1
17867 ConstantSDNode *XorConst = dyn_cast<ConstantSDNode>(Val&: RHS);
17868 if (!XorConst || !XorConst->isOne()) {
17869 XorConst = dyn_cast<ConstantSDNode>(Val&: LHS);
17870 if (!XorConst || !XorConst->isOne())
17871 return SDValue();
17872 // Swap so LHS is the SELECT_CC_I4 (or extension) and RHS is the constant
17873 std::swap(a&: LHS, b&: RHS);
17874 }
17875
17876 // Check if LHS has only one use
17877 if (!LHS.hasOneUse())
17878 return SDValue();
17879
17880 // Handle extensions: ZEXT, ANYEXT
17881 SDValue SelectNode = LHS;
17882
17883 if (LHS.getOpcode() == ISD::ZERO_EXTEND ||
17884 LHS.getOpcode() == ISD::ANY_EXTEND) {
17885 SelectNode = LHS.getOperand(i: 0);
17886
17887 // Check if the extension input has only one use
17888 if (!SelectNode.hasOneUse())
17889 return SDValue();
17890 }
17891
17892 // Check if SelectNode is a MachineSDNode with SELECT_CC_I4/I8 opcode
17893 if (!SelectNode.isMachineOpcode())
17894 return SDValue();
17895
17896 unsigned MachineOpc = SelectNode.getMachineOpcode();
17897
17898 // Handle both SELECT_CC_I4 and SELECT_CC_I8
17899 if (MachineOpc != PPC::SELECT_CC_I4 && MachineOpc != PPC::SELECT_CC_I8)
17900 return SDValue();
17901
17902 // SELECT_CC_I4 operands: (cond, true_val, false_val, bropc)
17903 if (SelectNode.getNumOperands() != 4)
17904 return SDValue();
17905
17906 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 1));
17907 ConstantSDNode *ConstOp2 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 2));
17908
17909 if (!ConstOp1 || !ConstOp2)
17910 return SDValue();
17911
17912 // Only optimize if operands are {0, 1} or {1, 0}
17913 if (!((ConstOp1->isOne() && ConstOp2->isZero()) ||
17914 (ConstOp1->isZero() && ConstOp2->isOne())))
17915 return SDValue();
17916
17917 // Pattern matched! Create new SELECT_CC with swapped 0/1 operands to
17918 // eliminate XOR. If original was SELECT_CC(cond, 1, 0, pred), create
17919 // SELECT_CC(cond, 0, 1, pred). If original was SELECT_CC(cond, 0, 1, pred),
17920 // create SELECT_CC(cond, 1, 0, pred).
17921 SDLoc DL(N);
17922 MachineOpc = (XorVT == MVT::i32) ? PPC::SELECT_CC_I4 : PPC::SELECT_CC_I8;
17923
17924 bool ConstOp1IsOne = ConstOp1->isOne();
17925 return SDValue(
17926 DAG.getMachineNode(Opcode: MachineOpc, dl: DL, VT: XorVT,
17927 Ops: {SelectNode.getOperand(i: 0),
17928 DAG.getConstant(Val: ConstOp1IsOne ? 0 : 1, DL, VT: XorVT),
17929 DAG.getConstant(Val: ConstOp1IsOne ? 1 : 0, DL, VT: XorVT),
17930 SelectNode.getOperand(i: 3)}),
17931 0);
17932}
17933
17934SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
17935 DAGCombinerInfo &DCI) const {
17936 SelectionDAG &DAG = DCI.DAG;
17937 SDLoc dl(N);
17938 switch (N->getOpcode()) {
17939 default: break;
17940 case ISD::ADD:
17941 return combineADD(N, DCI);
17942 case ISD::AND: {
17943 // We don't want (and (zext (shift...)), C) if C fits in the width of the
17944 // original input as that will prevent us from selecting optimal rotates.
17945 // This only matters if the input to the extend is i32 widened to i64.
17946 SDValue Op1 = N->getOperand(Num: 0);
17947 SDValue Op2 = N->getOperand(Num: 1);
17948 if ((Op1.getOpcode() != ISD::ZERO_EXTEND &&
17949 Op1.getOpcode() != ISD::ANY_EXTEND) ||
17950 !isa<ConstantSDNode>(Val: Op2) || N->getValueType(ResNo: 0) != MVT::i64 ||
17951 Op1.getOperand(i: 0).getValueType() != MVT::i32)
17952 break;
17953 SDValue NarrowOp = Op1.getOperand(i: 0);
17954 if (NarrowOp.getOpcode() != ISD::SHL && NarrowOp.getOpcode() != ISD::SRL &&
17955 NarrowOp.getOpcode() != ISD::ROTL && NarrowOp.getOpcode() != ISD::ROTR)
17956 break;
17957
17958 uint64_t Imm = Op2->getAsZExtVal();
17959 // Make sure that the constant is narrow enough to fit in the narrow type.
17960 if (!isUInt<32>(x: Imm))
17961 break;
17962 SDValue ConstOp = DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32);
17963 SDValue NarrowAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: NarrowOp, N2: ConstOp);
17964 return DAG.getZExtOrTrunc(Op: NarrowAnd, DL: dl, VT: N->getValueType(ResNo: 0));
17965 }
17966 case ISD::XOR: {
17967 // Optimize XOR(ISEL(1,0,CR), 1) -> ISEL(0,1,CR)
17968 if (SDValue V = combineXorSelectCC(N, DAG))
17969 return V;
17970 break;
17971 }
17972 case ISD::SHL:
17973 return combineSHL(N, DCI);
17974 case ISD::SRA:
17975 return combineSRA(N, DCI);
17976 case ISD::SRL:
17977 return combineSRL(N, DCI);
17978 case ISD::MUL:
17979 return combineMUL(N, DCI);
17980 case ISD::FMA:
17981 case PPCISD::FNMSUB:
17982 return combineFMALike(N, DCI);
17983 case PPCISD::SHL:
17984 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 << V -> 0.
17985 return N->getOperand(Num: 0);
17986 break;
17987 case PPCISD::SRL:
17988 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 >>u V -> 0.
17989 return N->getOperand(Num: 0);
17990 break;
17991 case PPCISD::SRA:
17992 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
17993 if (C->isZero() || // 0 >>s V -> 0.
17994 C->isAllOnes()) // -1 >>s V -> -1.
17995 return N->getOperand(Num: 0);
17996 }
17997 break;
17998 case ISD::SIGN_EXTEND: {
17999 // Optimize: sign_extend(scmp) -> scmp with i64 result (on PPC64)
18000 // This eliminates the sign_extend when scmp result is being extended to i64
18001 // Pattern: t6: i64 = sign_extend t5:i32
18002 // t5: i32 = scmp t2, t4
18003 // Result: t5: i64 = scmp t2, t4
18004 SDValue N0 = N->getOperand(Num: 0);
18005 if (Subtarget.isPPC64() && N0.getOpcode() == ISD::SCMP &&
18006 N->getValueType(ResNo: 0) == MVT::i64 && N0.getValueType() == MVT::i32 &&
18007 N0.hasOneUse()) {
18008 // Recreate the SCMP with i64 result type directly
18009 SDValue LHS = N0.getOperand(i: 0);
18010 SDValue RHS = N0.getOperand(i: 1);
18011 return DAG.getNode(Opcode: ISD::SCMP, DL: dl, VT: MVT::i64, N1: LHS, N2: RHS);
18012 }
18013
18014 if (SDValue SECC = combineSignExtendSetCC(N, DCI))
18015 return SECC;
18016 [[fallthrough]];
18017 }
18018 case ISD::ZERO_EXTEND:
18019 if (SDValue RetV = combineZextSetccWithZero(N, DAG&: DCI.DAG))
18020 return RetV;
18021 [[fallthrough]];
18022 case ISD::ANY_EXTEND:
18023 return DAGCombineExtBoolTrunc(N, DCI);
18024 case ISD::TRUNCATE:
18025 return combineTRUNCATE(N, DCI);
18026 case ISD::SETCC:
18027 if (SDValue CSCC = combineSetCC(N, DCI))
18028 return CSCC;
18029 [[fallthrough]];
18030 case ISD::SELECT_CC:
18031 if (SDValue V = combineSELECT_CCBitFloor(N, DAG))
18032 return V;
18033 return DAGCombineTruncBoolExt(N, DCI);
18034 case ISD::SINT_TO_FP:
18035 case ISD::UINT_TO_FP:
18036 return combineFPToIntToFP(N, DCI);
18037 case ISD::VECTOR_SHUFFLE:
18038 if (ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode())) {
18039 LSBaseSDNode* LSBase = cast<LSBaseSDNode>(Val: N->getOperand(Num: 0));
18040 return combineVReverseMemOP(SVN: cast<ShuffleVectorSDNode>(Val: N), LSBase, DCI);
18041 }
18042 return combineVectorShuffle(SVN: cast<ShuffleVectorSDNode>(Val: N), DAG&: DCI.DAG);
18043 case ISD::STORE: {
18044
18045 EVT Op1VT = N->getOperand(Num: 1).getValueType();
18046 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
18047
18048 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
18049 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT) {
18050 SDValue Val = combineStoreFPToInt(N, DCI);
18051 if (Val)
18052 return Val;
18053 }
18054
18055 if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) {
18056 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 1));
18057 SDValue Val= combineVReverseMemOP(SVN, LSBase: cast<LSBaseSDNode>(Val: N), DCI);
18058 if (Val)
18059 return Val;
18060 }
18061
18062 // Turn STORE (BSWAP) -> sthbrx/stwbrx.
18063 if (cast<StoreSDNode>(Val: N)->isUnindexed() && Opcode == ISD::BSWAP &&
18064 N->getOperand(Num: 1).getNode()->hasOneUse() &&
18065 (Op1VT == MVT::i32 || Op1VT == MVT::i16 ||
18066 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) {
18067
18068 // STBRX can only handle simple types and it makes no sense to store less
18069 // two bytes in byte-reversed order.
18070 EVT mVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
18071 if (mVT.isExtended() || mVT.getSizeInBits() < 16)
18072 break;
18073
18074 SDValue BSwapOp = N->getOperand(Num: 1).getOperand(i: 0);
18075 // Do an any-extend to 32-bits if this is a half-word input.
18076 if (BSwapOp.getValueType() == MVT::i16)
18077 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: BSwapOp);
18078
18079 // If the type of BSWAP operand is wider than stored memory width
18080 // it need to be shifted to the right side before STBRX.
18081 if (Op1VT.bitsGT(VT: mVT)) {
18082 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits();
18083 BSwapOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: Op1VT, N1: BSwapOp,
18084 N2: DAG.getConstant(Val: Shift, DL: dl, VT: MVT::i32));
18085 // Need to truncate if this is a bswap of i64 stored as i32/i16.
18086 if (Op1VT == MVT::i64)
18087 BSwapOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: BSwapOp);
18088 }
18089
18090 SDValue Ops[] = {
18091 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2), DAG.getValueType(mVT)
18092 };
18093 return
18094 DAG.getMemIntrinsicNode(Opcode: PPCISD::STBRX, dl, VTList: DAG.getVTList(VT: MVT::Other),
18095 Ops, MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
18096 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
18097 }
18098
18099 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0>
18100 // So it can increase the chance of CSE constant construction.
18101 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() &&
18102 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) && Op1VT == MVT::i32) {
18103 // Need to sign-extended to 64-bits to handle negative values.
18104 EVT MemVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
18105 uint64_t Val64 = SignExtend64(X: N->getConstantOperandVal(Num: 1),
18106 B: MemVT.getSizeInBits());
18107 SDValue Const64 = DAG.getConstant(Val: Val64, DL: dl, VT: MVT::i64);
18108
18109 auto *ST = cast<StoreSDNode>(Val: N);
18110 SDValue NewST = DAG.getStore(Chain: ST->getChain(), dl, Val: Const64,
18111 Ptr: ST->getBasePtr(), Offset: ST->getOffset(), SVT: MemVT,
18112 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
18113 /*IsTruncating=*/true);
18114 // Note we use CombineTo here to prevent DAGCombiner from visiting the
18115 // new store which will change the constant by removing non-demanded bits.
18116 return ST->isUnindexed()
18117 ? DCI.CombineTo(N, Res: NewST, /*AddTo=*/false)
18118 : DCI.CombineTo(N, Res0: NewST, Res1: NewST.getValue(R: 1), /*AddTo=*/false);
18119 }
18120
18121 // For little endian, VSX stores require generating xxswapd/lxvd2x.
18122 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18123 if (Op1VT.isSimple()) {
18124 MVT StoreVT = Op1VT.getSimpleVT();
18125 if (Subtarget.needsSwapsForVSXMemOps() &&
18126 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 ||
18127 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32))
18128 return expandVSXStoreForLE(N, DCI);
18129 }
18130 break;
18131 }
18132 case ISD::LOAD: {
18133 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
18134 EVT VT = LD->getValueType(ResNo: 0);
18135
18136 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18137 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18138 if (VT.isSimple()) {
18139 MVT LoadVT = VT.getSimpleVT();
18140 if (Subtarget.needsSwapsForVSXMemOps() &&
18141 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 ||
18142 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32))
18143 return expandVSXLoadForLE(N, DCI);
18144 }
18145
18146 // We sometimes end up with a 64-bit integer load, from which we extract
18147 // two single-precision floating-point numbers. This happens with
18148 // std::complex<float>, and other similar structures, because of the way we
18149 // canonicalize structure copies. However, if we lack direct moves,
18150 // then the final bitcasts from the extracted integer values to the
18151 // floating-point numbers turn into store/load pairs. Even with direct moves,
18152 // just loading the two floating-point numbers is likely better.
18153 auto ReplaceTwoFloatLoad = [&]() {
18154 if (VT != MVT::i64)
18155 return false;
18156
18157 if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
18158 LD->isVolatile())
18159 return false;
18160
18161 // We're looking for a sequence like this:
18162 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64
18163 // t16: i64 = srl t13, Constant:i32<32>
18164 // t17: i32 = truncate t16
18165 // t18: f32 = bitcast t17
18166 // t19: i32 = truncate t13
18167 // t20: f32 = bitcast t19
18168
18169 if (!LD->hasNUsesOfValue(NUses: 2, Value: 0))
18170 return false;
18171
18172 auto UI = LD->user_begin();
18173 while (UI.getUse().getResNo() != 0) ++UI;
18174 SDNode *Trunc = *UI++;
18175 while (UI.getUse().getResNo() != 0) ++UI;
18176 SDNode *RightShift = *UI;
18177 if (Trunc->getOpcode() != ISD::TRUNCATE)
18178 std::swap(a&: Trunc, b&: RightShift);
18179
18180 if (Trunc->getOpcode() != ISD::TRUNCATE ||
18181 Trunc->getValueType(ResNo: 0) != MVT::i32 ||
18182 !Trunc->hasOneUse())
18183 return false;
18184 if (RightShift->getOpcode() != ISD::SRL ||
18185 !isa<ConstantSDNode>(Val: RightShift->getOperand(Num: 1)) ||
18186 RightShift->getConstantOperandVal(Num: 1) != 32 ||
18187 !RightShift->hasOneUse())
18188 return false;
18189
18190 SDNode *Trunc2 = *RightShift->user_begin();
18191 if (Trunc2->getOpcode() != ISD::TRUNCATE ||
18192 Trunc2->getValueType(ResNo: 0) != MVT::i32 ||
18193 !Trunc2->hasOneUse())
18194 return false;
18195
18196 SDNode *Bitcast = *Trunc->user_begin();
18197 SDNode *Bitcast2 = *Trunc2->user_begin();
18198
18199 if (Bitcast->getOpcode() != ISD::BITCAST ||
18200 Bitcast->getValueType(ResNo: 0) != MVT::f32)
18201 return false;
18202 if (Bitcast2->getOpcode() != ISD::BITCAST ||
18203 Bitcast2->getValueType(ResNo: 0) != MVT::f32)
18204 return false;
18205
18206 if (Subtarget.isLittleEndian())
18207 std::swap(a&: Bitcast, b&: Bitcast2);
18208
18209 // Bitcast has the second float (in memory-layout order) and Bitcast2
18210 // has the first one.
18211
18212 SDValue BasePtr = LD->getBasePtr();
18213 if (LD->isIndexed()) {
18214 assert(LD->getAddressingMode() == ISD::PRE_INC &&
18215 "Non-pre-inc AM on PPC?");
18216 BasePtr =
18217 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18218 N2: LD->getOffset());
18219 }
18220
18221 auto MMOFlags =
18222 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile;
18223 SDValue FloatLoad = DAG.getLoad(VT: MVT::f32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18224 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign(),
18225 MMOFlags, Metadata: LD->getAAInfo());
18226 SDValue AddPtr =
18227 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(),
18228 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18229 SDValue FloatLoad2 = DAG.getLoad(
18230 VT: MVT::f32, dl, Chain: SDValue(FloatLoad.getNode(), 1), Ptr: AddPtr,
18231 PtrInfo: LD->getPointerInfo().getWithOffset(O: 4),
18232 Alignment: commonAlignment(A: LD->getAlign(), Offset: 4), MMOFlags, Metadata: LD->getAAInfo());
18233
18234 if (LD->isIndexed()) {
18235 // Note that DAGCombine should re-form any pre-increment load(s) from
18236 // what is produced here if that makes sense.
18237 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: BasePtr);
18238 }
18239
18240 DCI.CombineTo(N: Bitcast2, Res: FloatLoad);
18241 DCI.CombineTo(N: Bitcast, Res: FloatLoad2);
18242
18243 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, LD->isIndexed() ? 2 : 1),
18244 To: SDValue(FloatLoad2.getNode(), 1));
18245 return true;
18246 };
18247
18248 if (ReplaceTwoFloatLoad())
18249 return SDValue(N, 0);
18250
18251 EVT MemVT = LD->getMemoryVT();
18252 Type *Ty = MemVT.getTypeForEVT(Context&: *DAG.getContext());
18253 Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
18254 if (LD->isUnindexed() && VT.isVector() &&
18255 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
18256 // P8 and later hardware should just use LOAD.
18257 !Subtarget.hasP8Vector() &&
18258 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
18259 VT == MVT::v4f32))) &&
18260 LD->getAlign() < ABIAlignment) {
18261 // This is a type-legal unaligned Altivec load.
18262 SDValue Chain = LD->getChain();
18263 SDValue Ptr = LD->getBasePtr();
18264 bool isLittleEndian = Subtarget.isLittleEndian();
18265
18266 // This implements the loading of unaligned vectors as described in
18267 // the venerable Apple Velocity Engine overview. Specifically:
18268 // https://developer.apple.com/hardwaredrivers/ve/alignment.html
18269 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html
18270 //
18271 // The general idea is to expand a sequence of one or more unaligned
18272 // loads into an alignment-based permutation-control instruction (lvsl
18273 // or lvsr), a series of regular vector loads (which always truncate
18274 // their input address to an aligned address), and a series of
18275 // permutations. The results of these permutations are the requested
18276 // loaded values. The trick is that the last "extra" load is not taken
18277 // from the address you might suspect (sizeof(vector) bytes after the
18278 // last requested load), but rather sizeof(vector) - 1 bytes after the
18279 // last requested vector. The point of this is to avoid a page fault if
18280 // the base address happened to be aligned. This works because if the
18281 // base address is aligned, then adding less than a full vector length
18282 // will cause the last vector in the sequence to be (re)loaded.
18283 // Otherwise, the next vector will be fetched as you might suspect was
18284 // necessary.
18285
18286 // We might be able to reuse the permutation generation from
18287 // a different base address offset from this one by an aligned amount.
18288 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this
18289 // optimization later.
18290 Intrinsic::ID Intr, IntrLD, IntrPerm;
18291 MVT PermCntlTy, PermTy, LDTy;
18292 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18293 : Intrinsic::ppc_altivec_lvsl;
18294 IntrLD = Intrinsic::ppc_altivec_lvx;
18295 IntrPerm = Intrinsic::ppc_altivec_vperm;
18296 PermCntlTy = MVT::v16i8;
18297 PermTy = MVT::v4i32;
18298 LDTy = MVT::v4i32;
18299
18300 SDValue PermCntl = BuildIntrinsicOp(IID: Intr, Op: Ptr, DAG, dl, DestVT: PermCntlTy);
18301
18302 // Create the new MMO for the new base load. It is like the original MMO,
18303 // but represents an area in memory almost twice the vector size centered
18304 // on the original address. If the address is unaligned, we might start
18305 // reading up to (sizeof(vector)-1) bytes below the address of the
18306 // original unaligned load.
18307 MachineFunction &MF = DAG.getMachineFunction();
18308 MachineMemOperand *BaseMMO =
18309 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18310 Offset: -(int64_t)MemVT.getStoreSize()+1,
18311 Size: 2*MemVT.getStoreSize()-1);
18312
18313 // Create the new base load.
18314 SDValue LDXIntID =
18315 DAG.getTargetConstant(Val: IntrLD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18316 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
18317 SDValue BaseLoad =
18318 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18319 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18320 Ops: BaseLoadOps, MemVT: LDTy, MMO: BaseMMO);
18321
18322 // Note that the value of IncOffset (which is provided to the next
18323 // load's pointer info offset value, and thus used to calculate the
18324 // alignment), and the value of IncValue (which is actually used to
18325 // increment the pointer value) are different! This is because we
18326 // require the next load to appear to be aligned, even though it
18327 // is actually offset from the base pointer by a lesser amount.
18328 int IncOffset = VT.getSizeInBits() / 8;
18329 int IncValue = IncOffset;
18330
18331 // Walk (both up and down) the chain looking for another load at the real
18332 // (aligned) offset (the alignment of the other load does not matter in
18333 // this case). If found, then do not use the offset reduction trick, as
18334 // that will prevent the loads from being later combined (as they would
18335 // otherwise be duplicates).
18336 if (!findConsecutiveLoad(LD, DAG))
18337 --IncValue;
18338
18339 SDValue Increment =
18340 DAG.getConstant(Val: IncValue, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18341 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr, N2: Increment);
18342
18343 MachineMemOperand *ExtraMMO =
18344 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18345 Offset: 1, Size: 2*MemVT.getStoreSize()-1);
18346 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr };
18347 SDValue ExtraLoad =
18348 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18349 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18350 Ops: ExtraLoadOps, MemVT: LDTy, MMO: ExtraMMO);
18351
18352 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18353 N1: BaseLoad.getValue(R: 1), N2: ExtraLoad.getValue(R: 1));
18354
18355 // Because vperm has a big-endian bias, we must reverse the order
18356 // of the input vectors and complement the permute control vector
18357 // when generating little endian code. We have already handled the
18358 // latter by using lvsr instead of lvsl, so just reverse BaseLoad
18359 // and ExtraLoad here.
18360 SDValue Perm;
18361 if (isLittleEndian)
18362 Perm = BuildIntrinsicOp(IID: IntrPerm,
18363 Op0: ExtraLoad, Op1: BaseLoad, Op2: PermCntl, DAG, dl);
18364 else
18365 Perm = BuildIntrinsicOp(IID: IntrPerm,
18366 Op0: BaseLoad, Op1: ExtraLoad, Op2: PermCntl, DAG, dl);
18367
18368 if (VT != PermTy)
18369 Perm = Subtarget.hasAltivec()
18370 ? DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Perm)
18371 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Perm,
18372 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i64));
18373 // second argument is 1 because this rounding
18374 // is always exact.
18375
18376 // The output of the permutation is our loaded result, the TokenFactor is
18377 // our new chain.
18378 DCI.CombineTo(N, Res0: Perm, Res1: TF);
18379 return SDValue(N, 0);
18380 }
18381 }
18382 break;
18383 case ISD::INTRINSIC_WO_CHAIN: {
18384 bool isLittleEndian = Subtarget.isLittleEndian();
18385 unsigned IID = N->getConstantOperandVal(Num: 0);
18386 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18387 : Intrinsic::ppc_altivec_lvsl);
18388 if (IID == Intr && N->getOperand(Num: 1)->getOpcode() == ISD::ADD) {
18389 SDValue Add = N->getOperand(Num: 1);
18390
18391 int Bits = 4 /* 16 byte alignment */;
18392
18393 if (DAG.MaskedValueIsZero(Op: Add->getOperand(Num: 1),
18394 Mask: APInt::getAllOnes(numBits: Bits /* alignment */)
18395 .zext(width: Add.getScalarValueSizeInBits()))) {
18396 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18397 for (SDNode *U : BasePtr->users()) {
18398 if (U->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18399 U->getConstantOperandVal(Num: 0) == IID) {
18400 // We've found another LVSL/LVSR, and this address is an aligned
18401 // multiple of that one. The results will be the same, so use the
18402 // one we've just found instead.
18403
18404 return SDValue(U, 0);
18405 }
18406 }
18407 }
18408
18409 if (isa<ConstantSDNode>(Val: Add->getOperand(Num: 1))) {
18410 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18411 for (SDNode *U : BasePtr->users()) {
18412 if (U->getOpcode() == ISD::ADD &&
18413 isa<ConstantSDNode>(Val: U->getOperand(Num: 1)) &&
18414 (Add->getConstantOperandVal(Num: 1) - U->getConstantOperandVal(Num: 1)) %
18415 (1ULL << Bits) ==
18416 0) {
18417 SDNode *OtherAdd = U;
18418 for (SDNode *V : OtherAdd->users()) {
18419 if (V->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18420 V->getConstantOperandVal(Num: 0) == IID) {
18421 return SDValue(V, 0);
18422 }
18423 }
18424 }
18425 }
18426 }
18427 }
18428
18429 // Combine vmaxsw/h/b(a, a's negation) to abs(a)
18430 // Expose the vabsduw/h/b opportunity for down stream
18431 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() &&
18432 (IID == Intrinsic::ppc_altivec_vmaxsw ||
18433 IID == Intrinsic::ppc_altivec_vmaxsh ||
18434 IID == Intrinsic::ppc_altivec_vmaxsb)) {
18435 SDValue V1 = N->getOperand(Num: 1);
18436 SDValue V2 = N->getOperand(Num: 2);
18437 if ((V1.getSimpleValueType() == MVT::v4i32 ||
18438 V1.getSimpleValueType() == MVT::v8i16 ||
18439 V1.getSimpleValueType() == MVT::v16i8) &&
18440 V1.getSimpleValueType() == V2.getSimpleValueType()) {
18441 // (0-a, a)
18442 if (V1.getOpcode() == ISD::SUB &&
18443 ISD::isBuildVectorAllZeros(N: V1.getOperand(i: 0).getNode()) &&
18444 V1.getOperand(i: 1) == V2) {
18445 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V2.getValueType(), Operand: V2);
18446 }
18447 // (a, 0-a)
18448 if (V2.getOpcode() == ISD::SUB &&
18449 ISD::isBuildVectorAllZeros(N: V2.getOperand(i: 0).getNode()) &&
18450 V2.getOperand(i: 1) == V1) {
18451 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18452 }
18453 // (x-y, y-x)
18454 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB &&
18455 V1.getOperand(i: 0) == V2.getOperand(i: 1) &&
18456 V1.getOperand(i: 1) == V2.getOperand(i: 0)) {
18457 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18458 }
18459 }
18460 }
18461 }
18462
18463 break;
18464 case ISD::INTRINSIC_W_CHAIN:
18465 switch (N->getConstantOperandVal(Num: 1)) {
18466 default:
18467 break;
18468 case Intrinsic::ppc_altivec_vsum4sbs:
18469 case Intrinsic::ppc_altivec_vsum4shs:
18470 case Intrinsic::ppc_altivec_vsum4ubs: {
18471 // These sum-across intrinsics only have a chain due to the side effect
18472 // that they may set the SAT bit. If we know the SAT bit will not be set
18473 // for some inputs, we can replace any uses of their chain with the
18474 // input chain.
18475 if (BuildVectorSDNode *BVN =
18476 dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 3))) {
18477 APInt APSplatBits, APSplatUndef;
18478 unsigned SplatBitSize;
18479 bool HasAnyUndefs;
18480 bool BVNIsConstantSplat = BVN->isConstantSplat(
18481 SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 0,
18482 isBigEndian: !Subtarget.isLittleEndian());
18483 // If the constant splat vector is 0, the SAT bit will not be set.
18484 if (BVNIsConstantSplat && APSplatBits == 0)
18485 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
18486 }
18487 return SDValue();
18488 }
18489 case Intrinsic::ppc_vsx_lxvw4x:
18490 case Intrinsic::ppc_vsx_lxvd2x:
18491 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18492 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18493 if (Subtarget.needsSwapsForVSXMemOps())
18494 return expandVSXLoadForLE(N, DCI);
18495 break;
18496 }
18497 break;
18498 case ISD::INTRINSIC_VOID:
18499 // For little endian, VSX stores require generating xxswapd/stxvd2x.
18500 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18501 if (Subtarget.needsSwapsForVSXMemOps()) {
18502 switch (N->getConstantOperandVal(Num: 1)) {
18503 default:
18504 break;
18505 case Intrinsic::ppc_vsx_stxvw4x:
18506 case Intrinsic::ppc_vsx_stxvd2x:
18507 return expandVSXStoreForLE(N, DCI);
18508 }
18509 }
18510 break;
18511 case ISD::BSWAP: {
18512 // Turn BSWAP (LOAD) -> lhbrx/lwbrx.
18513 // For subtargets without LDBRX, we can still do better than the default
18514 // expansion even for 64-bit BSWAP (LOAD).
18515 bool Is64BitBswapOn64BitTgt =
18516 Subtarget.isPPC64() && N->getValueType(ResNo: 0) == MVT::i64;
18517 bool IsSingleUseNormalLd = ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode()) &&
18518 N->getOperand(Num: 0).hasOneUse();
18519 if (IsSingleUseNormalLd &&
18520 (N->getValueType(ResNo: 0) == MVT::i32 || N->getValueType(ResNo: 0) == MVT::i16 ||
18521 (Subtarget.hasLDBRX() && Is64BitBswapOn64BitTgt))) {
18522 SDValue Load = N->getOperand(Num: 0);
18523 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
18524 // Create the byte-swapping load.
18525 SDValue Ops[] = {
18526 LD->getChain(), // Chain
18527 LD->getBasePtr(), // Ptr
18528 DAG.getValueType(N->getValueType(ResNo: 0)) // VT
18529 };
18530 SDValue BSLoad =
18531 DAG.getMemIntrinsicNode(Opcode: PPCISD::LBRX, dl,
18532 VTList: DAG.getVTList(VT1: N->getValueType(ResNo: 0) == MVT::i64 ?
18533 MVT::i64 : MVT::i32, VT2: MVT::Other),
18534 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
18535
18536 // If this is an i16 load, insert the truncate.
18537 SDValue ResVal = BSLoad;
18538 if (N->getValueType(ResNo: 0) == MVT::i16)
18539 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: BSLoad);
18540
18541 // First, combine the bswap away. This makes the value produced by the
18542 // load dead.
18543 DCI.CombineTo(N, Res: ResVal);
18544
18545 // Next, combine the load away, we give it a bogus result value but a real
18546 // chain result. The result value is dead because the bswap is dead.
18547 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
18548
18549 // Return N so it doesn't get rechecked!
18550 return SDValue(N, 0);
18551 }
18552 // Convert this to two 32-bit bswap loads and a BUILD_PAIR. Do this only
18553 // before legalization so that the BUILD_PAIR is handled correctly.
18554 if (!DCI.isBeforeLegalize() || !Is64BitBswapOn64BitTgt ||
18555 !IsSingleUseNormalLd)
18556 return SDValue();
18557 LoadSDNode *LD = cast<LoadSDNode>(Val: N->getOperand(Num: 0));
18558
18559 // Can't split volatile or atomic loads.
18560 if (!LD->isSimple())
18561 return SDValue();
18562 SDValue BasePtr = LD->getBasePtr();
18563 SDValue Lo = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18564 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign());
18565 Lo = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Lo);
18566 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18567 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18568 MachineMemOperand *NewMMO = DAG.getMachineFunction().getMachineMemOperand(
18569 MMO: LD->getMemOperand(), Offset: 4, Size: 4);
18570 SDValue Hi = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr, MMO: NewMMO);
18571 Hi = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Hi);
18572 SDValue Res;
18573 if (Subtarget.isLittleEndian())
18574 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Hi, N2: Lo);
18575 else
18576 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
18577 SDValue TF =
18578 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18579 N1: Hi.getOperand(i: 0).getValue(R: 1), N2: Lo.getOperand(i: 0).getValue(R: 1));
18580 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: TF);
18581 return Res;
18582 }
18583 case PPCISD::VCMP:
18584 // If a VCMP_rec node already exists with exactly the same operands as this
18585 // node, use its result instead of this node (VCMP_rec computes both a CR6
18586 // and a normal output).
18587 //
18588 if (!N->getOperand(Num: 0).hasOneUse() &&
18589 !N->getOperand(Num: 1).hasOneUse() &&
18590 !N->getOperand(Num: 2).hasOneUse()) {
18591
18592 // Scan all of the users of the LHS, looking for VCMP_rec's that match.
18593 SDNode *VCMPrecNode = nullptr;
18594
18595 SDNode *LHSN = N->getOperand(Num: 0).getNode();
18596 for (SDNode *User : LHSN->users())
18597 if (User->getOpcode() == PPCISD::VCMP_rec &&
18598 User->getOperand(Num: 1) == N->getOperand(Num: 1) &&
18599 User->getOperand(Num: 2) == N->getOperand(Num: 2) &&
18600 User->getOperand(Num: 0) == N->getOperand(Num: 0)) {
18601 VCMPrecNode = User;
18602 break;
18603 }
18604
18605 // If there is no VCMP_rec node, or if the flag value has a single use,
18606 // don't transform this.
18607 if (!VCMPrecNode || VCMPrecNode->hasNUsesOfValue(NUses: 0, Value: 1))
18608 break;
18609
18610 // Look at the (necessarily single) use of the flag value. If it has a
18611 // chain, this transformation is more complex. Note that multiple things
18612 // could use the value result, which we should ignore.
18613 SDNode *FlagUser = nullptr;
18614 for (SDNode::use_iterator UI = VCMPrecNode->use_begin();
18615 FlagUser == nullptr; ++UI) {
18616 assert(UI != VCMPrecNode->use_end() && "Didn't find user!");
18617 SDNode *User = UI->getUser();
18618 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
18619 if (User->getOperand(Num: i) == SDValue(VCMPrecNode, 1)) {
18620 FlagUser = User;
18621 break;
18622 }
18623 }
18624 }
18625
18626 // If the user is a MFOCRF instruction, we know this is safe.
18627 // Otherwise we give up for right now.
18628 if (FlagUser->getOpcode() == PPCISD::MFOCRF)
18629 return SDValue(VCMPrecNode, 0);
18630 }
18631 break;
18632 case ISD::BR_CC: {
18633 // If this is a branch on an altivec predicate comparison, lower this so
18634 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This
18635 // lowering is done pre-legalize, because the legalizer lowers the predicate
18636 // compare down to code that is difficult to reassemble.
18637 // This code also handles branches that depend on the result of a store
18638 // conditional.
18639 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
18640 SDValue LHS = N->getOperand(Num: 2), RHS = N->getOperand(Num: 3);
18641
18642 int CompareOpc;
18643 bool isDot;
18644
18645 if (!isa<ConstantSDNode>(Val: RHS) || (CC != ISD::SETEQ && CC != ISD::SETNE))
18646 break;
18647
18648 // Since we are doing this pre-legalize, the RHS can be a constant of
18649 // arbitrary bitwidth which may cause issues when trying to get the value
18650 // from the underlying APInt.
18651 auto RHSAPInt = RHS->getAsAPIntVal();
18652 if (!RHSAPInt.isIntN(N: 64))
18653 break;
18654
18655 unsigned Val = RHSAPInt.getZExtValue();
18656 auto isImpossibleCompare = [&]() {
18657 // If this is a comparison against something other than 0/1, then we know
18658 // that the condition is never/always true.
18659 if (Val != 0 && Val != 1) {
18660 if (CC == ISD::SETEQ) // Cond never true, remove branch.
18661 return N->getOperand(Num: 0);
18662 // Always !=, turn it into an unconditional branch.
18663 return DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other,
18664 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 4));
18665 }
18666 return SDValue();
18667 };
18668 // Combine branches fed by store conditional instructions (st[bhwd]cx).
18669 unsigned StoreWidth = 0;
18670 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
18671 isStoreConditional(Intrin: LHS, StoreWidth)) {
18672 if (SDValue Impossible = isImpossibleCompare())
18673 return Impossible;
18674 PPC::Predicate CompOpc;
18675 // eq 0 => ne
18676 // ne 0 => eq
18677 // eq 1 => eq
18678 // ne 1 => ne
18679 if (Val == 0)
18680 CompOpc = CC == ISD::SETEQ ? PPC::PRED_NE : PPC::PRED_EQ;
18681 else
18682 CompOpc = CC == ISD::SETEQ ? PPC::PRED_EQ : PPC::PRED_NE;
18683
18684 SDValue Ops[] = {LHS.getOperand(i: 0), LHS.getOperand(i: 2), LHS.getOperand(i: 3),
18685 DAG.getConstant(Val: StoreWidth, DL: dl, VT: MVT::i32)};
18686 auto *MemNode = cast<MemSDNode>(Val&: LHS);
18687 SDValue ConstSt = DAG.getMemIntrinsicNode(
18688 Opcode: PPCISD::STORE_COND, dl,
18689 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other, VT3: MVT::Glue), Ops,
18690 MemVT: MemNode->getMemoryVT(), MMO: MemNode->getMemOperand());
18691
18692 SDValue InChain;
18693 // Unchain the branch from the original store conditional.
18694 if (N->getOperand(Num: 0) == LHS.getValue(R: 1))
18695 InChain = LHS.getOperand(i: 0);
18696 else if (N->getOperand(Num: 0).getOpcode() == ISD::TokenFactor) {
18697 SmallVector<SDValue, 4> InChains;
18698 SDValue InTF = N->getOperand(Num: 0);
18699 for (int i = 0, e = InTF.getNumOperands(); i < e; i++)
18700 if (InTF.getOperand(i) != LHS.getValue(R: 1))
18701 InChains.push_back(Elt: InTF.getOperand(i));
18702 InChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: InChains);
18703 }
18704
18705 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: InChain,
18706 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18707 N3: DAG.getRegister(Reg: PPC::CR0, VT: MVT::i32), N4: N->getOperand(Num: 4),
18708 N5: ConstSt.getValue(R: 2));
18709 }
18710
18711 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18712 getVectorCompareInfo(Intrin: LHS, CompareOpc, isDot, Subtarget)) {
18713 assert(isDot && "Can't compare against a vector result!");
18714
18715 if (SDValue Impossible = isImpossibleCompare())
18716 return Impossible;
18717
18718 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0);
18719 // Create the PPCISD altivec 'dot' comparison node.
18720 SDValue Ops[] = {
18721 LHS.getOperand(i: 2), // LHS of compare
18722 LHS.getOperand(i: 3), // RHS of compare
18723 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
18724 };
18725 EVT VTs[] = { LHS.getOperand(i: 2).getValueType(), MVT::Glue };
18726 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
18727
18728 // Unpack the result based on how the target uses it.
18729 PPC::Predicate CompOpc;
18730 switch (LHS.getConstantOperandVal(i: 1)) {
18731 default: // Can't happen, don't crash on invalid number though.
18732 case 0: // Branch on the value of the EQ bit of CR6.
18733 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE;
18734 break;
18735 case 1: // Branch on the inverted value of the EQ bit of CR6.
18736 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ;
18737 break;
18738 case 2: // Branch on the value of the LT bit of CR6.
18739 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE;
18740 break;
18741 case 3: // Branch on the inverted value of the LT bit of CR6.
18742 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT;
18743 break;
18744 }
18745
18746 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
18747 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18748 N3: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32),
18749 N4: N->getOperand(Num: 4), N5: CompNode.getValue(R: 1));
18750 }
18751 break;
18752 }
18753 case ISD::BUILD_VECTOR:
18754 return DAGCombineBuildVector(N, DCI);
18755 case PPCISD::ADDC:
18756 return DAGCombineAddc(N, DCI);
18757 case PPCISD::SUBE:
18758 return DAGCombineSube(N, DCI);
18759
18760 case ISD::BITCAST:
18761 return DAGCombineBitcast(N, DCI);
18762 }
18763
18764 return SDValue();
18765}
18766
18767SDValue
18768PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
18769 SelectionDAG &DAG,
18770 SmallVectorImpl<SDNode *> &Created) const {
18771 // fold (sdiv X, pow2)
18772 EVT VT = N->getValueType(ResNo: 0);
18773 if (VT == MVT::i64 && !Subtarget.isPPC64())
18774 return SDValue();
18775 if ((VT != MVT::i32 && VT != MVT::i64) ||
18776 !(Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()))
18777 return SDValue();
18778
18779 SDLoc DL(N);
18780 SDValue N0 = N->getOperand(Num: 0);
18781
18782 bool IsNegPow2 = Divisor.isNegatedPowerOf2();
18783 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countr_zero();
18784 SDValue ShiftAmt = DAG.getConstant(Val: Lg2, DL, VT);
18785
18786 SDValue Op = DAG.getNode(Opcode: PPCISD::SRA_ADDZE, DL, VT, N1: N0, N2: ShiftAmt);
18787 Created.push_back(Elt: Op.getNode());
18788
18789 if (IsNegPow2) {
18790 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
18791 Created.push_back(Elt: Op.getNode());
18792 }
18793
18794 return Op;
18795}
18796
18797//===----------------------------------------------------------------------===//
18798// Inline Assembly Support
18799//===----------------------------------------------------------------------===//
18800
18801void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
18802 KnownBits &Known,
18803 const APInt &DemandedElts,
18804 const SelectionDAG &DAG,
18805 unsigned Depth) const {
18806 Known.resetAll();
18807 switch (Op.getOpcode()) {
18808 default: break;
18809 case PPCISD::LBRX: {
18810 // lhbrx is known to have the top bits cleared out.
18811 if (cast<VTSDNode>(Val: Op.getOperand(i: 2))->getVT() == MVT::i16)
18812 Known.Zero = 0xFFFF0000;
18813 break;
18814 }
18815 case PPCISD::ADDE: {
18816 if (Op.getResNo() == 0) {
18817 // (0|1), _ = ADDE 0, 0, CARRY
18818 SDValue LHS = Op.getOperand(i: 0);
18819 SDValue RHS = Op.getOperand(i: 1);
18820 if (isNullConstant(V: LHS) && isNullConstant(V: RHS))
18821 Known.Zero = ~1ULL;
18822 }
18823 break;
18824 }
18825 case ISD::INTRINSIC_WO_CHAIN: {
18826 switch (Op.getConstantOperandVal(i: 0)) {
18827 default: break;
18828 case Intrinsic::ppc_altivec_vcmpbfp_p:
18829 case Intrinsic::ppc_altivec_vcmpeqfp_p:
18830 case Intrinsic::ppc_altivec_vcmpequb_p:
18831 case Intrinsic::ppc_altivec_vcmpequh_p:
18832 case Intrinsic::ppc_altivec_vcmpequw_p:
18833 case Intrinsic::ppc_altivec_vcmpequd_p:
18834 case Intrinsic::ppc_altivec_vcmpequq_p:
18835 case Intrinsic::ppc_altivec_vcmpgefp_p:
18836 case Intrinsic::ppc_altivec_vcmpgtfp_p:
18837 case Intrinsic::ppc_altivec_vcmpgtsb_p:
18838 case Intrinsic::ppc_altivec_vcmpgtsh_p:
18839 case Intrinsic::ppc_altivec_vcmpgtsw_p:
18840 case Intrinsic::ppc_altivec_vcmpgtsd_p:
18841 case Intrinsic::ppc_altivec_vcmpgtsq_p:
18842 case Intrinsic::ppc_altivec_vcmpgtub_p:
18843 case Intrinsic::ppc_altivec_vcmpgtuh_p:
18844 case Intrinsic::ppc_altivec_vcmpgtuw_p:
18845 case Intrinsic::ppc_altivec_vcmpgtud_p:
18846 case Intrinsic::ppc_altivec_vcmpgtuq_p:
18847 Known.Zero = ~1U; // All bits but the low one are known to be zero.
18848 break;
18849 }
18850 break;
18851 }
18852 case ISD::INTRINSIC_W_CHAIN: {
18853 switch (Op.getConstantOperandVal(i: 1)) {
18854 default:
18855 break;
18856 case Intrinsic::ppc_load2r:
18857 // Top bits are cleared for load2r (which is the same as lhbrx).
18858 Known.Zero = 0xFFFF0000;
18859 break;
18860 }
18861 break;
18862 }
18863 }
18864}
18865
18866Align PPCTargetLowering::getPrefLoopAlignment(
18867 MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
18868 switch (Subtarget.getCPUDirective()) {
18869 default: break;
18870 case PPC::DIR_970:
18871 case PPC::DIR_PWR4:
18872 case PPC::DIR_PWR5:
18873 case PPC::DIR_PWR5X:
18874 case PPC::DIR_PWR6:
18875 case PPC::DIR_PWR6X:
18876 case PPC::DIR_PWR7:
18877 case PPC::DIR_PWR8:
18878 case PPC::DIR_PWR9:
18879 case PPC::DIR_PWR10:
18880 case PPC::DIR_PWR11:
18881 case PPC::DIR_PWR_FUTURE: {
18882 if (!ML)
18883 break;
18884
18885 if (EnableInnermostLoopAlign32) {
18886 // If the nested loop is an innermost loop, prefer to a 32-byte alignment,
18887 // so that we can decrease cache misses and branch-prediction misses.
18888 // Actual alignment of the loop will depend on the hotness check and other
18889 // logic in alignBlocks.
18890 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty())
18891 return Align(32);
18892 }
18893
18894 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
18895
18896 // For small loops (between 5 and 8 instructions), align to a 32-byte
18897 // boundary so that the entire loop fits in one instruction-cache line.
18898 uint64_t LoopSize = 0;
18899 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I)
18900 for (const MachineInstr &J : **I) {
18901 LoopSize += TII->getInstSizeInBytes(MI: J);
18902 if (LoopSize > 32)
18903 break;
18904 }
18905
18906 if (LoopSize > 16 && LoopSize <= 32)
18907 return Align(32);
18908
18909 break;
18910 }
18911 }
18912
18913 return TargetLowering::getPrefLoopAlignment(ML);
18914}
18915
18916/// getConstraintType - Given a constraint, return the type of
18917/// constraint it is for this target.
18918PPCTargetLowering::ConstraintType
18919PPCTargetLowering::getConstraintType(StringRef Constraint) const {
18920 if (Constraint.size() == 1) {
18921 switch (Constraint[0]) {
18922 default: break;
18923 case 'b':
18924 case 'r':
18925 case 'f':
18926 case 'd':
18927 case 'v':
18928 case 'y':
18929 return C_RegisterClass;
18930 case 'Z':
18931 // FIXME: While Z does indicate a memory constraint, it specifically
18932 // indicates an r+r address (used in conjunction with the 'y' modifier
18933 // in the replacement string). Currently, we're forcing the base
18934 // register to be r0 in the asm printer (which is interpreted as zero)
18935 // and forming the complete address in the second register. This is
18936 // suboptimal.
18937 return C_Memory;
18938 }
18939 } else if (Constraint == "wc") { // individual CR bits.
18940 return C_RegisterClass;
18941 } else if (Constraint == "wa" || Constraint == "wd" ||
18942 Constraint == "wf" || Constraint == "ws" ||
18943 Constraint == "wi" || Constraint == "ww") {
18944 return C_RegisterClass; // VSX registers.
18945 }
18946 return TargetLowering::getConstraintType(Constraint);
18947}
18948
18949/// Examine constraint type and operand type and determine a weight value.
18950/// This object must already have been set up with the operand type
18951/// and the current alternative constraint selected.
18952TargetLowering::ConstraintWeight
18953PPCTargetLowering::getSingleConstraintMatchWeight(
18954 AsmOperandInfo &info, const char *constraint) const {
18955 ConstraintWeight weight = CW_Invalid;
18956 Value *CallOperandVal = info.CallOperandVal;
18957 // If we don't have a value, we can't do a match,
18958 // but allow it at the lowest weight.
18959 if (!CallOperandVal)
18960 return CW_Default;
18961 Type *type = CallOperandVal->getType();
18962
18963 // Look at the constraint type.
18964 if (StringRef(constraint) == "wc" && type->isIntegerTy(BitWidth: 1))
18965 return CW_Register; // an individual CR bit.
18966 else if ((StringRef(constraint) == "wa" ||
18967 StringRef(constraint) == "wd" ||
18968 StringRef(constraint) == "wf") &&
18969 type->isVectorTy())
18970 return CW_Register;
18971 else if (StringRef(constraint) == "wi" && type->isIntegerTy(BitWidth: 64))
18972 return CW_Register; // just hold 64-bit integers data.
18973 else if (StringRef(constraint) == "ws" && type->isDoubleTy())
18974 return CW_Register;
18975 else if (StringRef(constraint) == "ww" && type->isFloatTy())
18976 return CW_Register;
18977
18978 switch (*constraint) {
18979 default:
18980 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
18981 break;
18982 case 'b':
18983 if (type->isIntegerTy())
18984 weight = CW_Register;
18985 break;
18986 case 'f':
18987 if (type->isFloatTy())
18988 weight = CW_Register;
18989 break;
18990 case 'd':
18991 if (type->isDoubleTy())
18992 weight = CW_Register;
18993 break;
18994 case 'v':
18995 if (type->isVectorTy())
18996 weight = CW_Register;
18997 break;
18998 case 'y':
18999 weight = CW_Register;
19000 break;
19001 case 'Z':
19002 weight = CW_Memory;
19003 break;
19004 }
19005 return weight;
19006}
19007
19008std::pair<unsigned, const TargetRegisterClass *>
19009PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
19010 StringRef Constraint,
19011 MVT VT) const {
19012 if (Constraint.size() == 1) {
19013 // GCC RS6000 Constraint Letters
19014 switch (Constraint[0]) {
19015 case 'b': // R1-R31
19016 if (VT == MVT::i64 && Subtarget.isPPC64())
19017 return std::make_pair(x: 0U, y: &PPC::G8RC_NOX0RegClass);
19018 return std::make_pair(x: 0U, y: &PPC::GPRC_NOR0RegClass);
19019 case 'r': // R0-R31
19020 if (VT == MVT::i64 && Subtarget.isPPC64())
19021 return std::make_pair(x: 0U, y: &PPC::G8RCRegClass);
19022 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
19023 // 'd' and 'f' constraints are both defined to be "the floating point
19024 // registers", where one is for 32-bit and the other for 64-bit. We don't
19025 // really care overly much here so just give them all the same reg classes.
19026 case 'd':
19027 case 'f':
19028 if (Subtarget.hasSPE()) {
19029 if (VT == MVT::f32 || VT == MVT::i32)
19030 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
19031 if (VT == MVT::f64 || VT == MVT::i64)
19032 return std::make_pair(x: 0U, y: &PPC::SPERCRegClass);
19033 } else {
19034 if (VT == MVT::f32 || VT == MVT::i32)
19035 return std::make_pair(x: 0U, y: &PPC::F4RCRegClass);
19036 if (VT == MVT::f64 || VT == MVT::i64)
19037 return std::make_pair(x: 0U, y: &PPC::F8RCRegClass);
19038 }
19039 break;
19040 case 'v':
19041 if (Subtarget.hasAltivec() && VT.isVector())
19042 return std::make_pair(x: 0U, y: &PPC::VRRCRegClass);
19043 else if (Subtarget.hasVSX())
19044 // Scalars in Altivec registers only make sense with VSX.
19045 return std::make_pair(x: 0U, y: &PPC::VFRCRegClass);
19046 break;
19047 case 'y': // crrc
19048 return std::make_pair(x: 0U, y: &PPC::CRRCRegClass);
19049 }
19050 } else if (Constraint == "wc" && Subtarget.useCRBits()) {
19051 // An individual CR bit.
19052 return std::make_pair(x: 0U, y: &PPC::CRBITRCRegClass);
19053 } else if ((Constraint == "wa" || Constraint == "wd" ||
19054 Constraint == "wf" || Constraint == "wi") &&
19055 Subtarget.hasVSX()) {
19056 // A VSX register for either a scalar (FP) or vector. There is no
19057 // support for single precision scalars on subtargets prior to Power8.
19058 if (VT.isVector())
19059 return std::make_pair(x: 0U, y: &PPC::VSRCRegClass);
19060 if (VT == MVT::f32 && Subtarget.hasP8Vector())
19061 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
19062 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
19063 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) {
19064 if (VT == MVT::f32 && Subtarget.hasP8Vector())
19065 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
19066 else
19067 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
19068 } else if (Constraint == "lr") {
19069 if (VT == MVT::i64)
19070 return std::make_pair(x: 0U, y: &PPC::LR8RCRegClass);
19071 else
19072 return std::make_pair(x: 0U, y: &PPC::LRRCRegClass);
19073 }
19074
19075 // Handle special cases of physical registers that are not properly handled
19076 // by the base class.
19077 if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') {
19078 // If we name a VSX register, we can't defer to the base class because it
19079 // will not recognize the correct register (their names will be VSL{0-31}
19080 // and V{0-31} so they won't match). So we match them here.
19081 if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') {
19082 int VSNum = atoi(nptr: Constraint.data() + 3);
19083 assert(VSNum >= 0 && VSNum <= 63 &&
19084 "Attempted to access a vsr out of range");
19085 if (VSNum < 32)
19086 return std::make_pair(x: PPC::VSL0 + VSNum, y: &PPC::VSRCRegClass);
19087 return std::make_pair(x: PPC::V0 + VSNum - 32, y: &PPC::VSRCRegClass);
19088 }
19089
19090 // For float registers, we can't defer to the base class as it will match
19091 // the SPILLTOVSRRC class.
19092 if (Constraint.size() > 3 && Constraint[1] == 'f') {
19093 int RegNum = atoi(nptr: Constraint.data() + 2);
19094 if (RegNum > 31 || RegNum < 0)
19095 report_fatal_error(reason: "Invalid floating point register number");
19096 if (VT == MVT::f32 || VT == MVT::i32)
19097 return Subtarget.hasSPE()
19098 ? std::make_pair(x: PPC::R0 + RegNum, y: &PPC::GPRCRegClass)
19099 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F4RCRegClass);
19100 if (VT == MVT::f64 || VT == MVT::i64)
19101 return Subtarget.hasSPE()
19102 ? std::make_pair(x: PPC::S0 + RegNum, y: &PPC::SPERCRegClass)
19103 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F8RCRegClass);
19104 }
19105 }
19106
19107 std::pair<unsigned, const TargetRegisterClass *> R =
19108 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19109
19110 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers
19111 // (which we call X[0-9]+). If a 64-bit value has been requested, and a
19112 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent
19113 // register.
19114 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use
19115 // the AsmName field from *RegisterInfo.td, then this would not be necessary.
19116 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() &&
19117 PPC::GPRCRegClass.contains(Reg: R.first))
19118 return std::make_pair(x: TRI->getMatchingSuperReg(Reg: R.first,
19119 SubIdx: PPC::sub_32, RC: &PPC::G8RCRegClass),
19120 y: &PPC::G8RCRegClass);
19121
19122 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same.
19123 if (!R.second && StringRef("{cc}").equals_insensitive(RHS: Constraint)) {
19124 R.first = PPC::CR0;
19125 R.second = &PPC::CRRCRegClass;
19126 }
19127 // FIXME: This warning should ideally be emitted in the front end.
19128 if (Subtarget.isAIXABI() && !Subtarget.isAIXExtendedAltivecABI()) {
19129 if (((R.first >= PPC::V20 && R.first <= PPC::V31) ||
19130 (R.first >= PPC::VF20 && R.first <= PPC::VF31)) &&
19131 (R.second == &PPC::VSRCRegClass || R.second == &PPC::VSFRCRegClass))
19132 errs() << "warning: vector registers 20 to 32 are reserved in the "
19133 "default AIX AltiVec ABI and cannot be used\n";
19134 }
19135
19136 return R;
19137}
19138
19139/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
19140/// vector. If it is invalid, don't add anything to Ops.
19141void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
19142 StringRef Constraint,
19143 std::vector<SDValue> &Ops,
19144 SelectionDAG &DAG) const {
19145 SDValue Result;
19146
19147 // Only support length 1 constraints.
19148 if (Constraint.size() > 1)
19149 return;
19150
19151 char Letter = Constraint[0];
19152 switch (Letter) {
19153 default: break;
19154 case 'I':
19155 case 'J':
19156 case 'K':
19157 case 'L':
19158 case 'M':
19159 case 'N':
19160 case 'O':
19161 case 'P': {
19162 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Val&: Op);
19163 if (!CST) return; // Must be an immediate to match.
19164 SDLoc dl(Op);
19165 int64_t Value = CST->getSExtValue();
19166 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative
19167 // numbers are printed as such.
19168 switch (Letter) {
19169 default: llvm_unreachable("Unknown constraint letter!");
19170 case 'I': // "I" is a signed 16-bit constant.
19171 if (isInt<16>(x: Value))
19172 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19173 break;
19174 case 'J': // "J" is a constant with only the high-order 16 bits nonzero.
19175 if (isShiftedUInt<16, 16>(x: Value))
19176 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19177 break;
19178 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits.
19179 if (isShiftedInt<16, 16>(x: Value))
19180 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19181 break;
19182 case 'K': // "K" is a constant with only the low-order 16 bits nonzero.
19183 if (isUInt<16>(x: Value))
19184 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19185 break;
19186 case 'M': // "M" is a constant that is greater than 31.
19187 if (Value > 31)
19188 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19189 break;
19190 case 'N': // "N" is a positive constant that is an exact power of two.
19191 if (Value > 0 && isPowerOf2_64(Value))
19192 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19193 break;
19194 case 'O': // "O" is the constant zero.
19195 if (Value == 0)
19196 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19197 break;
19198 case 'P': // "P" is a constant whose negation is a signed 16-bit constant.
19199 if (isInt<16>(x: -Value))
19200 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19201 break;
19202 }
19203 break;
19204 }
19205 }
19206
19207 if (Result.getNode()) {
19208 Ops.push_back(x: Result);
19209 return;
19210 }
19211
19212 // Handle standard constraint letters.
19213 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
19214}
19215
19216void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I,
19217 SmallVectorImpl<SDValue> &Ops,
19218 SelectionDAG &DAG) const {
19219 if (I.getNumOperands() <= 1)
19220 return;
19221 if (!isa<ConstantSDNode>(Val: Ops[1].getNode()))
19222 return;
19223 auto IntrinsicID = Ops[1].getNode()->getAsZExtVal();
19224 if (IntrinsicID != Intrinsic::ppc_tdw && IntrinsicID != Intrinsic::ppc_tw &&
19225 IntrinsicID != Intrinsic::ppc_trapd && IntrinsicID != Intrinsic::ppc_trap)
19226 return;
19227
19228 if (MDNode *MDN = I.getMetadata(KindID: LLVMContext::MD_annotation))
19229 Ops.push_back(Elt: DAG.getMDNode(MD: MDN));
19230}
19231
19232// isLegalAddressingMode - Return true if the addressing mode represented
19233// by AM is legal for this target, for a load/store of the specified type.
19234bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL,
19235 const AddrMode &AM, Type *Ty,
19236 unsigned AS,
19237 Instruction *I) const {
19238 // Vector type r+i form is supported since power9 as DQ form. We don't check
19239 // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC,
19240 // imm form is preferred and the offset can be adjusted to use imm form later
19241 // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and
19242 // max offset to check legal addressing mode, we should be a little aggressive
19243 // to contain other offsets for that LSRUse.
19244 if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector())
19245 return false;
19246
19247 // PPC allows a sign-extended 16-bit immediate field.
19248 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
19249 return false;
19250
19251 // No global is ever allowed as a base.
19252 if (AM.BaseGV)
19253 return false;
19254
19255 // PPC only support r+r,
19256 switch (AM.Scale) {
19257 case 0: // "r+i" or just "i", depending on HasBaseReg.
19258 break;
19259 case 1:
19260 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
19261 return false;
19262 // Otherwise we have r+r or r+i.
19263 break;
19264 case 2:
19265 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
19266 return false;
19267 // Allow 2*r as r+r.
19268 break;
19269 default:
19270 // No other scales are supported.
19271 return false;
19272 }
19273
19274 return true;
19275}
19276
19277SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
19278 SelectionDAG &DAG) const {
19279 MachineFunction &MF = DAG.getMachineFunction();
19280 MachineFrameInfo &MFI = MF.getFrameInfo();
19281 MFI.setReturnAddressIsTaken(true);
19282
19283 SDLoc dl(Op);
19284 unsigned Depth = Op.getConstantOperandVal(i: 0);
19285
19286 // Make sure the function does not optimize away the store of the RA to
19287 // the stack.
19288 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
19289 FuncInfo->setLRStoreRequired();
19290 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
19291
19292 if (Depth > 0) {
19293 // The link register (return address) is saved in the caller's frame
19294 // not the callee's stack frame. So we must get the caller's frame
19295 // address and load the return address at the LR offset from there.
19296 SDValue FrameAddr =
19297 DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19298 Ptr: LowerFRAMEADDR(Op, DAG), PtrInfo: MachinePointerInfo());
19299 SDValue Offset =
19300 DAG.getConstant(Val: Subtarget.getFrameLowering()->getReturnSaveOffset(), DL: dl,
19301 VT: Subtarget.getScalarIntVT());
19302 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(),
19303 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FrameAddr, N2: Offset),
19304 PtrInfo: MachinePointerInfo());
19305 }
19306
19307 // Just load the return address off the stack.
19308 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
19309 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: RetAddrFI,
19310 PtrInfo: MachinePointerInfo());
19311}
19312
19313SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
19314 SelectionDAG &DAG) const {
19315 SDLoc dl(Op);
19316 unsigned Depth = Op.getConstantOperandVal(i: 0);
19317
19318 MachineFunction &MF = DAG.getMachineFunction();
19319 MachineFrameInfo &MFI = MF.getFrameInfo();
19320 MFI.setFrameAddressIsTaken(true);
19321
19322 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
19323 bool isPPC64 = PtrVT == MVT::i64;
19324
19325 // Naked functions never have a frame pointer, and so we use r1. For all
19326 // other functions, this decision must be delayed until during PEI.
19327 unsigned FrameReg;
19328 if (MF.getFunction().hasFnAttribute(Kind: Attribute::Naked))
19329 FrameReg = isPPC64 ? PPC::X1 : PPC::R1;
19330 else
19331 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP;
19332
19333 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: FrameReg,
19334 VT: PtrVT);
19335 while (Depth--)
19336 FrameAddr = DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19337 Ptr: FrameAddr, PtrInfo: MachinePointerInfo());
19338 return FrameAddr;
19339}
19340
19341#define GET_REGISTER_MATCHER
19342#include "PPCGenAsmMatcher.inc"
19343
19344Register PPCTargetLowering::getRegisterByName(const char *RegName, LLT VT,
19345 const MachineFunction &MF) const {
19346 bool IsPPC64 = Subtarget.isPPC64();
19347
19348 bool Is64Bit = IsPPC64 && VT == LLT::scalar(SizeInBits: 64);
19349 if (!Is64Bit && VT != LLT::scalar(SizeInBits: 32))
19350 report_fatal_error(reason: "Invalid register global variable type");
19351
19352 Register Reg = MatchRegisterName(Name: RegName);
19353 if (!Reg)
19354 return Reg;
19355
19356 // FIXME: Unable to generate code for `-O2` but okay for `-O0`.
19357 // Need followup investigation as to why.
19358 if ((IsPPC64 && Reg == PPC::R2) || Reg == PPC::R0)
19359 report_fatal_error(reason: Twine("Trying to reserve an invalid register \"" +
19360 StringRef(RegName) + "\"."));
19361
19362 // Convert GPR to GP8R register for 64bit.
19363 if (Is64Bit && StringRef(RegName).starts_with_insensitive(Prefix: "r"))
19364 Reg = Reg.id() - PPC::R0 + PPC::X0;
19365
19366 return Reg;
19367}
19368
19369bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
19370 // 32-bit SVR4 ABI access everything as got-indirect.
19371 if (Subtarget.is32BitELFABI())
19372 return true;
19373
19374 // AIX accesses everything indirectly through the TOC, which is similar to
19375 // the GOT.
19376 if (Subtarget.isAIXABI())
19377 return true;
19378
19379 CodeModel::Model CModel = getTargetMachine().getCodeModel();
19380 // If it is small or large code model, module locals are accessed
19381 // indirectly by loading their address from .toc/.got.
19382 if (CModel == CodeModel::Small || CModel == CodeModel::Large)
19383 return true;
19384
19385 // JumpTable and BlockAddress are accessed as got-indirect.
19386 if (isa<JumpTableSDNode>(Val: GA) || isa<BlockAddressSDNode>(Val: GA))
19387 return true;
19388
19389 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: GA))
19390 return Subtarget.isGVIndirectSymbol(GV: G->getGlobal());
19391
19392 return false;
19393}
19394
19395bool
19396PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
19397 // The PowerPC target isn't yet aware of offsets.
19398 return false;
19399}
19400
19401void PPCTargetLowering::getTgtMemIntrinsic(
19402 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
19403 MachineFunction &MF, unsigned Intrinsic) const {
19404 IntrinsicInfo Info;
19405 switch (Intrinsic) {
19406 case Intrinsic::ppc_atomicrmw_xchg_i128:
19407 case Intrinsic::ppc_atomicrmw_add_i128:
19408 case Intrinsic::ppc_atomicrmw_sub_i128:
19409 case Intrinsic::ppc_atomicrmw_nand_i128:
19410 case Intrinsic::ppc_atomicrmw_and_i128:
19411 case Intrinsic::ppc_atomicrmw_or_i128:
19412 case Intrinsic::ppc_atomicrmw_xor_i128:
19413 case Intrinsic::ppc_cmpxchg_i128:
19414 Info.opc = ISD::INTRINSIC_W_CHAIN;
19415 Info.memVT = MVT::i128;
19416 Info.ptrVal = I.getArgOperand(i: 0);
19417 Info.offset = 0;
19418 Info.align = Align(16);
19419 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
19420 MachineMemOperand::MOVolatile;
19421 Infos.push_back(Elt: Info);
19422 return;
19423 case Intrinsic::ppc_atomic_load_i128:
19424 Info.opc = ISD::INTRINSIC_W_CHAIN;
19425 Info.memVT = MVT::i128;
19426 Info.ptrVal = I.getArgOperand(i: 0);
19427 Info.offset = 0;
19428 Info.align = Align(16);
19429 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
19430 Infos.push_back(Elt: Info);
19431 return;
19432 case Intrinsic::ppc_atomic_store_i128:
19433 Info.opc = ISD::INTRINSIC_VOID;
19434 Info.memVT = MVT::i128;
19435 Info.ptrVal = I.getArgOperand(i: 2);
19436 Info.offset = 0;
19437 Info.align = Align(16);
19438 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19439 Infos.push_back(Elt: Info);
19440 return;
19441 case Intrinsic::ppc_altivec_lvx:
19442 case Intrinsic::ppc_altivec_lvxl:
19443 case Intrinsic::ppc_altivec_lvebx:
19444 case Intrinsic::ppc_altivec_lvehx:
19445 case Intrinsic::ppc_altivec_lvewx:
19446 case Intrinsic::ppc_vsx_lxvd2x:
19447 case Intrinsic::ppc_vsx_lxvw4x:
19448 case Intrinsic::ppc_vsx_lxvd2x_be:
19449 case Intrinsic::ppc_vsx_lxvw4x_be:
19450 case Intrinsic::ppc_vsx_lxvl:
19451 case Intrinsic::ppc_vsx_lxvll: {
19452 EVT VT;
19453 switch (Intrinsic) {
19454 case Intrinsic::ppc_altivec_lvebx:
19455 VT = MVT::i8;
19456 break;
19457 case Intrinsic::ppc_altivec_lvehx:
19458 VT = MVT::i16;
19459 break;
19460 case Intrinsic::ppc_altivec_lvewx:
19461 VT = MVT::i32;
19462 break;
19463 case Intrinsic::ppc_vsx_lxvd2x:
19464 case Intrinsic::ppc_vsx_lxvd2x_be:
19465 VT = MVT::v2f64;
19466 break;
19467 default:
19468 VT = MVT::v4i32;
19469 break;
19470 }
19471
19472 Info.opc = ISD::INTRINSIC_W_CHAIN;
19473 Info.memVT = VT;
19474 Info.ptrVal = I.getArgOperand(i: 0);
19475 Info.offset = -VT.getStoreSize()+1;
19476 Info.size = 2*VT.getStoreSize()-1;
19477 Info.align = Align(1);
19478 Info.flags = MachineMemOperand::MOLoad;
19479 Infos.push_back(Elt: Info);
19480 return;
19481 }
19482 case Intrinsic::ppc_altivec_stvx:
19483 case Intrinsic::ppc_altivec_stvxl:
19484 case Intrinsic::ppc_altivec_stvebx:
19485 case Intrinsic::ppc_altivec_stvehx:
19486 case Intrinsic::ppc_altivec_stvewx:
19487 case Intrinsic::ppc_vsx_stxvd2x:
19488 case Intrinsic::ppc_vsx_stxvw4x:
19489 case Intrinsic::ppc_vsx_stxvd2x_be:
19490 case Intrinsic::ppc_vsx_stxvw4x_be:
19491 case Intrinsic::ppc_vsx_stxvl:
19492 case Intrinsic::ppc_vsx_stxvll: {
19493 EVT VT;
19494 switch (Intrinsic) {
19495 case Intrinsic::ppc_altivec_stvebx:
19496 VT = MVT::i8;
19497 break;
19498 case Intrinsic::ppc_altivec_stvehx:
19499 VT = MVT::i16;
19500 break;
19501 case Intrinsic::ppc_altivec_stvewx:
19502 VT = MVT::i32;
19503 break;
19504 case Intrinsic::ppc_vsx_stxvd2x:
19505 case Intrinsic::ppc_vsx_stxvd2x_be:
19506 VT = MVT::v2f64;
19507 break;
19508 default:
19509 VT = MVT::v4i32;
19510 break;
19511 }
19512
19513 Info.opc = ISD::INTRINSIC_VOID;
19514 Info.memVT = VT;
19515 Info.ptrVal = I.getArgOperand(i: 1);
19516 Info.offset = -VT.getStoreSize()+1;
19517 Info.size = 2*VT.getStoreSize()-1;
19518 Info.align = Align(1);
19519 Info.flags = MachineMemOperand::MOStore;
19520 Infos.push_back(Elt: Info);
19521 return;
19522 }
19523 case Intrinsic::ppc_stdcx:
19524 case Intrinsic::ppc_stwcx:
19525 case Intrinsic::ppc_sthcx:
19526 case Intrinsic::ppc_stbcx: {
19527 EVT VT;
19528 auto Alignment = Align(8);
19529 switch (Intrinsic) {
19530 case Intrinsic::ppc_stdcx:
19531 VT = MVT::i64;
19532 break;
19533 case Intrinsic::ppc_stwcx:
19534 VT = MVT::i32;
19535 Alignment = Align(4);
19536 break;
19537 case Intrinsic::ppc_sthcx:
19538 VT = MVT::i16;
19539 Alignment = Align(2);
19540 break;
19541 case Intrinsic::ppc_stbcx:
19542 VT = MVT::i8;
19543 Alignment = Align(1);
19544 break;
19545 }
19546 Info.opc = ISD::INTRINSIC_W_CHAIN;
19547 Info.memVT = VT;
19548 Info.ptrVal = I.getArgOperand(i: 0);
19549 Info.offset = 0;
19550 Info.align = Alignment;
19551 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19552 Infos.push_back(Elt: Info);
19553 return;
19554 }
19555 default:
19556 break;
19557 }
19558}
19559
19560/// It returns EVT::Other if the type should be determined using generic
19561/// target-independent logic.
19562EVT PPCTargetLowering::getOptimalMemOpType(
19563 LLVMContext &Context, const MemOp &Op,
19564 const AttributeList &FuncAttributes) const {
19565 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None) {
19566 // We should use Altivec/VSX loads and stores when available. For unaligned
19567 // addresses, unaligned VSX loads are only fast starting with the P8.
19568 if (Subtarget.hasAltivec() && Op.size() >= 16) {
19569 if (Op.isMemset() && Subtarget.hasVSX()) {
19570 uint64_t TailSize = Op.size() % 16;
19571 // For memset lowering, EXTRACT_VECTOR_ELT tries to return constant
19572 // element if vector element type matches tail store. For tail size
19573 // 3/4, the tail store is i32, v4i32 cannot be used, need a legal one.
19574 if (TailSize > 2 && TailSize <= 4) {
19575 return MVT::v8i16;
19576 }
19577 return MVT::v4i32;
19578 }
19579 if (Op.isAligned(AlignCheck: Align(16)) || Subtarget.hasP8Vector())
19580 return MVT::v4i32;
19581 }
19582 }
19583
19584 if (Subtarget.isPPC64()) {
19585 return MVT::i64;
19586 }
19587
19588 return MVT::i32;
19589}
19590
19591/// Returns true if it is beneficial to convert a load of a constant
19592/// to just the constant itself.
19593bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
19594 Type *Ty) const {
19595 assert(Ty->isIntegerTy());
19596
19597 unsigned BitSize = Ty->getPrimitiveSizeInBits();
19598 return !(BitSize == 0 || BitSize > 64);
19599}
19600
19601bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
19602 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19603 return false;
19604 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
19605 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
19606 return NumBits1 == 64 && NumBits2 == 32;
19607}
19608
19609bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
19610 if (!VT1.isInteger() || !VT2.isInteger())
19611 return false;
19612 unsigned NumBits1 = VT1.getSizeInBits();
19613 unsigned NumBits2 = VT2.getSizeInBits();
19614 return NumBits1 == 64 && NumBits2 == 32;
19615}
19616
19617bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
19618 // Generally speaking, zexts are not free, but they are free when they can be
19619 // folded with other operations.
19620 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) {
19621 EVT MemVT = LD->getMemoryVT();
19622 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 ||
19623 (Subtarget.isPPC64() && MemVT == MVT::i32)) &&
19624 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
19625 LD->getExtensionType() == ISD::ZEXTLOAD))
19626 return true;
19627 }
19628
19629 // FIXME: Add other cases...
19630 // - 32-bit shifts with a zext to i64
19631 // - zext after ctlz, bswap, etc.
19632 // - zext after and by a constant mask
19633
19634 return TargetLowering::isZExtFree(Val, VT2);
19635}
19636
19637bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const {
19638 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
19639 "invalid fpext types");
19640 // Extending to float128 is not free.
19641 if (DestVT == MVT::f128)
19642 return false;
19643 return true;
19644}
19645
19646bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
19647 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19648}
19649
19650bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const {
19651 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19652}
19653
19654bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
19655 MachineMemOperand::Flags,
19656 unsigned *Fast) const {
19657 if (!EnablePPCUnaligned)
19658 return false;
19659
19660 // PowerPC supports unaligned memory access for simple non-vector types.
19661 // Although accessing unaligned addresses is not as efficient as accessing
19662 // aligned addresses, it is generally more efficient than manual expansion,
19663 // and generally only traps for software emulation when crossing page
19664 // boundaries.
19665
19666 if (!VT.isSimple())
19667 return false;
19668
19669 if (VT.isFloatingPoint() && !VT.isVector() &&
19670 !Subtarget.allowsUnalignedFPAccess())
19671 return false;
19672
19673 if (VT.getSimpleVT().isVector()) {
19674 if (Subtarget.hasVSX()) {
19675 if (VT != MVT::v2f64 && VT != MVT::v2i64 &&
19676 VT != MVT::v4f32 && VT != MVT::v4i32)
19677 return false;
19678 } else {
19679 return false;
19680 }
19681 }
19682
19683 if (VT == MVT::ppcf128)
19684 return false;
19685
19686 if (Fast)
19687 *Fast = 1;
19688
19689 return true;
19690}
19691
19692bool PPCTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
19693 SDValue C) const {
19694 // Check integral scalar types.
19695 if (!VT.isScalarInteger())
19696 return false;
19697 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
19698 if (!ConstNode->getAPIntValue().isSignedIntN(N: 64))
19699 return false;
19700 // This transformation will generate >= 2 operations. But the following
19701 // cases will generate <= 2 instructions during ISEL. So exclude them.
19702 // 1. If the constant multiplier fits 16 bits, it can be handled by one
19703 // HW instruction, ie. MULLI
19704 // 2. If the multiplier after shifted fits 16 bits, an extra shift
19705 // instruction is needed than case 1, ie. MULLI and RLDICR
19706 int64_t Imm = ConstNode->getSExtValue();
19707 unsigned Shift = llvm::countr_zero<uint64_t>(Val: Imm);
19708 Imm >>= Shift;
19709 if (isInt<16>(x: Imm))
19710 return false;
19711 uint64_t UImm = static_cast<uint64_t>(Imm);
19712 if (isPowerOf2_64(Value: UImm + 1) || isPowerOf2_64(Value: UImm - 1) ||
19713 isPowerOf2_64(Value: 1 - UImm) || isPowerOf2_64(Value: -1 - UImm))
19714 return true;
19715 }
19716 return false;
19717}
19718
19719bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19720 EVT VT) const {
19721 return isFMAFasterThanFMulAndFAdd(
19722 F: MF.getFunction(), Ty: VT.getTypeForEVT(Context&: MF.getFunction().getContext()));
19723}
19724
19725bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
19726 Type *Ty) const {
19727 if (Subtarget.hasSPE() || Subtarget.useSoftFloat())
19728 return false;
19729 switch (Ty->getScalarType()->getTypeID()) {
19730 case Type::FloatTyID:
19731 case Type::DoubleTyID:
19732 return true;
19733 case Type::FP128TyID:
19734 return Subtarget.hasP9Vector();
19735 default:
19736 return false;
19737 }
19738}
19739
19740// FIXME: add more patterns which are not profitable to hoist.
19741bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const {
19742 if (!I->hasOneUse())
19743 return true;
19744
19745 Instruction *User = I->user_back();
19746 assert(User && "A single use instruction with no uses.");
19747
19748 switch (I->getOpcode()) {
19749 case Instruction::FMul: {
19750 // Don't break FMA, PowerPC prefers FMA.
19751 if (User->getOpcode() != Instruction::FSub &&
19752 User->getOpcode() != Instruction::FAdd)
19753 return true;
19754
19755 const Function *F = I->getFunction();
19756 const DataLayout &DL = F->getDataLayout();
19757 Type *Ty = User->getOperand(i: 0)->getType();
19758 bool AllowContract = I->getFastMathFlags().allowContract() &&
19759 User->getFastMathFlags().allowContract();
19760
19761 return !(isFMAFasterThanFMulAndFAdd(F: *F, Ty) &&
19762 isOperationLegalOrCustom(Op: ISD::FMA, VT: getValueType(DL, Ty)) &&
19763 AllowContract);
19764 }
19765 case Instruction::Load: {
19766 // Don't break "store (load float*)" pattern, this pattern will be combined
19767 // to "store (load int32)" in later InstCombine pass. See function
19768 // combineLoadToOperationType. On PowerPC, loading a float point takes more
19769 // cycles than loading a 32 bit integer.
19770 LoadInst *LI = cast<LoadInst>(Val: I);
19771 // For the loads that combineLoadToOperationType does nothing, like
19772 // ordered load, it should be profitable to hoist them.
19773 // For swifterror load, it can only be used for pointer to pointer type, so
19774 // later type check should get rid of this case.
19775 if (!LI->isUnordered())
19776 return true;
19777
19778 if (User->getOpcode() != Instruction::Store)
19779 return true;
19780
19781 if (I->getType()->getTypeID() != Type::FloatTyID)
19782 return true;
19783
19784 return false;
19785 }
19786 default:
19787 return true;
19788 }
19789 return true;
19790}
19791
19792const MCPhysReg *
19793PPCTargetLowering::getScratchRegisters(CallingConv::ID) const {
19794 // LR is a callee-save register, but we must treat it as clobbered by any call
19795 // site. Hence we include LR in the scratch registers, which are in turn added
19796 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies
19797 // to CTR, which is used by any indirect call.
19798 static const MCPhysReg ScratchRegs[] = {
19799 PPC::X12, PPC::LR8, PPC::CTR8, 0
19800 };
19801
19802 return ScratchRegs;
19803}
19804
19805Register PPCTargetLowering::getExceptionPointerRegister(
19806 ExceptionHandling EH, const Constant *PersonalityFn) const {
19807 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3;
19808}
19809
19810Register PPCTargetLowering::getExceptionSelectorRegister(
19811 ExceptionHandling EH, const Constant *PersonalityFn) const {
19812 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4;
19813}
19814
19815bool
19816PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
19817 EVT VT , unsigned DefinedValues) const {
19818 if (VT == MVT::v2i64)
19819 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
19820
19821 if (Subtarget.hasVSX())
19822 return true;
19823
19824 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
19825}
19826
19827Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const {
19828 if (!EnableILPPref || Subtarget.enableMachineScheduler())
19829 return TargetLowering::getSchedulingPreference(N);
19830
19831 return Sched::ILP;
19832}
19833
19834// Create a fast isel object.
19835FastISel *PPCTargetLowering::createFastISel(
19836 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo,
19837 const LibcallLoweringInfo *LibcallLowering) const {
19838 return PPC::createFastISel(FuncInfo, LibInfo, LibcallLowering);
19839}
19840
19841// 'Inverted' means the FMA opcode after negating one multiplicand.
19842// For example, (fma -a b c) = (fnmsub a b c)
19843static unsigned invertFMAOpcode(unsigned Opc) {
19844 switch (Opc) {
19845 default:
19846 llvm_unreachable("Invalid FMA opcode for PowerPC!");
19847 case ISD::FMA:
19848 return PPCISD::FNMSUB;
19849 case PPCISD::FNMSUB:
19850 return ISD::FMA;
19851 }
19852}
19853
19854SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
19855 bool LegalOps, bool OptForSize,
19856 NegatibleCost &Cost,
19857 unsigned Depth) const {
19858 if (Depth > SelectionDAG::MaxRecursionDepth)
19859 return SDValue();
19860
19861 unsigned Opc = Op.getOpcode();
19862 EVT VT = Op.getValueType();
19863 SDNodeFlags Flags = Op.getNode()->getFlags();
19864
19865 switch (Opc) {
19866 case PPCISD::FNMSUB:
19867 if (!Op.hasOneUse() || !isTypeLegal(VT))
19868 break;
19869
19870 SDValue N0 = Op.getOperand(i: 0);
19871 SDValue N1 = Op.getOperand(i: 1);
19872 SDValue N2 = Op.getOperand(i: 2);
19873 SDLoc Loc(Op);
19874
19875 NegatibleCost N2Cost = NegatibleCost::Expensive;
19876 SDValue NegN2 =
19877 getNegatedExpression(Op: N2, DAG, LegalOps, OptForSize, Cost&: N2Cost, Depth: Depth + 1);
19878
19879 if (!NegN2)
19880 return SDValue();
19881
19882 // (fneg (fnmsub a b c)) => (fnmsub (fneg a) b (fneg c))
19883 // (fneg (fnmsub a b c)) => (fnmsub a (fneg b) (fneg c))
19884 // These transformations may change sign of zeroes. For example,
19885 // -(-ab-(-c))=-0 while -(-(ab-c))=+0 when a=b=c=1.
19886 if (Flags.hasNoSignedZeros()) {
19887 // Try and choose the cheaper one to negate.
19888 NegatibleCost N0Cost = NegatibleCost::Expensive;
19889 SDValue NegN0 = getNegatedExpression(Op: N0, DAG, LegalOps, OptForSize,
19890 Cost&: N0Cost, Depth: Depth + 1);
19891
19892 NegatibleCost N1Cost = NegatibleCost::Expensive;
19893 SDValue NegN1 = getNegatedExpression(Op: N1, DAG, LegalOps, OptForSize,
19894 Cost&: N1Cost, Depth: Depth + 1);
19895
19896 if (NegN0 && N0Cost <= N1Cost) {
19897 Cost = std::min(a: N0Cost, b: N2Cost);
19898 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: NegN0, N2: N1, N3: NegN2, Flags);
19899 } else if (NegN1) {
19900 Cost = std::min(a: N1Cost, b: N2Cost);
19901 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: N0, N2: NegN1, N3: NegN2, Flags);
19902 }
19903 }
19904
19905 // (fneg (fnmsub a b c)) => (fma a b (fneg c))
19906 if (isOperationLegal(Op: ISD::FMA, VT)) {
19907 Cost = N2Cost;
19908 return DAG.getNode(Opcode: ISD::FMA, DL: Loc, VT, N1: N0, N2: N1, N3: NegN2, Flags);
19909 }
19910
19911 break;
19912 }
19913
19914 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps, OptForSize,
19915 Cost, Depth);
19916}
19917
19918// Override to enable LOAD_STACK_GUARD lowering on Linux.
19919bool PPCTargetLowering::useLoadStackGuardNode(const Module &M) const {
19920 if (M.getStackProtectorGuard() == "tls" || Subtarget.isTargetLinux())
19921 return true;
19922 return TargetLowering::useLoadStackGuardNode(M);
19923}
19924
19925bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
19926 bool ForCodeSize) const {
19927 if (!VT.isSimple() || !Subtarget.hasVSX())
19928 return false;
19929
19930 switch(VT.getSimpleVT().SimpleTy) {
19931 default:
19932 // For FP types that are currently not supported by PPC backend, return
19933 // false. Examples: f16, f80.
19934 return false;
19935 case MVT::f32:
19936 case MVT::f64: {
19937 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
19938 // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP.
19939 return true;
19940 }
19941 bool IsExact;
19942 APSInt IntResult(16, false);
19943 // The rounding mode doesn't really matter because we only care about floats
19944 // that can be converted to integers exactly.
19945 Imm.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
19946 // For exact values in the range [-16, 15] we can materialize the float.
19947 if (IsExact && IntResult <= 15 && IntResult >= -16)
19948 return true;
19949 return Imm.isZero();
19950 }
19951 case MVT::ppcf128:
19952 return Imm.isPosZero();
19953 }
19954}
19955
19956// For vector shift operation op, fold
19957// (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y)
19958static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N,
19959 SelectionDAG &DAG) {
19960 SDValue N0 = N->getOperand(Num: 0);
19961 SDValue N1 = N->getOperand(Num: 1);
19962 EVT VT = N0.getValueType();
19963 unsigned OpSizeInBits = VT.getScalarSizeInBits();
19964 unsigned Opcode = N->getOpcode();
19965 unsigned TargetOpcode;
19966
19967 switch (Opcode) {
19968 default:
19969 llvm_unreachable("Unexpected shift operation");
19970 case ISD::SHL:
19971 TargetOpcode = PPCISD::SHL;
19972 break;
19973 case ISD::SRL:
19974 TargetOpcode = PPCISD::SRL;
19975 break;
19976 case ISD::SRA:
19977 TargetOpcode = PPCISD::SRA;
19978 break;
19979 }
19980
19981 if (VT.isVector() && TLI.isOperationLegal(Op: Opcode, VT) &&
19982 N1->getOpcode() == ISD::AND)
19983 if (ConstantSDNode *Mask = isConstOrConstSplat(N: N1->getOperand(Num: 1)))
19984 if (Mask->getZExtValue() == OpSizeInBits - 1)
19985 return DAG.getNode(Opcode: TargetOpcode, DL: SDLoc(N), VT, N1: N0, N2: N1->getOperand(Num: 0));
19986
19987 return SDValue();
19988}
19989
19990SDValue PPCTargetLowering::combineVectorShift(SDNode *N,
19991 DAGCombinerInfo &DCI) const {
19992 EVT VT = N->getValueType(ResNo: 0);
19993 assert(VT.isVector() && "Vector type expected.");
19994
19995 unsigned Opc = N->getOpcode();
19996 assert((Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) &&
19997 "Unexpected opcode.");
19998
19999 if (!isOperationLegal(Op: Opc, VT))
20000 return SDValue();
20001
20002 EVT EltTy = VT.getScalarType();
20003 unsigned EltBits = EltTy.getSizeInBits();
20004 if (EltTy != MVT::i64 && EltTy != MVT::i32)
20005 return SDValue();
20006
20007 SDValue N1 = N->getOperand(Num: 1);
20008 uint64_t SplatBits = 0;
20009 bool AddSplatCase = false;
20010 unsigned OpcN1 = N1.getOpcode();
20011 if (OpcN1 == PPCISD::VADD_SPLAT &&
20012 N1.getConstantOperandVal(i: 1) == VT.getVectorNumElements()) {
20013 AddSplatCase = true;
20014 SplatBits = N1.getConstantOperandVal(i: 0);
20015 }
20016
20017 if (!AddSplatCase) {
20018 if (OpcN1 != ISD::BUILD_VECTOR)
20019 return SDValue();
20020
20021 unsigned SplatBitSize;
20022 bool HasAnyUndefs;
20023 APInt APSplatBits, APSplatUndef;
20024 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val&: N1);
20025 bool BVNIsConstantSplat =
20026 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
20027 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
20028 if (!BVNIsConstantSplat || SplatBitSize != EltBits)
20029 return SDValue();
20030 SplatBits = APSplatBits.getZExtValue();
20031 }
20032
20033 SDLoc DL(N);
20034 SDValue N0 = N->getOperand(Num: 0);
20035 // PPC vector shifts by word/double look at only the low 5/6 bits of the
20036 // shift vector, which means the max value is 31/63. A shift vector of all
20037 // 1s will be truncated to 31/63, which is useful as vspltiw is limited to
20038 // -16 to 15 range.
20039 if (SplatBits == (EltBits - 1)) {
20040 unsigned NewOpc;
20041 switch (Opc) {
20042 case ISD::SHL:
20043 NewOpc = PPCISD::SHL;
20044 break;
20045 case ISD::SRL:
20046 NewOpc = PPCISD::SRL;
20047 break;
20048 case ISD::SRA:
20049 NewOpc = PPCISD::SRA;
20050 break;
20051 }
20052 SDValue SplatOnes = getCanonicalConstSplat(Val: 255, SplatSize: 1, VT, DAG&: DCI.DAG, dl: DL);
20053 return DCI.DAG.getNode(Opcode: NewOpc, DL, VT, N1: N0, N2: SplatOnes);
20054 }
20055
20056 if (Opc != ISD::SHL || !isOperationLegal(Op: ISD::ADD, VT))
20057 return SDValue();
20058
20059 // For 64-bit there is no splat immediate so we want to catch shift by 1 here
20060 // before the BUILD_VECTOR is replaced by a load.
20061 if (EltTy != MVT::i64 || SplatBits != 1)
20062 return SDValue();
20063
20064 return DCI.DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT, N1: N0, N2: N0);
20065}
20066
20067SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const {
20068 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20069 return Value;
20070
20071 if (N->getValueType(ResNo: 0).isVector())
20072 return combineVectorShift(N, DCI);
20073
20074 SDValue N0 = N->getOperand(Num: 0);
20075 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
20076 if (!Subtarget.isISA3_0() || !Subtarget.isPPC64() ||
20077 N0.getOpcode() != ISD::SIGN_EXTEND ||
20078 N0.getOperand(i: 0).getValueType() != MVT::i32 || CN1 == nullptr ||
20079 N->getValueType(ResNo: 0) != MVT::i64)
20080 return SDValue();
20081
20082 // We can't save an operation here if the value is already extended, and
20083 // the existing shift is easier to combine.
20084 SDValue ExtsSrc = N0.getOperand(i: 0);
20085 if (ExtsSrc.getOpcode() == ISD::TRUNCATE &&
20086 ExtsSrc.getOperand(i: 0).getOpcode() == ISD::AssertSext)
20087 return SDValue();
20088
20089 SDLoc DL(N0);
20090 SDValue ShiftBy = SDValue(CN1, 0);
20091 // We want the shift amount to be i32 on the extswli, but the shift could
20092 // have an i64.
20093 if (ShiftBy.getValueType() == MVT::i64)
20094 ShiftBy = DCI.DAG.getConstant(Val: CN1->getZExtValue(), DL, VT: MVT::i32);
20095
20096 return DCI.DAG.getNode(Opcode: PPCISD::EXTSWSLI, DL, VT: MVT::i64, N1: N0->getOperand(Num: 0),
20097 N2: ShiftBy);
20098}
20099
20100SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const {
20101 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20102 return Value;
20103
20104 if (N->getValueType(ResNo: 0).isVector())
20105 return combineVectorShift(N, DCI);
20106
20107 return SDValue();
20108}
20109
20110SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const {
20111 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20112 return Value;
20113
20114 if (N->getValueType(ResNo: 0).isVector())
20115 return combineVectorShift(N, DCI);
20116
20117 return SDValue();
20118}
20119
20120// Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1))
20121// Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0))
20122// When C is zero, the equation (addi Z, -C) can be simplified to Z
20123// Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types
20124static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG,
20125 const PPCSubtarget &Subtarget) {
20126 if (!Subtarget.isPPC64())
20127 return SDValue();
20128
20129 SDValue LHS = N->getOperand(Num: 0);
20130 SDValue RHS = N->getOperand(Num: 1);
20131
20132 auto isZextOfCompareWithConstant = [](SDValue Op) {
20133 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() ||
20134 Op.getValueType() != MVT::i64)
20135 return false;
20136
20137 SDValue Cmp = Op.getOperand(i: 0);
20138 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() ||
20139 Cmp.getOperand(i: 0).getValueType() != MVT::i64)
20140 return false;
20141
20142 if (auto *Constant = dyn_cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1))) {
20143 int64_t NegConstant = 0 - Constant->getSExtValue();
20144 // Due to the limitations of the addi instruction,
20145 // -C is required to be [-32768, 32767].
20146 return isInt<16>(x: NegConstant);
20147 }
20148
20149 return false;
20150 };
20151
20152 bool LHSHasPattern = isZextOfCompareWithConstant(LHS);
20153 bool RHSHasPattern = isZextOfCompareWithConstant(RHS);
20154
20155 // If there is a pattern, canonicalize a zext operand to the RHS.
20156 if (LHSHasPattern && !RHSHasPattern)
20157 std::swap(a&: LHS, b&: RHS);
20158 else if (!LHSHasPattern && !RHSHasPattern)
20159 return SDValue();
20160
20161 SDLoc DL(N);
20162 EVT CarryType = Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
20163 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: CarryType);
20164 SDValue Cmp = RHS.getOperand(i: 0);
20165 SDValue Z = Cmp.getOperand(i: 0);
20166 auto *Constant = cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1));
20167 int64_t NegConstant = 0 - Constant->getSExtValue();
20168
20169 switch(cast<CondCodeSDNode>(Val: Cmp.getOperand(i: 2))->get()) {
20170 default: break;
20171 case ISD::SETNE: {
20172 // when C == 0
20173 // --> addze X, (addic Z, -1).carry
20174 // /
20175 // add X, (zext(setne Z, C))--
20176 // \ when -32768 <= -C <= 32767 && C != 0
20177 // --> addze X, (addic (addi Z, -C), -1).carry
20178 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20179 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20180 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20181 SDValue Addc =
20182 DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20183 N1: AddOrZ, N2: DAG.getAllOnesConstant(DL, VT: MVT::i64),
20184 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20185 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20186 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64),
20187 N3: SDValue(Addc.getNode(), 1));
20188 }
20189 case ISD::SETEQ: {
20190 // when C == 0
20191 // --> addze X, (subfic Z, 0).carry
20192 // /
20193 // add X, (zext(sete Z, C))--
20194 // \ when -32768 <= -C <= 32767 && C != 0
20195 // --> addze X, (subfic (addi Z, -C), 0).carry
20196 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20197 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20198 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20199 SDValue Subc =
20200 DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20201 N1: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N2: AddOrZ,
20202 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20203 SDValue Invert = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Subc.getValue(R: 1),
20204 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
20205 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20206 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N3: Invert);
20207 }
20208 }
20209
20210 return SDValue();
20211}
20212
20213// Transform
20214// (add C1, (MAT_PCREL_ADDR GlobalAddr+C2)) to
20215// (MAT_PCREL_ADDR GlobalAddr+(C1+C2))
20216// In this case both C1 and C2 must be known constants.
20217// C1+C2 must fit into a 34 bit signed integer.
20218static SDValue combineADDToMAT_PCREL_ADDR(SDNode *N, SelectionDAG &DAG,
20219 const PPCSubtarget &Subtarget) {
20220 if (!Subtarget.isUsingPCRelativeCalls())
20221 return SDValue();
20222
20223 // Check both Operand 0 and Operand 1 of the ADD node for the PCRel node.
20224 // If we find that node try to cast the Global Address and the Constant.
20225 SDValue LHS = N->getOperand(Num: 0);
20226 SDValue RHS = N->getOperand(Num: 1);
20227
20228 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20229 std::swap(a&: LHS, b&: RHS);
20230
20231 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20232 return SDValue();
20233
20234 // Operand zero of PPCISD::MAT_PCREL_ADDR is the GA node.
20235 GlobalAddressSDNode *GSDN = dyn_cast<GlobalAddressSDNode>(Val: LHS.getOperand(i: 0));
20236 ConstantSDNode* ConstNode = dyn_cast<ConstantSDNode>(Val&: RHS);
20237
20238 // Check that both casts succeeded.
20239 if (!GSDN || !ConstNode)
20240 return SDValue();
20241
20242 int64_t NewOffset = GSDN->getOffset() + ConstNode->getSExtValue();
20243 SDLoc DL(GSDN);
20244
20245 // The signed int offset needs to fit in 34 bits.
20246 if (!isInt<34>(x: NewOffset))
20247 return SDValue();
20248
20249 // The new global address is a copy of the old global address except
20250 // that it has the updated Offset.
20251 SDValue GA =
20252 DAG.getTargetGlobalAddress(GV: GSDN->getGlobal(), DL, VT: GSDN->getValueType(ResNo: 0),
20253 offset: NewOffset, TargetFlags: GSDN->getTargetFlags());
20254 SDValue MatPCRel =
20255 DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: GSDN->getValueType(ResNo: 0), Operand: GA);
20256 return MatPCRel;
20257}
20258
20259// Transform (add X, (build_vector (T 1), (T 1), ...)) -> (sub X, (XXLEQVOnes))
20260// XXLEQVOnes creates an all-1s vector (0xFFFFFFFF...) efficiently via xxleqv
20261// Mathematical identity: X + 1 = X - (-1)
20262// Applies to v4i32, v2i64, v8i16, v16i8 where all elements are constant 1
20263// Requirement: VSX feature for efficient xxleqv generation
20264static SDValue combineADDToSUB(SDNode *N, SelectionDAG &DAG,
20265 const PPCSubtarget &Subtarget) {
20266
20267 EVT VT = N->getValueType(ResNo: 0);
20268 if (!Subtarget.hasVSX())
20269 return SDValue();
20270
20271 // Handle v2i64, v4i32, v8i16 and v16i8 types
20272 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
20273 VT == MVT::v2i64))
20274 return SDValue();
20275
20276 SDValue LHS = N->getOperand(Num: 0);
20277 SDValue RHS = N->getOperand(Num: 1);
20278
20279 // Check if RHS is BUILD_VECTOR
20280 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
20281 return SDValue();
20282
20283 // Check if all the elements are 1
20284 unsigned NumOfEles = RHS.getNumOperands();
20285 for (unsigned i = 0; i < NumOfEles; ++i) {
20286 auto *CN = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i));
20287 if (!CN || CN->getSExtValue() != 1)
20288 return SDValue();
20289 }
20290 SDLoc DL(N);
20291
20292 SDValue MinusOne = DAG.getConstant(Val: APInt::getAllOnes(numBits: 32), DL, VT: MVT::i32);
20293 SmallVector<SDValue, 4> Ops(4, MinusOne);
20294 SDValue AllOnesVec = DAG.getBuildVector(VT: MVT::v4i32, DL, Ops);
20295
20296 // Bitcast to the target vector type
20297 SDValue Bitcast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: AllOnesVec);
20298
20299 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Bitcast);
20300}
20301
20302SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const {
20303 if (auto Value = combineADDToADDZE(N, DAG&: DCI.DAG, Subtarget))
20304 return Value;
20305
20306 if (auto Value = combineADDToMAT_PCREL_ADDR(N, DAG&: DCI.DAG, Subtarget))
20307 return Value;
20308
20309 if (auto Value = combineADDToSUB(N, DAG&: DCI.DAG, Subtarget))
20310 return Value;
20311 return SDValue();
20312}
20313
20314// Detect TRUNCATE operations on bitcasts of float128 values.
20315// What we are looking for here is the situtation where we extract a subset
20316// of bits from a 128 bit float.
20317// This can be of two forms:
20318// 1) BITCAST of f128 feeding TRUNCATE
20319// 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE
20320// The reason this is required is because we do not have a legal i128 type
20321// and so we want to prevent having to store the f128 and then reload part
20322// of it.
20323SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N,
20324 DAGCombinerInfo &DCI) const {
20325 // If we are using CRBits then try that first.
20326 if (Subtarget.useCRBits()) {
20327 // Check if CRBits did anything and return that if it did.
20328 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI))
20329 return CRTruncValue;
20330 }
20331
20332 SDLoc dl(N);
20333 SDValue Op0 = N->getOperand(Num: 0);
20334
20335 // Looking for a truncate of i128 to i64.
20336 if (Op0.getValueType() != MVT::i128 || N->getValueType(ResNo: 0) != MVT::i64)
20337 return SDValue();
20338
20339 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0;
20340
20341 // SRL feeding TRUNCATE.
20342 if (Op0.getOpcode() == ISD::SRL) {
20343 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
20344 // The right shift has to be by 64 bits.
20345 if (!ConstNode || ConstNode->getZExtValue() != 64)
20346 return SDValue();
20347
20348 // Switch the element number to extract.
20349 EltToExtract = EltToExtract ? 0 : 1;
20350 // Update Op0 past the SRL.
20351 Op0 = Op0.getOperand(i: 0);
20352 }
20353
20354 // BITCAST feeding a TRUNCATE possibly via SRL.
20355 if (Op0.getOpcode() == ISD::BITCAST &&
20356 Op0.getValueType() == MVT::i128 &&
20357 Op0.getOperand(i: 0).getValueType() == MVT::f128) {
20358 SDValue Bitcast = DCI.DAG.getBitcast(VT: MVT::v2i64, V: Op0.getOperand(i: 0));
20359 return DCI.DAG.getNode(
20360 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Bitcast,
20361 N2: DCI.DAG.getTargetConstant(Val: EltToExtract, DL: dl, VT: MVT::i32));
20362 }
20363 return SDValue();
20364}
20365
20366SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const {
20367 SelectionDAG &DAG = DCI.DAG;
20368
20369 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N: N->getOperand(Num: 1));
20370 if (!ConstOpOrElement)
20371 return SDValue();
20372
20373 // An imul is usually smaller than the alternative sequence for legal type.
20374 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
20375 isOperationLegal(Op: ISD::MUL, VT: N->getValueType(ResNo: 0)))
20376 return SDValue();
20377
20378 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool {
20379 switch (this->Subtarget.getCPUDirective()) {
20380 default:
20381 // TODO: enhance the condition for subtarget before pwr8
20382 return false;
20383 case PPC::DIR_PWR8:
20384 // type mul add shl
20385 // scalar 4 1 1
20386 // vector 7 2 2
20387 return true;
20388 case PPC::DIR_PWR9:
20389 case PPC::DIR_PWR10:
20390 case PPC::DIR_PWR11:
20391 case PPC::DIR_PWR_FUTURE:
20392 // type mul add shl
20393 // scalar 5 2 2
20394 // vector 7 2 2
20395
20396 // The cycle RATIO of related operations are showed as a table above.
20397 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both
20398 // scalar and vector type. For 2 instrs patterns, add/sub + shl
20399 // are 4, it is always profitable; but for 3 instrs patterns
20400 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6.
20401 // So we should only do it for vector type.
20402 return IsAddOne && IsNeg ? VT.isVector() : true;
20403 }
20404 };
20405
20406 EVT VT = N->getValueType(ResNo: 0);
20407 SDLoc DL(N);
20408
20409 const APInt &MulAmt = ConstOpOrElement->getAPIntValue();
20410 bool IsNeg = MulAmt.isNegative();
20411 APInt MulAmtAbs = MulAmt.abs();
20412
20413 if ((MulAmtAbs - 1).isPowerOf2()) {
20414 // (mul x, 2^N + 1) => (add (shl x, N), x)
20415 // (mul x, -(2^N + 1)) => -(add (shl x, N), x)
20416
20417 if (!IsProfitable(IsNeg, true, VT))
20418 return SDValue();
20419
20420 SDValue Op0 = N->getOperand(Num: 0);
20421 SDValue Op1 =
20422 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20423 N2: DAG.getConstant(Val: (MulAmtAbs - 1).logBase2(), DL, VT));
20424 SDValue Res = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: Op1);
20425
20426 if (!IsNeg)
20427 return Res;
20428
20429 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Res);
20430 } else if ((MulAmtAbs + 1).isPowerOf2()) {
20431 // (mul x, 2^N - 1) => (sub (shl x, N), x)
20432 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
20433
20434 if (!IsProfitable(IsNeg, false, VT))
20435 return SDValue();
20436
20437 SDValue Op0 = N->getOperand(Num: 0);
20438 SDValue Op1 =
20439 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20440 N2: DAG.getConstant(Val: (MulAmtAbs + 1).logBase2(), DL, VT));
20441
20442 if (!IsNeg)
20443 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op1, N2: Op0);
20444 else
20445 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0, N2: Op1);
20446
20447 } else {
20448 return SDValue();
20449 }
20450}
20451
20452// Combine fma-like op (like fnmsub) with fnegs to appropriate op. Do this
20453// in combiner since we need to check SD flags and other subtarget features.
20454SDValue PPCTargetLowering::combineFMALike(SDNode *N,
20455 DAGCombinerInfo &DCI) const {
20456 SDValue N0 = N->getOperand(Num: 0);
20457 SDValue N1 = N->getOperand(Num: 1);
20458 SDValue N2 = N->getOperand(Num: 2);
20459 SDNodeFlags Flags = N->getFlags();
20460 EVT VT = N->getValueType(ResNo: 0);
20461 SelectionDAG &DAG = DCI.DAG;
20462 unsigned Opc = N->getOpcode();
20463 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
20464 bool LegalOps = !DCI.isBeforeLegalizeOps();
20465 SDLoc Loc(N);
20466
20467 if (!isOperationLegal(Op: ISD::FMA, VT))
20468 return SDValue();
20469
20470 // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
20471 // since (fnmsub a b c)=-0 while c-ab=+0.
20472 if (!Flags.hasNoSignedZeros())
20473 return SDValue();
20474
20475 // (fma (fneg a) b c) => (fnmsub a b c)
20476 // (fnmsub (fneg a) b c) => (fma a b c)
20477 if (SDValue NegN0 = getCheaperNegatedExpression(Op: N0, DAG, LegalOps, OptForSize: CodeSize))
20478 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: NegN0, N2: N1, N3: N2, Flags);
20479
20480 // (fma a (fneg b) c) => (fnmsub a b c)
20481 // (fnmsub a (fneg b) c) => (fma a b c)
20482 if (SDValue NegN1 = getCheaperNegatedExpression(Op: N1, DAG, LegalOps, OptForSize: CodeSize))
20483 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: N0, N2: NegN1, N3: N2, Flags);
20484
20485 return SDValue();
20486}
20487
20488bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
20489 // Only duplicate to increase tail-calls for the 64bit SysV ABIs.
20490 if (!Subtarget.is64BitELFABI())
20491 return false;
20492
20493 // If not a tail call then no need to proceed.
20494 if (!CI->isTailCall())
20495 return false;
20496
20497 // If sibling calls have been disabled and tail-calls aren't guaranteed
20498 // there is no reason to duplicate.
20499 auto &TM = getTargetMachine();
20500 if (!TM.Options.GuaranteedTailCallOpt && !EnableSCO)
20501 return false;
20502
20503 // Can't tail call a function called indirectly, or if it has variadic args.
20504 const Function *Callee = CI->getCalledFunction();
20505 if (!Callee || Callee->isVarArg())
20506 return false;
20507
20508 // Make sure the callee and caller calling conventions are eligible for tco.
20509 const Function *Caller = CI->getParent()->getParent();
20510 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC: Caller->getCallingConv(),
20511 CalleeCC: CI->getCallingConv()))
20512 return false;
20513
20514 // If the function is local then we have a good chance at tail-calling it
20515 return getTargetMachine().shouldAssumeDSOLocal(GV: Callee);
20516}
20517
20518bool PPCTargetLowering::
20519isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
20520 const Value *Mask = AndI.getOperand(i: 1);
20521 // If the mask is suitable for andi. or andis. we should sink the and.
20522 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Mask)) {
20523 // Can't handle constants wider than 64-bits.
20524 if (CI->getBitWidth() > 64)
20525 return false;
20526 int64_t ConstVal = CI->getZExtValue();
20527 return isUInt<16>(x: ConstVal) ||
20528 (isUInt<16>(x: ConstVal >> 16) && !(ConstVal & 0xFFFF));
20529 }
20530
20531 // For non-constant masks, we can always use the record-form and.
20532 return true;
20533}
20534
20535/// getAddrModeForFlags - Based on the set of address flags, select the most
20536/// optimal instruction format to match by.
20537PPC::AddrMode PPCTargetLowering::getAddrModeForFlags(unsigned Flags) const {
20538 // This is not a node we should be handling here.
20539 if (Flags == PPC::MOF_None)
20540 return PPC::AM_None;
20541 // Unaligned D-Forms are tried first, followed by the aligned D-Forms.
20542 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DForm))
20543 if ((Flags & FlagSet) == FlagSet)
20544 return PPC::AM_DForm;
20545 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DSForm))
20546 if ((Flags & FlagSet) == FlagSet)
20547 return PPC::AM_DSForm;
20548 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DQForm))
20549 if ((Flags & FlagSet) == FlagSet)
20550 return PPC::AM_DQForm;
20551 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_PrefixDForm))
20552 if ((Flags & FlagSet) == FlagSet)
20553 return PPC::AM_PrefixDForm;
20554 // If no other forms are selected, return an X-Form as it is the most
20555 // general addressing mode.
20556 return PPC::AM_XForm;
20557}
20558
20559/// Set alignment flags based on whether or not the Frame Index is aligned.
20560/// Utilized when computing flags for address computation when selecting
20561/// load and store instructions.
20562static void setAlignFlagsForFI(SDValue N, unsigned &FlagSet,
20563 SelectionDAG &DAG) {
20564 bool IsAdd = ((N.getOpcode() == ISD::ADD) || (N.getOpcode() == ISD::OR));
20565 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: IsAdd ? N.getOperand(i: 0) : N);
20566 if (!FI)
20567 return;
20568 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
20569 unsigned FrameIndexAlign = MFI.getObjectAlign(ObjectIdx: FI->getIndex()).value();
20570 // If this is (add $FI, $S16Imm), the alignment flags are already set
20571 // based on the immediate. We just need to clear the alignment flags
20572 // if the FI alignment is weaker.
20573 if ((FrameIndexAlign % 4) != 0)
20574 FlagSet &= ~PPC::MOF_RPlusSImm16Mult4;
20575 if ((FrameIndexAlign % 16) != 0)
20576 FlagSet &= ~PPC::MOF_RPlusSImm16Mult16;
20577 // If the address is a plain FrameIndex, set alignment flags based on
20578 // FI alignment.
20579 if (!IsAdd) {
20580 if ((FrameIndexAlign % 4) == 0)
20581 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20582 if ((FrameIndexAlign % 16) == 0)
20583 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20584 }
20585}
20586
20587/// Given a node, compute flags that are used for address computation when
20588/// selecting load and store instructions. The flags computed are stored in
20589/// FlagSet. This function takes into account whether the node is a constant,
20590/// an ADD, OR, or a constant, and computes the address flags accordingly.
20591static void computeFlagsForAddressComputation(SDValue N, unsigned &FlagSet,
20592 SelectionDAG &DAG) {
20593 // Set the alignment flags for the node depending on if the node is
20594 // 4-byte or 16-byte aligned.
20595 auto SetAlignFlagsForImm = [&](uint64_t Imm) {
20596 if ((Imm & 0x3) == 0)
20597 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20598 if ((Imm & 0xf) == 0)
20599 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20600 };
20601
20602 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
20603 // All 32-bit constants can be computed as LIS + Disp.
20604 const APInt &ConstImm = CN->getAPIntValue();
20605 if (ConstImm.isSignedIntN(N: 32)) { // Flag to handle 32-bit constants.
20606 FlagSet |= PPC::MOF_AddrIsSImm32;
20607 SetAlignFlagsForImm(ConstImm.getZExtValue());
20608 setAlignFlagsForFI(N, FlagSet, DAG);
20609 }
20610 if (ConstImm.isSignedIntN(N: 34)) // Flag to handle 34-bit constants.
20611 FlagSet |= PPC::MOF_RPlusSImm34;
20612 else // Let constant materialization handle large constants.
20613 FlagSet |= PPC::MOF_NotAddNorCst;
20614 } else if (N.getOpcode() == ISD::ADD || provablyDisjointOr(DAG, N)) {
20615 // This address can be represented as an addition of:
20616 // - Register + Imm16 (possibly a multiple of 4/16)
20617 // - Register + Imm34
20618 // - Register + PPCISD::Lo
20619 // - Register + Register
20620 // In any case, we won't have to match this as Base + Zero.
20621 SDValue RHS = N.getOperand(i: 1);
20622 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: RHS)) {
20623 const APInt &ConstImm = CN->getAPIntValue();
20624 if (ConstImm.isSignedIntN(N: 16)) {
20625 FlagSet |= PPC::MOF_RPlusSImm16; // Signed 16-bit immediates.
20626 SetAlignFlagsForImm(ConstImm.getZExtValue());
20627 setAlignFlagsForFI(N, FlagSet, DAG);
20628 }
20629 if (ConstImm.isSignedIntN(N: 34))
20630 FlagSet |= PPC::MOF_RPlusSImm34; // Signed 34-bit immediates.
20631 else
20632 FlagSet |= PPC::MOF_RPlusR; // Register.
20633 } else if (RHS.getOpcode() == PPCISD::Lo && !RHS.getConstantOperandVal(i: 1))
20634 FlagSet |= PPC::MOF_RPlusLo; // PPCISD::Lo.
20635 else
20636 FlagSet |= PPC::MOF_RPlusR;
20637 } else { // The address computation is not a constant or an addition.
20638 setAlignFlagsForFI(N, FlagSet, DAG);
20639 FlagSet |= PPC::MOF_NotAddNorCst;
20640 }
20641}
20642
20643static bool isPCRelNode(SDValue N) {
20644 return (N.getOpcode() == PPCISD::MAT_PCREL_ADDR ||
20645 isValidPCRelNode<ConstantPoolSDNode>(N) ||
20646 isValidPCRelNode<GlobalAddressSDNode>(N) ||
20647 isValidPCRelNode<JumpTableSDNode>(N) ||
20648 isValidPCRelNode<BlockAddressSDNode>(N));
20649}
20650
20651/// computeMOFlags - Given a node N and it's Parent (a MemSDNode), compute
20652/// the address flags of the load/store instruction that is to be matched.
20653unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N,
20654 SelectionDAG &DAG) const {
20655 unsigned FlagSet = PPC::MOF_None;
20656
20657 // Compute subtarget flags.
20658 if (!Subtarget.hasP9Vector())
20659 FlagSet |= PPC::MOF_SubtargetBeforeP9;
20660 else
20661 FlagSet |= PPC::MOF_SubtargetP9;
20662
20663 if (Subtarget.hasPrefixInstrs())
20664 FlagSet |= PPC::MOF_SubtargetP10;
20665
20666 if (Subtarget.hasSPE())
20667 FlagSet |= PPC::MOF_SubtargetSPE;
20668
20669 // Check if we have a PCRel node and return early.
20670 if ((FlagSet & PPC::MOF_SubtargetP10) && isPCRelNode(N))
20671 return FlagSet;
20672
20673 // If the node is the paired load/store intrinsics, compute flags for
20674 // address computation and return early.
20675 unsigned ParentOp = Parent->getOpcode();
20676 if (Subtarget.isISA3_1() && ((ParentOp == ISD::INTRINSIC_W_CHAIN) ||
20677 (ParentOp == ISD::INTRINSIC_VOID))) {
20678 unsigned ID = Parent->getConstantOperandVal(Num: 1);
20679 if ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) {
20680 SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp)
20681 ? Parent->getOperand(Num: 2)
20682 : Parent->getOperand(Num: 3);
20683 computeFlagsForAddressComputation(N: IntrinOp, FlagSet, DAG);
20684 FlagSet |= PPC::MOF_Vector;
20685 return FlagSet;
20686 }
20687 }
20688
20689 // Mark this as something we don't want to handle here if it is atomic
20690 // or pre-increment instruction.
20691 if (const LSBaseSDNode *LSB = dyn_cast<LSBaseSDNode>(Val: Parent))
20692 if (LSB->isIndexed())
20693 return PPC::MOF_None;
20694
20695 // Compute in-memory type flags. This is based on if there are scalars,
20696 // floats or vectors.
20697 const MemSDNode *MN = dyn_cast<MemSDNode>(Val: Parent);
20698 assert(MN && "Parent should be a MemSDNode!");
20699 EVT MemVT = MN->getMemoryVT();
20700 unsigned Size = MemVT.getSizeInBits();
20701 if (MemVT.isScalarInteger()) {
20702 assert(Size <= 128 &&
20703 "Not expecting scalar integers larger than 16 bytes!");
20704 if (Size < 32)
20705 FlagSet |= PPC::MOF_SubWordInt;
20706 else if (Size == 32)
20707 FlagSet |= PPC::MOF_WordInt;
20708 else
20709 FlagSet |= PPC::MOF_DoubleWordInt;
20710 } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors.
20711 if (Size == 128)
20712 FlagSet |= PPC::MOF_Vector;
20713 else if (Size == 256) {
20714 assert(Subtarget.pairedVectorMemops() &&
20715 "256-bit vectors are only available when paired vector memops is "
20716 "enabled!");
20717 FlagSet |= PPC::MOF_Vector;
20718 } else
20719 llvm_unreachable("Not expecting illegal vectors!");
20720 } else { // Floating point type: can be scalar, f128 or vector types.
20721 if (Size == 32 || Size == 64)
20722 FlagSet |= PPC::MOF_ScalarFloat;
20723 else if (MemVT == MVT::f128 || MemVT.isVector())
20724 FlagSet |= PPC::MOF_Vector;
20725 else
20726 llvm_unreachable("Not expecting illegal scalar floats!");
20727 }
20728
20729 // Compute flags for address computation.
20730 computeFlagsForAddressComputation(N, FlagSet, DAG);
20731
20732 // Compute type extension flags.
20733 if (const LoadSDNode *LN = dyn_cast<LoadSDNode>(Val: Parent)) {
20734 switch (LN->getExtensionType()) {
20735 case ISD::SEXTLOAD:
20736 FlagSet |= PPC::MOF_SExt;
20737 break;
20738 case ISD::EXTLOAD:
20739 case ISD::ZEXTLOAD:
20740 FlagSet |= PPC::MOF_ZExt;
20741 break;
20742 case ISD::NON_EXTLOAD:
20743 FlagSet |= PPC::MOF_NoExt;
20744 break;
20745 }
20746 } else
20747 FlagSet |= PPC::MOF_NoExt;
20748
20749 // For integers, no extension is the same as zero extension.
20750 // We set the extension mode to zero extension so we don't have
20751 // to add separate entries in AddrModesMap for loads and stores.
20752 if (MemVT.isScalarInteger() && (FlagSet & PPC::MOF_NoExt)) {
20753 FlagSet |= PPC::MOF_ZExt;
20754 FlagSet &= ~PPC::MOF_NoExt;
20755 }
20756
20757 // If we don't have prefixed instructions, 34-bit constants should be
20758 // treated as PPC::MOF_NotAddNorCst so they can match D-Forms.
20759 bool IsNonP1034BitConst =
20760 ((PPC::MOF_RPlusSImm34 | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubtargetP10) &
20761 FlagSet) == PPC::MOF_RPlusSImm34;
20762 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::OR &&
20763 IsNonP1034BitConst)
20764 FlagSet |= PPC::MOF_NotAddNorCst;
20765
20766 return FlagSet;
20767}
20768
20769/// SelectForceXFormMode - Given the specified address, force it to be
20770/// represented as an indexed [r+r] operation (an XForm instruction).
20771PPC::AddrMode PPCTargetLowering::SelectForceXFormMode(SDValue N, SDValue &Disp,
20772 SDValue &Base,
20773 SelectionDAG &DAG) const {
20774
20775 PPC::AddrMode Mode = PPC::AM_XForm;
20776 int16_t ForceXFormImm = 0;
20777 if (provablyDisjointOr(DAG, N) &&
20778 !isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm)) {
20779 Disp = N.getOperand(i: 0);
20780 Base = N.getOperand(i: 1);
20781 return Mode;
20782 }
20783
20784 // If the address is the result of an add, we will utilize the fact that the
20785 // address calculation includes an implicit add. However, we can reduce
20786 // register pressure if we do not materialize a constant just for use as the
20787 // index register. We only get rid of the add if it is not an add of a
20788 // value and a 16-bit signed constant and both have a single use.
20789 if (N.getOpcode() == ISD::ADD &&
20790 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm) ||
20791 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
20792 Disp = N.getOperand(i: 0);
20793 Base = N.getOperand(i: 1);
20794 return Mode;
20795 }
20796
20797 // Otherwise, use R0 as the base register.
20798 Disp = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
20799 VT: N.getValueType());
20800 Base = N;
20801
20802 return Mode;
20803}
20804
20805bool PPCTargetLowering::splitValueIntoRegisterParts(
20806 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
20807 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
20808 EVT ValVT = Val.getValueType();
20809 // If we are splitting a scalar integer into f64 parts (i.e. so they
20810 // can be placed into VFRC registers), we need to zero extend and
20811 // bitcast the values. This will ensure the value is placed into a
20812 // VSR using direct moves or stack operations as needed.
20813 if (PartVT == MVT::f64 &&
20814 (ValVT == MVT::i32 || ValVT == MVT::i16 || ValVT == MVT::i8)) {
20815 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: Val);
20816 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Val);
20817 Parts[0] = Val;
20818 return true;
20819 }
20820 return false;
20821}
20822
20823SDValue PPCTargetLowering::lowerToLibCall(const char *LibCallName, SDValue Op,
20824 SelectionDAG &DAG) const {
20825 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20826 TargetLowering::CallLoweringInfo CLI(DAG);
20827 EVT RetVT = Op.getValueType();
20828 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
20829 SDValue Callee =
20830 DAG.getExternalSymbol(Sym: LibCallName, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
20831 bool SignExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: false);
20832 TargetLowering::ArgListTy Args;
20833 for (const SDValue &N : Op->op_values()) {
20834 EVT ArgVT = N.getValueType();
20835 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
20836 TargetLowering::ArgListEntry Entry(N, ArgTy);
20837 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: SignExtend);
20838 Entry.IsZExt = !Entry.IsSExt;
20839 Args.push_back(x: Entry);
20840 }
20841
20842 SDValue InChain = DAG.getEntryNode();
20843 SDValue TCChain = InChain;
20844 const Function &F = DAG.getMachineFunction().getFunction();
20845 bool isTailCall =
20846 TLI.isInTailCallPosition(DAG, Node: Op.getNode(), Chain&: TCChain) &&
20847 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
20848 if (isTailCall)
20849 InChain = TCChain;
20850 CLI.setDebugLoc(SDLoc(Op))
20851 .setChain(InChain)
20852 .setLibCallee(CC: CallingConv::C, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
20853 .setTailCall(isTailCall)
20854 .setSExtResult(SignExtend)
20855 .setZExtResult(!SignExtend)
20856 .setIsPostTypeLegalization(true);
20857 return TLI.LowerCallTo(CLI).first;
20858}
20859
20860SDValue PPCTargetLowering::lowerLibCallBasedOnType(
20861 const char *LibCallFloatName, const char *LibCallDoubleName, SDValue Op,
20862 SelectionDAG &DAG) const {
20863 if (Op.getValueType() == MVT::f32)
20864 return lowerToLibCall(LibCallName: LibCallFloatName, Op, DAG);
20865
20866 if (Op.getValueType() == MVT::f64)
20867 return lowerToLibCall(LibCallName: LibCallDoubleName, Op, DAG);
20868
20869 return SDValue();
20870}
20871
20872bool PPCTargetLowering::isLowringToMASSFiniteSafe(SDValue Op) const {
20873 SDNodeFlags Flags = Op.getNode()->getFlags();
20874 return isLowringToMASSSafe(Op) && Flags.hasNoSignedZeros() &&
20875 Flags.hasNoNaNs() && Flags.hasNoInfs();
20876}
20877
20878bool PPCTargetLowering::isLowringToMASSSafe(SDValue Op) const {
20879 return Op.getNode()->getFlags().hasApproximateFuncs();
20880}
20881
20882bool PPCTargetLowering::isScalarMASSConversionEnabled() const {
20883 return getTargetMachine().getOptLevel() == CodeGenOptLevel::Aggressive &&
20884 EnablePPCGenScalarMASSEntries;
20885}
20886
20887SDValue PPCTargetLowering::lowerLibCallBase(const char *LibCallDoubleName,
20888 const char *LibCallFloatName,
20889 const char *LibCallDoubleNameFinite,
20890 const char *LibCallFloatNameFinite,
20891 SDValue Op,
20892 SelectionDAG &DAG) const {
20893 if (!isScalarMASSConversionEnabled() || !isLowringToMASSSafe(Op))
20894 return SDValue();
20895
20896 if (!isLowringToMASSFiniteSafe(Op))
20897 return lowerLibCallBasedOnType(LibCallFloatName, LibCallDoubleName, Op,
20898 DAG);
20899
20900 return lowerLibCallBasedOnType(LibCallFloatName: LibCallFloatNameFinite,
20901 LibCallDoubleName: LibCallDoubleNameFinite, Op, DAG);
20902}
20903
20904SDValue PPCTargetLowering::lowerPow(SDValue Op, SelectionDAG &DAG) const {
20905 return lowerLibCallBase(LibCallDoubleName: "__xl_pow", LibCallFloatName: "__xl_powf", LibCallDoubleNameFinite: "__xl_pow_finite",
20906 LibCallFloatNameFinite: "__xl_powf_finite", Op, DAG);
20907}
20908
20909SDValue PPCTargetLowering::lowerSin(SDValue Op, SelectionDAG &DAG) const {
20910 return lowerLibCallBase(LibCallDoubleName: "__xl_sin", LibCallFloatName: "__xl_sinf", LibCallDoubleNameFinite: "__xl_sin_finite",
20911 LibCallFloatNameFinite: "__xl_sinf_finite", Op, DAG);
20912}
20913
20914SDValue PPCTargetLowering::lowerCos(SDValue Op, SelectionDAG &DAG) const {
20915 return lowerLibCallBase(LibCallDoubleName: "__xl_cos", LibCallFloatName: "__xl_cosf", LibCallDoubleNameFinite: "__xl_cos_finite",
20916 LibCallFloatNameFinite: "__xl_cosf_finite", Op, DAG);
20917}
20918
20919SDValue PPCTargetLowering::lowerLog(SDValue Op, SelectionDAG &DAG) const {
20920 return lowerLibCallBase(LibCallDoubleName: "__xl_log", LibCallFloatName: "__xl_logf", LibCallDoubleNameFinite: "__xl_log_finite",
20921 LibCallFloatNameFinite: "__xl_logf_finite", Op, DAG);
20922}
20923
20924SDValue PPCTargetLowering::lowerLog10(SDValue Op, SelectionDAG &DAG) const {
20925 return lowerLibCallBase(LibCallDoubleName: "__xl_log10", LibCallFloatName: "__xl_log10f", LibCallDoubleNameFinite: "__xl_log10_finite",
20926 LibCallFloatNameFinite: "__xl_log10f_finite", Op, DAG);
20927}
20928
20929SDValue PPCTargetLowering::lowerExp(SDValue Op, SelectionDAG &DAG) const {
20930 return lowerLibCallBase(LibCallDoubleName: "__xl_exp", LibCallFloatName: "__xl_expf", LibCallDoubleNameFinite: "__xl_exp_finite",
20931 LibCallFloatNameFinite: "__xl_expf_finite", Op, DAG);
20932}
20933
20934// If we happen to match to an aligned D-Form, check if the Frame Index is
20935// adequately aligned. If it is not, reset the mode to match to X-Form.
20936static void setXFormForUnalignedFI(SDValue N, unsigned Flags,
20937 PPC::AddrMode &Mode) {
20938 if (!isa<FrameIndexSDNode>(Val: N))
20939 return;
20940 if ((Mode == PPC::AM_DSForm && !(Flags & PPC::MOF_RPlusSImm16Mult4)) ||
20941 (Mode == PPC::AM_DQForm && !(Flags & PPC::MOF_RPlusSImm16Mult16)))
20942 Mode = PPC::AM_XForm;
20943}
20944
20945/// SelectOptimalAddrMode - Based on a node N and it's Parent (a MemSDNode),
20946/// compute the address flags of the node, get the optimal address mode based
20947/// on the flags, and set the Base and Disp based on the address mode.
20948PPC::AddrMode PPCTargetLowering::SelectOptimalAddrMode(const SDNode *Parent,
20949 SDValue N, SDValue &Disp,
20950 SDValue &Base,
20951 SelectionDAG &DAG,
20952 MaybeAlign Align) const {
20953 SDLoc DL(Parent);
20954
20955 // Compute the address flags.
20956 unsigned Flags = computeMOFlags(Parent, N, DAG);
20957
20958 // Get the optimal address mode based on the Flags.
20959 PPC::AddrMode Mode = getAddrModeForFlags(Flags);
20960
20961 // If the address mode is DS-Form or DQ-Form, check if the FI is aligned.
20962 // Select an X-Form load if it is not.
20963 setXFormForUnalignedFI(N, Flags, Mode);
20964
20965 // Set the mode to PC-Relative addressing mode if we have a valid PC-Rel node.
20966 if ((Mode == PPC::AM_XForm) && isPCRelNode(N)) {
20967 assert(Subtarget.isUsingPCRelativeCalls() &&
20968 "Must be using PC-Relative calls when a valid PC-Relative node is "
20969 "present!");
20970 Mode = PPC::AM_PCRel;
20971 }
20972
20973 // Set Base and Disp accordingly depending on the address mode.
20974 switch (Mode) {
20975 case PPC::AM_DForm:
20976 case PPC::AM_DSForm:
20977 case PPC::AM_DQForm: {
20978 // This is a register plus a 16-bit immediate. The base will be the
20979 // register and the displacement will be the immediate unless it
20980 // isn't sufficiently aligned.
20981 if (Flags & PPC::MOF_RPlusSImm16) {
20982 SDValue Op0 = N.getOperand(i: 0);
20983 SDValue Op1 = N.getOperand(i: 1);
20984 int16_t Imm = Op1->getAsZExtVal();
20985 if (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm)) {
20986 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: N.getValueType());
20987 Base = Op0;
20988 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op0)) {
20989 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
20990 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
20991 }
20992 break;
20993 }
20994 }
20995 // This is a register plus the @lo relocation. The base is the register
20996 // and the displacement is the global address.
20997 else if (Flags & PPC::MOF_RPlusLo) {
20998 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
20999 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
21000 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
21001 Disp.getOpcode() == ISD::TargetConstantPool ||
21002 Disp.getOpcode() == ISD::TargetJumpTable);
21003 Base = N.getOperand(i: 0);
21004 break;
21005 }
21006 // This is a constant address at most 32 bits. The base will be
21007 // zero or load-immediate-shifted and the displacement will be
21008 // the low 16 bits of the address.
21009 else if (Flags & PPC::MOF_AddrIsSImm32) {
21010 auto *CN = cast<ConstantSDNode>(Val&: N);
21011 EVT CNType = CN->getValueType(ResNo: 0);
21012 uint64_t CNImm = CN->getZExtValue();
21013 // If this address fits entirely in a 16-bit sext immediate field, codegen
21014 // this as "d, 0".
21015 int16_t Imm;
21016 if (isIntS16Immediate(N: CN, Imm) && (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm))) {
21017 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: CNType);
21018 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
21019 VT: CNType);
21020 break;
21021 }
21022 // Handle 32-bit sext immediate with LIS + Addr mode.
21023 if ((CNType == MVT::i32 || isInt<32>(x: CNImm)) &&
21024 (!Align || isAligned(Lhs: *Align, SizeInBytes: CNImm))) {
21025 int32_t Addr = (int32_t)CNImm;
21026 // Otherwise, break this down into LIS + Disp.
21027 Disp = DAG.getSignedTargetConstant(Val: (int16_t)Addr, DL, VT: MVT::i32);
21028 Base = DAG.getSignedTargetConstant(Val: (Addr - (int16_t)Addr) >> 16, DL,
21029 VT: MVT::i32);
21030 uint32_t LIS = CNType == MVT::i32 ? PPC::LIS : PPC::LIS8;
21031 Base = SDValue(DAG.getMachineNode(Opcode: LIS, dl: DL, VT: CNType, Op1: Base), 0);
21032 break;
21033 }
21034 }
21035 // Otherwise, the PPC:MOF_NotAdd flag is set. Load/Store is Non-foldable.
21036 Disp = DAG.getTargetConstant(Val: 0, DL, VT: getPointerTy(DL: DAG.getDataLayout()));
21037 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
21038 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
21039 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
21040 } else
21041 Base = N;
21042 break;
21043 }
21044 case PPC::AM_PrefixDForm: {
21045 int64_t Imm34 = 0;
21046 unsigned Opcode = N.getOpcode();
21047 if (((Opcode == ISD::ADD) || (Opcode == ISD::OR)) &&
21048 (isIntS34Immediate(Op: N.getOperand(i: 1), Imm&: Imm34))) {
21049 // N is an Add/OR Node, and it's operand is a 34-bit signed immediate.
21050 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
21051 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
21052 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
21053 else
21054 Base = N.getOperand(i: 0);
21055 } else if (isIntS34Immediate(Op: N, Imm&: Imm34)) {
21056 // The address is a 34-bit signed immediate.
21057 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
21058 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
21059 }
21060 break;
21061 }
21062 case PPC::AM_PCRel: {
21063 // When selecting PC-Relative instructions, "Base" is not utilized as
21064 // we select the address as [PC+imm].
21065 Disp = N;
21066 break;
21067 }
21068 case PPC::AM_None:
21069 break;
21070 default: { // By default, X-Form is always available to be selected.
21071 // When a frame index is not aligned, we also match by XForm.
21072 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N);
21073 Base = FI ? N : N.getOperand(i: 1);
21074 Disp = FI ? DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
21075 VT: N.getValueType())
21076 : N.getOperand(i: 0);
21077 break;
21078 }
21079 }
21080 return Mode;
21081}
21082
21083CCAssignFn *PPCTargetLowering::ccAssignFnForCall(CallingConv::ID CC,
21084 bool Return,
21085 bool IsVarArg) const {
21086 switch (CC) {
21087 case CallingConv::Cold:
21088 return (Return ? RetCC_PPC_Cold : CC_PPC64_ELF);
21089 default:
21090 return CC_PPC64_ELF;
21091 }
21092}
21093
21094bool PPCTargetLowering::shouldInlineQuadwordAtomics() const {
21095 return Subtarget.isPPC64() && Subtarget.hasQuadwordAtomics();
21096}
21097
21098TargetLowering::AtomicExpansionKind
21099PPCTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
21100 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21101 if (shouldInlineQuadwordAtomics() && Size == 128)
21102 return AtomicExpansionKind::MaskedIntrinsic;
21103
21104 switch (AI->getOperation()) {
21105 case AtomicRMWInst::UIncWrap:
21106 case AtomicRMWInst::UDecWrap:
21107 case AtomicRMWInst::USubCond:
21108 case AtomicRMWInst::USubSat:
21109 return AtomicExpansionKind::CmpXChg;
21110 default:
21111 return TargetLowering::shouldExpandAtomicRMWInIR(RMW: AI);
21112 }
21113
21114 llvm_unreachable("unreachable atomicrmw operation");
21115}
21116
21117TargetLowering::AtomicExpansionKind
21118PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(
21119 const AtomicCmpXchgInst *AI) const {
21120 unsigned Size = AI->getNewValOperand()->getType()->getPrimitiveSizeInBits();
21121 if (shouldInlineQuadwordAtomics() && Size == 128)
21122 return AtomicExpansionKind::MaskedIntrinsic;
21123 return AtomicExpansionKind::LLSC;
21124}
21125
21126static Intrinsic::ID
21127getIntrinsicForAtomicRMWBinOp128(AtomicRMWInst::BinOp BinOp) {
21128 switch (BinOp) {
21129 default:
21130 llvm_unreachable("Unexpected AtomicRMW BinOp");
21131 case AtomicRMWInst::Xchg:
21132 return Intrinsic::ppc_atomicrmw_xchg_i128;
21133 case AtomicRMWInst::Add:
21134 return Intrinsic::ppc_atomicrmw_add_i128;
21135 case AtomicRMWInst::Sub:
21136 return Intrinsic::ppc_atomicrmw_sub_i128;
21137 case AtomicRMWInst::And:
21138 return Intrinsic::ppc_atomicrmw_and_i128;
21139 case AtomicRMWInst::Or:
21140 return Intrinsic::ppc_atomicrmw_or_i128;
21141 case AtomicRMWInst::Xor:
21142 return Intrinsic::ppc_atomicrmw_xor_i128;
21143 case AtomicRMWInst::Nand:
21144 return Intrinsic::ppc_atomicrmw_nand_i128;
21145 }
21146}
21147
21148Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic(
21149 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
21150 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
21151 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21152 Module *M = Builder.getModule();
21153 Type *ValTy = Incr->getType();
21154 assert(ValTy->getPrimitiveSizeInBits() == 128);
21155 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21156 Value *IncrLo = Builder.CreateTrunc(V: Incr, DestTy: Int64Ty, Name: "incr_lo");
21157 Value *IncrHi =
21158 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: Incr, RHS: 64), DestTy: Int64Ty, Name: "incr_hi");
21159 Value *LoHi = Builder.CreateIntrinsic(
21160 ID: getIntrinsicForAtomicRMWBinOp128(BinOp: AI->getOperation()), OverloadTypes: {},
21161 Args: {AlignedAddr, IncrLo, IncrHi});
21162 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21163 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21164 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21165 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21166 return Builder.CreateOr(
21167 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21168}
21169
21170Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
21171 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
21172 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
21173 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21174 Module *M = Builder.getModule();
21175 Type *ValTy = CmpVal->getType();
21176 assert(ValTy->getPrimitiveSizeInBits() == 128);
21177 Function *IntCmpXchg =
21178 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::ppc_cmpxchg_i128);
21179 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21180 Value *CmpLo = Builder.CreateTrunc(V: CmpVal, DestTy: Int64Ty, Name: "cmp_lo");
21181 Value *CmpHi =
21182 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: CmpVal, RHS: 64), DestTy: Int64Ty, Name: "cmp_hi");
21183 Value *NewLo = Builder.CreateTrunc(V: NewVal, DestTy: Int64Ty, Name: "new_lo");
21184 Value *NewHi =
21185 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: NewVal, RHS: 64), DestTy: Int64Ty, Name: "new_hi");
21186 emitLeadingFence(Builder, Inst: CI, Ord);
21187 Value *LoHi =
21188 Builder.CreateCall(Callee: IntCmpXchg, Args: {AlignedAddr, CmpLo, CmpHi, NewLo, NewHi});
21189 emitTrailingFence(Builder, Inst: CI, Ord);
21190 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21191 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21192 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21193 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21194 return Builder.CreateOr(
21195 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21196}
21197
21198bool PPCTargetLowering::hasMultipleConditionRegisters(EVT VT) const {
21199 return Subtarget.useCRBits();
21200}
21201
21202/// Shuffle masks for vectors of bits are not legal as such vectors are
21203/// reserved for MMA/DM.
21204bool PPCTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const {
21205 if (VT.getScalarType() == MVT::i1)
21206 return false;
21207 return TargetLowering::isShuffleMaskLegal(Mask, VT);
21208}
21209
21210// Optimize the following patterns using vbpermq/vbpermd:
21211// i16 = bitcast(v16i1 truncate(v16i8))
21212// i8 = bitcast(v8i1 truncate(v8i16))
21213// i8 = bitcast(v8i1 truncate(v8i8))
21214SDValue PPCTargetLowering::DAGCombineBitcast(SDNode *N,
21215 DAGCombinerInfo &DCI) const {
21216 SDValue Op0 = N->getOperand(Num: 0);
21217 if (Op0.getOpcode() != ISD::TRUNCATE)
21218 return SDValue();
21219 SDValue Src = Op0.getOperand(i: 0);
21220 EVT ResVT = N->getValueType(ResNo: 0);
21221 EVT TruncResVT = Op0.getValueType();
21222 EVT SrcVT = Src.getValueType();
21223 SDLoc dl(N);
21224 SelectionDAG &DAG = DCI.DAG;
21225 bool IsLittleEndian = Subtarget.isLittleEndian();
21226
21227 if (ResVT != MVT::i16 && ResVT != MVT::i8)
21228 return SDValue();
21229 SDValue VBPerm =
21230 GenerateVBPERM(DAG, dl, Src, SrcVT, ResVT: TruncResVT, IsLE: IsLittleEndian);
21231 if (!VBPerm)
21232 return SDValue();
21233 SDValue ForExtract = DAG.getBitcast(VT: MVT::v4i32, V: VBPerm);
21234 SDValue Extracted =
21235 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: ForExtract,
21236 N2: DAG.getIntPtrConstant(Val: IsLittleEndian ? 2 : 1, DL: dl));
21237 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT, Operand: Extracted);
21238}
21239
21240SDValue PPCTargetLowering::GenerateVBPERM(SelectionDAG &DAG, SDLoc dl,
21241 SDValue Src, EVT SrcVT, EVT ResVT,
21242 bool IsLE) const {
21243 bool IsV16i8 = (ResVT == MVT::v16i1 && SrcVT == MVT::v16i8);
21244 bool IsV8i16 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i16);
21245 bool IsV8i8 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i8);
21246
21247 if (!IsV16i8 && !IsV8i16 && !IsV8i8)
21248 return SDValue();
21249
21250 if (IsV8i8) {
21251 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: MVT::v16i8,
21252 N1: DAG.getUNDEF(VT: MVT::v16i8), N2: Src,
21253 N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
21254 }
21255 SmallVector<int, 16> BitIndices(16, 128);
21256 unsigned NumElts = SrcVT.getVectorNumElements();
21257 unsigned EltSize = SrcVT.getScalarType().getSizeInBits();
21258 for (int Idx = 0, End = SrcVT.getVectorNumElements(); Idx < End; Idx++) {
21259 BitIndices[Idx] = EltSize * (NumElts - Idx) - 1;
21260 if (IsV8i8 && IsLE)
21261 BitIndices[Idx] += 64;
21262 }
21263 if (!IsLE)
21264 std::reverse(first: BitIndices.begin(), last: BitIndices.end());
21265 SmallVector<SDValue, 16> BVOps;
21266 for (auto Idx : BitIndices)
21267 BVOps.push_back(Elt: DAG.getConstant(Val: Idx, DL: dl, VT: MVT::i8));
21268 SDValue VRB = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: BVOps);
21269 return DAG.getNode(
21270 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v16i8,
21271 N1: DAG.getConstant(Val: Intrinsic::ppc_altivec_vbpermq, DL: dl, VT: MVT::i32),
21272 N2: DAG.getBitcast(VT: MVT::v16i8, V: Src), N3: VRB);
21273}
21274
21275// For Power8/9, optimize vec splats of small FP values that can be
21276// represented as integers. Use vspltisw + xvcvsxwdp/xvcvsxwsp instead of
21277// loading from constant pool.
21278SDValue PPCTargetLowering::LowerVecSplatSmallFP(SDValue Op, SelectionDAG &DAG,
21279 bool BVNIsConstantSplat,
21280 unsigned SplatBitSize) const {
21281
21282 if (!BVNIsConstantSplat || !Subtarget.hasVSX() || !Subtarget.hasP8Vector() ||
21283 Subtarget.hasP10Vector())
21284 return SDValue();
21285
21286 EVT VT = Op->getValueType(ResNo: 0);
21287 if (!((SplatBitSize == 64 && VT == MVT::v2f64) ||
21288 (SplatBitSize == 32 && VT == MVT::v4f32)))
21289 return SDValue();
21290
21291 auto *CN = dyn_cast<ConstantFPSDNode>(Val: Op.getOperand(i: 0));
21292 if (!CN)
21293 return SDValue();
21294
21295 APFloat APFloatVal = CN->getValueAPF();
21296 bool IsExact;
21297 APSInt IntResult(16, false);
21298 APFloatVal.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
21299
21300 if (!(IsExact && IntResult <= 15 && IntResult >= -16 && !APFloatVal.isZero()))
21301 return SDValue();
21302
21303 int64_t IntVal = IntResult.getSExtValue();
21304
21305 SDLoc dl(Op);
21306 SDValue IntSplat = getCanonicalConstSplat(Val: IntVal, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
21307
21308 if (SplatBitSize == 64)
21309 return DAG.getNode(
21310 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v2f64,
21311 N1: DAG.getConstant(Val: Intrinsic::ppc_vsx_xvcvsxwdp, DL: dl, VT: MVT::i32), N2: IntSplat);
21312
21313 return DAG.getNode(Opcode: PPCISD::XVCVSXWSP, DL: dl, VT: MVT::v4f32, Operand: IntSplat);
21314}
21315