1//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the PPCISelLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCISelLowering.h"
14#include "MCTargetDesc/PPCMCTargetDesc.h"
15#include "MCTargetDesc/PPCPredicates.h"
16#include "PPC.h"
17#include "PPCCallingConv.h"
18#include "PPCFrameLowering.h"
19#include "PPCInstrInfo.h"
20#include "PPCMachineFunctionInfo.h"
21#include "PPCPerfectShuffle.h"
22#include "PPCRegisterInfo.h"
23#include "PPCSelectionDAGInfo.h"
24#include "PPCSubtarget.h"
25#include "PPCTargetMachine.h"
26#include "llvm/ADT/APFloat.h"
27#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/APSInt.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SmallPtrSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/Statistic.h"
35#include "llvm/ADT/StringRef.h"
36#include "llvm/CodeGen/CallingConvLower.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/LivePhysRegs.h"
39#include "llvm/CodeGen/MachineBasicBlock.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachineJumpTableInfo.h"
45#include "llvm/CodeGen/MachineLoopInfo.h"
46#include "llvm/CodeGen/MachineMemOperand.h"
47#include "llvm/CodeGen/MachineModuleInfo.h"
48#include "llvm/CodeGen/MachineOperand.h"
49#include "llvm/CodeGen/MachineRegisterInfo.h"
50#include "llvm/CodeGen/SelectionDAG.h"
51#include "llvm/CodeGen/SelectionDAGNodes.h"
52#include "llvm/CodeGen/TargetInstrInfo.h"
53#include "llvm/CodeGen/TargetLowering.h"
54#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/ValueTypes.h"
57#include "llvm/CodeGenTypes/MachineValueType.h"
58#include "llvm/IR/CallingConv.h"
59#include "llvm/IR/Constant.h"
60#include "llvm/IR/Constants.h"
61#include "llvm/IR/DataLayout.h"
62#include "llvm/IR/DebugLoc.h"
63#include "llvm/IR/DerivedTypes.h"
64#include "llvm/IR/Function.h"
65#include "llvm/IR/GlobalValue.h"
66#include "llvm/IR/IRBuilder.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/Intrinsics.h"
69#include "llvm/IR/IntrinsicsPowerPC.h"
70#include "llvm/IR/Module.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/Use.h"
73#include "llvm/IR/Value.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/MC/MCExpr.h"
76#include "llvm/MC/MCSectionXCOFF.h"
77#include "llvm/MC/MCSymbolXCOFF.h"
78#include "llvm/Support/AtomicOrdering.h"
79#include "llvm/Support/BranchProbability.h"
80#include "llvm/Support/Casting.h"
81#include "llvm/Support/CodeGen.h"
82#include "llvm/Support/CommandLine.h"
83#include "llvm/Support/Compiler.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/Format.h"
87#include "llvm/Support/KnownBits.h"
88#include "llvm/Support/MathExtras.h"
89#include "llvm/Support/raw_ostream.h"
90#include "llvm/Target/TargetMachine.h"
91#include "llvm/Target/TargetOptions.h"
92#include <algorithm>
93#include <cassert>
94#include <cstdint>
95#include <iterator>
96#include <list>
97#include <optional>
98#include <utility>
99#include <vector>
100
101using namespace llvm;
102
103#define DEBUG_TYPE "ppc-lowering"
104
105extern cl::opt<bool> EnablePPCGenScalarMASSEntries;
106
107static cl::opt<bool> DisableP10StoreForward(
108 "disable-p10-store-forward",
109 cl::desc("disable P10 store forward-friendly conversion"), cl::Hidden,
110 cl::init(Val: false));
111
112static cl::opt<bool> DisablePPCPreinc("disable-ppc-preinc",
113cl::desc("disable preincrement load/store generation on PPC"), cl::Hidden);
114
115static cl::opt<bool> DisableILPPref("disable-ppc-ilp-pref",
116cl::desc("disable setting the node scheduling preference to ILP on PPC"), cl::Hidden);
117
118static cl::opt<bool> DisablePPCUnaligned("disable-ppc-unaligned",
119cl::desc("disable unaligned load/store generation on PPC"), cl::Hidden);
120
121static cl::opt<bool> DisableSCO("disable-ppc-sco",
122cl::desc("disable sibling call optimization on ppc"), cl::Hidden);
123
124static cl::opt<bool> DisableInnermostLoopAlign32("disable-ppc-innermost-loop-align32",
125cl::desc("don't always align innermost loop to 32 bytes on ppc"), cl::Hidden);
126
127static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables",
128cl::desc("use absolute jump tables on ppc"), cl::Hidden);
129
130static cl::opt<bool>
131 DisablePerfectShuffle("ppc-disable-perfect-shuffle",
132 cl::desc("disable vector permute decomposition"),
133 cl::init(Val: true), cl::Hidden);
134
135cl::opt<bool> DisableAutoPairedVecSt(
136 "disable-auto-paired-vec-st",
137 cl::desc("disable automatically generated 32byte paired vector stores"),
138 cl::init(Val: true), cl::Hidden);
139
140static cl::opt<unsigned> PPCMinimumJumpTableEntries(
141 "ppc-min-jump-table-entries", cl::init(Val: 64), cl::Hidden,
142 cl::desc("Set minimum number of entries to use a jump table on PPC"));
143
144static cl::opt<unsigned> PPCMinimumBitTestCmps(
145 "ppc-min-bit-test-cmps", cl::init(Val: 3), cl::Hidden,
146 cl::desc("Set minimum of largest number of comparisons to use bit test for "
147 "switch on PPC."));
148
149static cl::opt<unsigned> PPCGatherAllAliasesMaxDepth(
150 "ppc-gather-alias-max-depth", cl::init(Val: 18), cl::Hidden,
151 cl::desc("max depth when checking alias info in GatherAllAliases()"));
152
153static cl::opt<unsigned> PPCAIXTLSModelOptUseIEForLDLimit(
154 "ppc-aix-shared-lib-tls-model-opt-limit", cl::init(Val: 1), cl::Hidden,
155 cl::desc("Set inclusive limit count of TLS local-dynamic access(es) in a "
156 "function to use initial-exec"));
157
158STATISTIC(NumTailCalls, "Number of tail calls");
159STATISTIC(NumSiblingCalls, "Number of sibling calls");
160STATISTIC(ShufflesHandledWithVPERM,
161 "Number of shuffles lowered to a VPERM or XXPERM");
162STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed");
163
164static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
165
166static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
167
168static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
169 unsigned OpIdx, bool IsByte,
170 const PPCInstrInfo *TII);
171
172// A faster local-[exec|dynamic] TLS access sequence (enabled with the
173// -maix-small-local-[exec|dynamic]-tls option) can be produced for TLS
174// variables; consistent with the IBM XL compiler, we apply a max size of
175// slightly under 32KB.
176constexpr uint64_t AIXSmallTlsPolicySizeLimit = 32751;
177
178// FIXME: Remove this once the bug has been fixed!
179extern cl::opt<bool> ANDIGlueBug;
180
181PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
182 const PPCSubtarget &STI)
183 : TargetLowering(TM, STI), Subtarget(STI) {
184 // Initialize map that relates the PPC addressing modes to the computed flags
185 // of a load/store instruction. The map is used to determine the optimal
186 // addressing mode when selecting load and stores.
187 initializeAddrModeMap();
188 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
189 // arguments are at least 4/8 bytes aligned.
190 bool isPPC64 = Subtarget.isPPC64();
191 setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4));
192 const MVT RegVT = Subtarget.getScalarIntVT();
193
194 // Set up the register classes.
195 addRegisterClass(VT: MVT::i32, RC: &PPC::GPRCRegClass);
196 if (!useSoftFloat()) {
197 if (hasSPE()) {
198 addRegisterClass(VT: MVT::f32, RC: &PPC::GPRCRegClass);
199 // EFPU2 APU only supports f32
200 if (!Subtarget.hasEFPU2())
201 addRegisterClass(VT: MVT::f64, RC: &PPC::SPERCRegClass);
202 } else {
203 addRegisterClass(VT: MVT::f32, RC: &PPC::F4RCRegClass);
204 addRegisterClass(VT: MVT::f64, RC: &PPC::F8RCRegClass);
205 }
206 }
207
208 setOperationAction(Op: ISD::UADDO, VT: RegVT, Action: Custom);
209 setOperationAction(Op: ISD::USUBO, VT: RegVT, Action: Custom);
210
211 // PowerPC uses addo_carry,subo_carry to propagate carry.
212 setOperationAction(Op: ISD::UADDO_CARRY, VT: RegVT, Action: Custom);
213 setOperationAction(Op: ISD::USUBO_CARRY, VT: RegVT, Action: Custom);
214
215 // On P10, the default lowering generates better code using the
216 // setbc instruction.
217 if (!Subtarget.hasP10Vector()) {
218 setOperationAction(Op: ISD::SSUBO, VT: MVT::i32, Action: Custom);
219 setOperationAction(Op: ISD::SADDO, VT: MVT::i32, Action: Custom);
220 if (isPPC64) {
221 setOperationAction(Op: ISD::SSUBO, VT: MVT::i64, Action: Custom);
222 setOperationAction(Op: ISD::SADDO, VT: MVT::i64, Action: Custom);
223 }
224 }
225
226 // Match BITREVERSE to customized fast code sequence in the td file.
227 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
228 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
229
230 // Sub-word ATOMIC_CMP_SWAP need to ensure that the input is zero-extended.
231 setOperationAction(Op: ISD::ATOMIC_CMP_SWAP, VT: MVT::i32, Action: Custom);
232
233 // Custom lower inline assembly to check for special registers.
234 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
235 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
236
237 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD.
238 for (MVT VT : MVT::integer_valuetypes()) {
239 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
240 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i8, Action: Expand);
241 }
242
243 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
244 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f128, Action: Expand);
245
246 if (Subtarget.isISA3_0()) {
247 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Legal);
248 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
249 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
250 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
251 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
252 } else {
253 // No extending loads from f16 or HW conversions back and forth.
254 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
255 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f128, Action: Expand);
256 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
257 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
258 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
259 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
260 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
261 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
262 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
263 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
264 }
265
266 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
267
268 // PowerPC has pre-inc load and store's.
269 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
270 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
271 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
272 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
273 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
274 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
275 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
276 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
277 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
278 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
279 if (!Subtarget.hasSPE()) {
280 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
281 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
282 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
283 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
284 }
285
286 if (Subtarget.useCRBits()) {
287 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
288
289 if (isPPC64 || Subtarget.hasFPCVT()) {
290 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Promote);
291 AddPromotedToType(Opc: ISD::STRICT_SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
292 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Promote);
293 AddPromotedToType(Opc: ISD::STRICT_UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
294
295 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
296 AddPromotedToType(Opc: ISD::SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
297 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
298 AddPromotedToType(Opc: ISD::UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
299
300 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i1, Action: Promote);
301 AddPromotedToType(Opc: ISD::STRICT_FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
302 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i1, Action: Promote);
303 AddPromotedToType(Opc: ISD::STRICT_FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
304
305 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
306 AddPromotedToType(Opc: ISD::FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
307 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
308 AddPromotedToType(Opc: ISD::FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
309 } else {
310 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Custom);
311 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Custom);
312 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Custom);
313 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Custom);
314 }
315
316 // PowerPC does not support direct load/store of condition registers.
317 setOperationAction(Op: ISD::LOAD, VT: MVT::i1, Action: Custom);
318 setOperationAction(Op: ISD::STORE, VT: MVT::i1, Action: Custom);
319
320 // FIXME: Remove this once the ANDI glue bug is fixed:
321 if (ANDIGlueBug)
322 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::i1, Action: Custom);
323
324 for (MVT VT : MVT::integer_valuetypes()) {
325 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
326 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
327 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
328 }
329
330 addRegisterClass(VT: MVT::i1, RC: &PPC::CRBITRCRegClass);
331 }
332
333 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
334 // PPC (the libcall is not available).
335 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
336 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
337 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
338 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
339
340 // We do not currently implement these libm ops for PowerPC.
341 setOperationAction(Op: ISD::FFLOOR, VT: MVT::ppcf128, Action: Expand);
342 setOperationAction(Op: ISD::FCEIL, VT: MVT::ppcf128, Action: Expand);
343 setOperationAction(Op: ISD::FTRUNC, VT: MVT::ppcf128, Action: Expand);
344 setOperationAction(Op: ISD::FRINT, VT: MVT::ppcf128, Action: Expand);
345 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::ppcf128, Action: Expand);
346 setOperationAction(Op: ISD::FREM, VT: MVT::ppcf128, Action: LibCall);
347
348 // PowerPC has no SREM/UREM instructions unless we are on P9
349 // On P9 we may use a hardware instruction to compute the remainder.
350 // When the result of both the remainder and the division is required it is
351 // more efficient to compute the remainder from the result of the division
352 // rather than use the remainder instruction. The instructions are legalized
353 // directly because the DivRemPairsPass performs the transformation at the IR
354 // level.
355 if (Subtarget.isISA3_0()) {
356 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Legal);
357 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Legal);
358 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Legal);
359 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Legal);
360 } else {
361 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Expand);
362 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Expand);
363 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Expand);
364 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Expand);
365 }
366
367 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM.
368 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i32, Action: Expand);
369 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i32, Action: Expand);
370 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i64, Action: Expand);
371 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i64, Action: Expand);
372 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Expand);
373 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Expand);
374 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i64, Action: Expand);
375 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i64, Action: Expand);
376
377 // Handle constrained floating-point operations of scalar.
378 // TODO: Handle SPE specific operation.
379 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f32, Action: Legal);
380 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f32, Action: Legal);
381 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f32, Action: Legal);
382 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f32, Action: Legal);
383 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
384
385 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f64, Action: Legal);
386 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f64, Action: Legal);
387 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f64, Action: Legal);
388 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f64, Action: Legal);
389
390 if (!Subtarget.hasSPE()) {
391 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f32, Action: Legal);
392 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f64, Action: Legal);
393 }
394
395 if (Subtarget.hasVSX()) {
396 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f32, Action: Legal);
397 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f64, Action: Legal);
398 }
399
400 if (Subtarget.hasFSQRT()) {
401 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f32, Action: Legal);
402 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f64, Action: Legal);
403 }
404
405 if (Subtarget.hasFPRND()) {
406 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f32, Action: Legal);
407 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f32, Action: Legal);
408 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f32, Action: Legal);
409 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f32, Action: Legal);
410
411 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f64, Action: Legal);
412 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f64, Action: Legal);
413 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f64, Action: Legal);
414 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f64, Action: Legal);
415 }
416
417 // We don't support sin/cos/sqrt/fmod/pow
418 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Expand);
419 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Expand);
420 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
421 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
422 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Expand);
423 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Expand);
424 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Expand);
425 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
426 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
427 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Expand);
428
429 // MASS transformation for LLVM intrinsics with replicating fast-math flag
430 // to be consistent to PPCGenScalarMASSEntries pass
431 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive) {
432 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Custom);
433 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Custom);
434 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Custom);
435 setOperationAction(Op: ISD::FLOG, VT: MVT::f64, Action: Custom);
436 setOperationAction(Op: ISD::FLOG10, VT: MVT::f64, Action: Custom);
437 setOperationAction(Op: ISD::FEXP, VT: MVT::f64, Action: Custom);
438 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Custom);
439 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Custom);
440 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Custom);
441 setOperationAction(Op: ISD::FLOG, VT: MVT::f32, Action: Custom);
442 setOperationAction(Op: ISD::FLOG10, VT: MVT::f32, Action: Custom);
443 setOperationAction(Op: ISD::FEXP, VT: MVT::f32, Action: Custom);
444 }
445
446 if (Subtarget.hasSPE()) {
447 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Expand);
448 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Expand);
449 } else {
450 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Legal);
451 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Legal);
452 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
453 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
454 }
455
456 if (Subtarget.hasSPE())
457 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
458
459 // If we're enabling GP optimizations, use hardware square root
460 if (!Subtarget.hasFSQRT() && !(Subtarget.hasFRSQRTE() && Subtarget.hasFRE()))
461 setOperationAction(Op: ISD::FSQRT, VT: MVT::f64, Action: Expand);
462
463 if (!Subtarget.hasFSQRT() &&
464 !(Subtarget.hasFRSQRTES() && Subtarget.hasFRES()))
465 setOperationAction(Op: ISD::FSQRT, VT: MVT::f32, Action: Expand);
466
467 if (Subtarget.hasFCPSGN()) {
468 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Legal);
469 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Legal);
470 } else {
471 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Expand);
472 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Expand);
473 }
474
475 if (Subtarget.hasFPRND()) {
476 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
477 setOperationAction(Op: ISD::FCEIL, VT: MVT::f64, Action: Legal);
478 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f64, Action: Legal);
479 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
480
481 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f32, Action: Legal);
482 setOperationAction(Op: ISD::FCEIL, VT: MVT::f32, Action: Legal);
483 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f32, Action: Legal);
484 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
485 }
486
487 // Prior to P10, PowerPC does not have BSWAP, but we can use vector BSWAP
488 // instruction xxbrd to speed up scalar BSWAP64.
489 if (Subtarget.isISA3_1()) {
490 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
491 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
492 } else {
493 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Expand);
494 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64,
495 Action: ((Subtarget.hasP8Vector()) && isPPC64) ? Custom
496 : Expand);
497 }
498
499 // CTPOP or CTTZ were introduced in P8/P9 respectively
500 if (Subtarget.isISA3_0()) {
501 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Legal);
502 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Legal);
503 } else {
504 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Expand);
505 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Expand);
506 }
507
508 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
509 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Legal);
510 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Legal);
511 } else {
512 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Expand);
513 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Expand);
514 }
515
516 // PowerPC does not have ROTR
517 setOperationAction(Op: ISD::ROTR, VT: MVT::i32 , Action: Expand);
518 setOperationAction(Op: ISD::ROTR, VT: MVT::i64 , Action: Expand);
519
520 if (!Subtarget.useCRBits()) {
521 // PowerPC does not have Select
522 setOperationAction(Op: ISD::SELECT, VT: MVT::i32, Action: Expand);
523 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Expand);
524 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Expand);
525 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Expand);
526 }
527
528 // PowerPC wants to turn select_cc of FP into fsel when possible.
529 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Custom);
530 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Custom);
531
532 // PowerPC wants to optimize integer setcc a bit
533 if (!Subtarget.useCRBits())
534 setOperationAction(Op: ISD::SETCC, VT: MVT::i32, Action: Custom);
535
536 if (Subtarget.hasFPU()) {
537 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
538 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
539 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Legal);
540
541 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
542 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
543 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Legal);
544 }
545
546 // PowerPC does not have BRCOND which requires SetCC
547 if (!Subtarget.useCRBits())
548 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
549
550 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
551
552 if (Subtarget.hasSPE()) {
553 // SPE has built-in conversions
554 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Legal);
555 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Legal);
556 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Legal);
557 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Legal);
558 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Legal);
559 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Legal);
560
561 // SPE supports signaling compare of f32/f64.
562 // But it doesn't comply IEEE-754 rules for comparing
563 // special values like NaNs, Infs.
564 setOperationAction(Op: ISD::SETCC, VT: MVT::f32, Action: Custom);
565 setOperationAction(Op: ISD::SETCC, VT: MVT::f64, Action: Custom);
566 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Custom);
567 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Custom);
568 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Custom);
569 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Custom);
570 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Custom);
571 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Custom);
572 } else {
573 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores.
574 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
575 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
576
577 // PowerPC does not have [U|S]INT_TO_FP
578 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Expand);
579 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Expand);
580 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Expand);
581 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Expand);
582 }
583
584 if (Subtarget.hasDirectMove() && isPPC64) {
585 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Legal);
586 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Legal);
587 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Legal);
588 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Legal);
589
590 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f64, Action: Custom);
591 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f32, Action: Custom);
592 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f64, Action: Custom);
593 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f32, Action: Custom);
594 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f64, Action: Custom);
595 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f32, Action: Custom);
596 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f64, Action: Custom);
597 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f32, Action: Custom);
598 } else {
599 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Expand);
600 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Expand);
601 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Expand);
602 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Expand);
603 }
604
605 // We cannot sextinreg(i1). Expand to shifts.
606 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
607
608 // Custom handling for PowerPC ucmp and scmp instructions
609 setOperationAction(Op: ISD::UCMP, VT: MVT::i32, Action: Custom);
610 setOperationAction(Op: ISD::UCMP, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
611 setOperationAction(Op: ISD::ABDU, VT: MVT::i32, Action: Custom);
612 setOperationAction(Op: ISD::ABDU, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
613 setOperationAction(Op: ISD::SCMP, VT: MVT::i32, Action: Custom);
614 setOperationAction(Op: ISD::SCMP, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
615
616 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
617 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
618 // support continuation, user-level threading, and etc.. As a result, no
619 // other SjLj exception interfaces are implemented and please don't build
620 // your own exception handling based on them.
621 // LLVM/Clang supports zero-cost DWARF exception handling.
622 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
623 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
624
625 // We want to legalize GlobalAddress and ConstantPool nodes into the
626 // appropriate instructions to materialize the address.
627 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
628 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
629 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
630 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
631 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
632 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i64, Action: Custom);
633 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i64, Action: Custom);
634 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i64, Action: Custom);
635 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i64, Action: Custom);
636 setOperationAction(Op: ISD::JumpTable, VT: MVT::i64, Action: Custom);
637
638 // TRAP is legal.
639 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
640
641 // TRAMPOLINE is custom lowered.
642 setOperationAction(Op: ISD::INIT_TRAMPOLINE, VT: MVT::Other, Action: Custom);
643 setOperationAction(Op: ISD::ADJUST_TRAMPOLINE, VT: MVT::Other, Action: Custom);
644
645 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
646 setOperationAction(Op: ISD::VASTART , VT: MVT::Other, Action: Custom);
647
648 if (Subtarget.is64BitELFABI()) {
649 // VAARG always uses double-word chunks, so promote anything smaller.
650 setOperationAction(Op: ISD::VAARG, VT: MVT::i1, Action: Promote);
651 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i1, DestVT: MVT::i64);
652 setOperationAction(Op: ISD::VAARG, VT: MVT::i8, Action: Promote);
653 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i8, DestVT: MVT::i64);
654 setOperationAction(Op: ISD::VAARG, VT: MVT::i16, Action: Promote);
655 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i16, DestVT: MVT::i64);
656 setOperationAction(Op: ISD::VAARG, VT: MVT::i32, Action: Promote);
657 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i32, DestVT: MVT::i64);
658 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
659 } else if (Subtarget.is32BitELFABI()) {
660 // VAARG is custom lowered with the 32-bit SVR4 ABI.
661 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
662 setOperationAction(Op: ISD::VAARG, VT: MVT::i64, Action: Custom);
663 } else
664 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
665
666 // VACOPY is custom lowered with the 32-bit SVR4 ABI.
667 if (Subtarget.is32BitELFABI())
668 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Custom);
669 else
670 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Expand);
671
672 // Use the default implementation.
673 setOperationAction(Op: ISD::VAEND , VT: MVT::Other, Action: Expand);
674 setOperationAction(Op: ISD::STACKSAVE , VT: MVT::Other, Action: Expand);
675 setOperationAction(Op: ISD::STACKRESTORE , VT: MVT::Other, Action: Custom);
676 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32 , Action: Custom);
677 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i64 , Action: Custom);
678 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i32, Action: Custom);
679 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i64, Action: Custom);
680 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
681 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i64, Action: Custom);
682
683 if (Subtarget.isISA3_0() && isPPC64) {
684 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v16i1, Action: Custom);
685 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v8i1, Action: Custom);
686 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v4i1, Action: Custom);
687 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v2i1, Action: Custom);
688 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v16i1, Action: Custom);
689 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v8i1, Action: Custom);
690 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v4i1, Action: Custom);
691 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v2i1, Action: Custom);
692 }
693
694 // We want to custom lower some of our intrinsics.
695 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
696 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::f64, Action: Custom);
697 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::ppcf128, Action: Custom);
698 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v4f32, Action: Custom);
699 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v2f64, Action: Custom);
700
701 // To handle counter-based loop conditions.
702 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i1, Action: Custom);
703 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
704
705 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i8, Action: Custom);
706 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i16, Action: Custom);
707 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
708 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
709
710 // Comparisons that require checking two conditions.
711 if (Subtarget.hasSPE()) {
712 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f32, Action: Expand);
713 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f64, Action: Expand);
714 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f32, Action: Expand);
715 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f64, Action: Expand);
716 }
717 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f32, Action: Expand);
718 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f64, Action: Expand);
719 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f32, Action: Expand);
720 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f64, Action: Expand);
721 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f32, Action: Expand);
722 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f64, Action: Expand);
723 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f32, Action: Expand);
724 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f64, Action: Expand);
725 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f32, Action: Expand);
726 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f64, Action: Expand);
727 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f32, Action: Expand);
728 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f64, Action: Expand);
729
730 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
731 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
732
733 if (Subtarget.has64BitSupport()) {
734 // They also have instructions for converting between i64 and fp.
735 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
736 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Expand);
737 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
738 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Expand);
739 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
740 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Expand);
741 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
742 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Expand);
743 // This is just the low 32 bits of a (signed) fp->i64 conversion.
744 // We cannot do this with Promote because i64 is not a legal type.
745 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
746 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
747
748 if (Subtarget.hasLFIWAX() || isPPC64) {
749 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
750 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
751 }
752 } else {
753 // PowerPC does not have FP_TO_UINT on 32-bit implementations.
754 if (Subtarget.hasSPE()) {
755 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Legal);
756 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Legal);
757 } else {
758 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Expand);
759 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Expand);
760 }
761 }
762
763 // With the instructions enabled under FPCVT, we can do everything.
764 if (Subtarget.hasFPCVT()) {
765 if (Subtarget.has64BitSupport()) {
766 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
767 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Custom);
768 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
769 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Custom);
770 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
771 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Custom);
772 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
773 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
774 }
775
776 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
777 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
778 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
779 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Custom);
780 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
781 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
782 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
783 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Custom);
784 }
785
786 if (Subtarget.use64BitRegs()) {
787 // 64-bit PowerPC implementations can support i64 types directly
788 addRegisterClass(VT: MVT::i64, RC: &PPC::G8RCRegClass);
789 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
790 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
791 // 64-bit PowerPC wants to expand i128 shifts itself.
792 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Custom);
793 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Custom);
794 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Custom);
795 } else {
796 // 32-bit PowerPC wants to expand i64 shifts itself.
797 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Custom);
798 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Custom);
799 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Custom);
800 }
801
802 // PowerPC has better expansions for funnel shifts than the generic
803 // TargetLowering::expandFunnelShift.
804 if (Subtarget.has64BitSupport()) {
805 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Custom);
806 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Custom);
807 }
808 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Custom);
809 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Custom);
810
811 if (Subtarget.hasVSX()) {
812 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
813 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
814 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
815 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
816 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
817 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
818 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
819 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
820 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
821 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
822 }
823
824 if (Subtarget.hasAltivec()) {
825 for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
826 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
827 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
828 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
829 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
830 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
831 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
832 }
833 // First set operation action for all vector types to expand. Then we
834 // will selectively turn on ones that can be effectively codegen'd.
835 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
836 // add/sub are legal for all supported vector VT's.
837 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
838 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
839
840 // For v2i64, these are only valid with P8Vector. This is corrected after
841 // the loop.
842 if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) {
843 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
844 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
845 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
846 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
847 }
848 else {
849 setOperationAction(Op: ISD::SMAX, VT, Action: Expand);
850 setOperationAction(Op: ISD::SMIN, VT, Action: Expand);
851 setOperationAction(Op: ISD::UMAX, VT, Action: Expand);
852 setOperationAction(Op: ISD::UMIN, VT, Action: Expand);
853 }
854
855 if (Subtarget.hasVSX()) {
856 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT, Action: Legal);
857 setOperationAction(Op: ISD::FMINNUM_IEEE, VT, Action: Legal);
858 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
859 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
860 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Legal);
861 }
862
863 // Vector instructions introduced in P8
864 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
865 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
866 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
867 }
868 else {
869 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
870 setOperationAction(Op: ISD::CTLZ, VT, Action: Expand);
871 }
872
873 // Vector instructions introduced in P9
874 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128))
875 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
876 else
877 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
878
879 // We promote all shuffles to v16i8.
880 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Promote);
881 AddPromotedToType (Opc: ISD::VECTOR_SHUFFLE, OrigVT: VT, DestVT: MVT::v16i8);
882
883 // We promote all non-typed operations to v4i32.
884 setOperationAction(Op: ISD::AND , VT, Action: Promote);
885 AddPromotedToType (Opc: ISD::AND , OrigVT: VT, DestVT: MVT::v4i32);
886 setOperationAction(Op: ISD::OR , VT, Action: Promote);
887 AddPromotedToType (Opc: ISD::OR , OrigVT: VT, DestVT: MVT::v4i32);
888 setOperationAction(Op: ISD::XOR , VT, Action: Promote);
889 AddPromotedToType (Opc: ISD::XOR , OrigVT: VT, DestVT: MVT::v4i32);
890 setOperationAction(Op: ISD::LOAD , VT, Action: Promote);
891 AddPromotedToType (Opc: ISD::LOAD , OrigVT: VT, DestVT: MVT::v4i32);
892 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
893 AddPromotedToType (Opc: ISD::SELECT, OrigVT: VT, DestVT: MVT::v4i32);
894 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
895 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Promote);
896 AddPromotedToType (Opc: ISD::SELECT_CC, OrigVT: VT, DestVT: MVT::v4i32);
897 setOperationAction(Op: ISD::STORE, VT, Action: Promote);
898 AddPromotedToType (Opc: ISD::STORE, OrigVT: VT, DestVT: MVT::v4i32);
899
900 // No other operations are legal.
901 setOperationAction(Op: ISD::MUL , VT, Action: Expand);
902 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
903 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
904 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
905 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
906 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
907 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
908 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
909 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
910 setOperationAction(Op: ISD::FLOG, VT, Action: Expand);
911 setOperationAction(Op: ISD::FLOG10, VT, Action: Expand);
912 setOperationAction(Op: ISD::FLOG2, VT, Action: Expand);
913 setOperationAction(Op: ISD::FEXP, VT, Action: Expand);
914 setOperationAction(Op: ISD::FEXP2, VT, Action: Expand);
915 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
916 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
917 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
918 setOperationAction(Op: ISD::FFLOOR, VT, Action: Expand);
919 setOperationAction(Op: ISD::FCEIL, VT, Action: Expand);
920 setOperationAction(Op: ISD::FTRUNC, VT, Action: Expand);
921 setOperationAction(Op: ISD::FRINT, VT, Action: Expand);
922 setOperationAction(Op: ISD::FLDEXP, VT, Action: Expand);
923 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Expand);
924 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Expand);
925 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Expand);
926 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Expand);
927 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
928 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
929 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
930 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
931 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
932 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
933 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Expand);
934 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
935 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
936 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT, Action: Expand);
937 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
938 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
939
940 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
941 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
942 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
943 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
944 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
945 }
946 }
947 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::v4i32, Action: Expand);
948 if (!Subtarget.hasP8Vector()) {
949 setOperationAction(Op: ISD::SMAX, VT: MVT::v2i64, Action: Expand);
950 setOperationAction(Op: ISD::SMIN, VT: MVT::v2i64, Action: Expand);
951 setOperationAction(Op: ISD::UMAX, VT: MVT::v2i64, Action: Expand);
952 setOperationAction(Op: ISD::UMIN, VT: MVT::v2i64, Action: Expand);
953 }
954
955 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
956 // with merges, splats, etc.
957 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v16i8, Action: Custom);
958
959 // Vector truncates to sub-word integer that fit in an Altivec/VSX register
960 // are cheap, so handle them before they get expanded to scalar.
961 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v8i8, Action: Custom);
962 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i8, Action: Custom);
963 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i8, Action: Custom);
964 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i16, Action: Custom);
965 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i16, Action: Custom);
966
967 setOperationAction(Op: ISD::AND , VT: MVT::v4i32, Action: Legal);
968 setOperationAction(Op: ISD::OR , VT: MVT::v4i32, Action: Legal);
969 setOperationAction(Op: ISD::XOR , VT: MVT::v4i32, Action: Legal);
970 setOperationAction(Op: ISD::LOAD , VT: MVT::v4i32, Action: Legal);
971 setOperationAction(Op: ISD::SELECT, VT: MVT::v4i32,
972 Action: Subtarget.useCRBits() ? Legal : Expand);
973 setOperationAction(Op: ISD::STORE , VT: MVT::v4i32, Action: Legal);
974 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
975 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
976 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
977 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
978 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
979 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
980 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
981 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
982 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
983 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
984 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
985 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
986
987 // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8.
988 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Custom);
989 // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w).
990 if (Subtarget.hasAltivec())
991 for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8})
992 setOperationAction(Op: ISD::ROTL, VT, Action: Legal);
993 // With hasP8Altivec set, we can lower ISD::ROTL to vrld.
994 if (Subtarget.hasP8Altivec())
995 setOperationAction(Op: ISD::ROTL, VT: MVT::v2i64, Action: Legal);
996
997 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VRRCRegClass);
998 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VRRCRegClass);
999 addRegisterClass(VT: MVT::v8i16, RC: &PPC::VRRCRegClass);
1000 addRegisterClass(VT: MVT::v16i8, RC: &PPC::VRRCRegClass);
1001
1002 setOperationAction(Op: ISD::MUL, VT: MVT::v4f32, Action: Legal);
1003 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
1004
1005 if (Subtarget.hasVSX()) {
1006 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
1007 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
1008 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
1009 }
1010
1011 if (Subtarget.hasP8Altivec())
1012 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Legal);
1013 else
1014 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Custom);
1015
1016 if (Subtarget.isISA3_1()) {
1017 setOperationAction(Op: ISD::MUL, VT: MVT::v2i64, Action: Legal);
1018 setOperationAction(Op: ISD::MULHS, VT: MVT::v2i64, Action: Legal);
1019 setOperationAction(Op: ISD::MULHU, VT: MVT::v2i64, Action: Legal);
1020 setOperationAction(Op: ISD::MULHS, VT: MVT::v4i32, Action: Legal);
1021 setOperationAction(Op: ISD::MULHU, VT: MVT::v4i32, Action: Legal);
1022 setOperationAction(Op: ISD::UDIV, VT: MVT::v2i64, Action: Legal);
1023 setOperationAction(Op: ISD::SDIV, VT: MVT::v2i64, Action: Legal);
1024 setOperationAction(Op: ISD::UDIV, VT: MVT::v4i32, Action: Legal);
1025 setOperationAction(Op: ISD::SDIV, VT: MVT::v4i32, Action: Legal);
1026 setOperationAction(Op: ISD::UREM, VT: MVT::v2i64, Action: Legal);
1027 setOperationAction(Op: ISD::SREM, VT: MVT::v2i64, Action: Legal);
1028 setOperationAction(Op: ISD::UREM, VT: MVT::v4i32, Action: Legal);
1029 setOperationAction(Op: ISD::SREM, VT: MVT::v4i32, Action: Legal);
1030 setOperationAction(Op: ISD::UREM, VT: MVT::v1i128, Action: Legal);
1031 setOperationAction(Op: ISD::SREM, VT: MVT::v1i128, Action: Legal);
1032 setOperationAction(Op: ISD::UDIV, VT: MVT::v1i128, Action: Legal);
1033 setOperationAction(Op: ISD::SDIV, VT: MVT::v1i128, Action: Legal);
1034 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Legal);
1035 }
1036
1037 setOperationAction(Op: ISD::MUL, VT: MVT::v8i16, Action: Legal);
1038 setOperationAction(Op: ISD::MUL, VT: MVT::v16i8, Action: Custom);
1039
1040 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Custom);
1041 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Custom);
1042 // LE is P8+/64-bit so direct moves are supported and these operations
1043 // are legal. The custom transformation requires 64-bit since we need a
1044 // pair of stores that will cover a 128-bit load for P10.
1045 if (!DisableP10StoreForward && isPPC64 && !Subtarget.isLittleEndian()) {
1046 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Custom);
1047 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Custom);
1048 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Custom);
1049 }
1050
1051 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v16i8, Action: Custom);
1052 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v8i16, Action: Custom);
1053 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i32, Action: Custom);
1054 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4f32, Action: Custom);
1055
1056 // Altivec does not contain unordered floating-point compare instructions
1057 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v4f32, Action: Expand);
1058 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v4f32, Action: Expand);
1059 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v4f32, Action: Expand);
1060 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v4f32, Action: Expand);
1061
1062 if (Subtarget.hasVSX()) {
1063 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
1064 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1065 if (Subtarget.hasP8Vector()) {
1066 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
1067 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Legal);
1068 }
1069 if (Subtarget.hasDirectMove() && isPPC64) {
1070 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Legal);
1071 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Legal);
1072 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Legal);
1073 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Legal);
1074 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1075 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1076 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1077 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1078 }
1079 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1080
1081 // The nearbyint variants are not allowed to raise the inexact exception
1082 // so we can only code-gen them with fpexcept.ignore.
1083 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f64, Action: Custom);
1084 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f32, Action: Custom);
1085 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Custom);
1086 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Custom);
1087
1088 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
1089 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
1090 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
1091 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
1092 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
1093 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
1094 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
1095
1096 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
1097 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
1098 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
1099 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
1100
1101 setOperationAction(Op: ISD::MUL, VT: MVT::v2f64, Action: Legal);
1102 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
1103
1104 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
1105 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
1106
1107 // Share the Altivec comparison restrictions.
1108 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v2f64, Action: Expand);
1109 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v2f64, Action: Expand);
1110 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v2f64, Action: Expand);
1111 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v2f64, Action: Expand);
1112
1113 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Legal);
1114 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Legal);
1115
1116 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2f64, Action: Custom);
1117
1118 if (Subtarget.hasP8Vector())
1119 addRegisterClass(VT: MVT::f32, RC: &PPC::VSSRCRegClass);
1120
1121 addRegisterClass(VT: MVT::f64, RC: &PPC::VSFRCRegClass);
1122
1123 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VSRCRegClass);
1124 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VSRCRegClass);
1125 addRegisterClass(VT: MVT::v2f64, RC: &PPC::VSRCRegClass);
1126
1127 if (Subtarget.hasP8Altivec()) {
1128 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Legal);
1129 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Legal);
1130 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Legal);
1131
1132 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1133 // SRL, but not for SRA because of the instructions available:
1134 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth
1135 // doing
1136 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Expand);
1137 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Expand);
1138 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1139
1140 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Legal);
1141 }
1142 else {
1143 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Expand);
1144 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Expand);
1145 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Expand);
1146
1147 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Custom);
1148
1149 // VSX v2i64 only supports non-arithmetic operations.
1150 setOperationAction(Op: ISD::ADD, VT: MVT::v2i64, Action: Expand);
1151 setOperationAction(Op: ISD::SUB, VT: MVT::v2i64, Action: Expand);
1152 }
1153
1154 if (Subtarget.isISA3_1())
1155 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Legal);
1156 else
1157 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Expand);
1158
1159 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
1160 AddPromotedToType (Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1161 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
1162 AddPromotedToType (Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1163
1164 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2i64, Action: Custom);
1165
1166 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1167 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1168 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1169 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1170 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1171 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1172 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1173 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1174
1175 // Custom handling for partial vectors of integers converted to
1176 // floating point. We already have optimal handling for v2i32 through
1177 // the DAG combine, so those aren't necessary.
1178 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1179 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1180 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1181 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1182 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1183 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1184 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1185 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1186 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1187 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1188 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1189 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1190 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1191 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1192 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1193 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1194
1195 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
1196 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
1197 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
1198 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
1199 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v4f32, Action: Legal);
1200 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f64, Action: Legal);
1201
1202 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2i64, Action: Custom);
1203 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2f64, Action: Custom);
1204
1205 // Handle constrained floating-point operations of vector.
1206 // The predictor is `hasVSX` because altivec instruction has
1207 // no exception but VSX vector instruction has.
1208 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
1209 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
1210 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
1211 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
1212 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
1213 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
1214 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v4f32, Action: Legal);
1215 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v4f32, Action: Legal);
1216 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
1217 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
1218 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
1219 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
1220 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
1221
1222 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
1223 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
1224 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
1225 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
1226 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
1227 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
1228 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v2f64, Action: Legal);
1229 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v2f64, Action: Legal);
1230 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
1231 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
1232 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
1233 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
1234 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
1235
1236 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VSRCRegClass);
1237 addRegisterClass(VT: MVT::f128, RC: &PPC::VRRCRegClass);
1238
1239 for (MVT FPT : MVT::fp_valuetypes())
1240 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: FPT, Action: Expand);
1241
1242 // Expand the SELECT to SELECT_CC
1243 setOperationAction(Op: ISD::SELECT, VT: MVT::f128, Action: Expand);
1244
1245 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
1246 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
1247
1248 // No implementation for these ops for PowerPC.
1249 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f128, Action: Expand);
1250 setOperationAction(Op: ISD::FSIN, VT: MVT::f128, Action: Expand);
1251 setOperationAction(Op: ISD::FCOS, VT: MVT::f128, Action: Expand);
1252 setOperationAction(Op: ISD::FPOW, VT: MVT::f128, Action: Expand);
1253 setOperationAction(Op: ISD::FPOWI, VT: MVT::f128, Action: Expand);
1254 setOperationAction(Op: ISD::FREM, VT: MVT::f128, Action: LibCall);
1255 }
1256
1257 if (Subtarget.hasP8Altivec()) {
1258 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VRRCRegClass);
1259 addRegisterClass(VT: MVT::v1i128, RC: &PPC::VRRCRegClass);
1260 }
1261
1262 if (Subtarget.hasP9Vector()) {
1263 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Custom);
1264 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
1265
1266 // Test data class instructions store results in CR bits.
1267 if (Subtarget.useCRBits()) {
1268 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Custom);
1269 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Custom);
1270 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f128, Action: Custom);
1271 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::ppcf128, Action: Custom);
1272 }
1273
1274 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1275 // SRL, but not for SRA because of the instructions available:
1276 // VS{RL} and VS{RL}O.
1277 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Legal);
1278 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Legal);
1279 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1280
1281 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: Legal);
1282 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: Legal);
1283 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Legal);
1284 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Legal);
1285 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Legal);
1286
1287 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Legal);
1288 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f128, Action: Expand);
1289 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f128, Action: Expand);
1290 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f128, Action: Expand);
1291 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f128, Action: Expand);
1292 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f128, Action: Expand);
1293 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f128, Action: Expand);
1294
1295 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f128, Action: Legal);
1296 setOperationAction(Op: ISD::FRINT, VT: MVT::f128, Action: Legal);
1297 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f128, Action: Legal);
1298 setOperationAction(Op: ISD::FCEIL, VT: MVT::f128, Action: Legal);
1299 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f128, Action: Legal);
1300 setOperationAction(Op: ISD::FROUND, VT: MVT::f128, Action: Legal);
1301
1302 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f64, Action: Legal);
1303 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f32, Action: Legal);
1304 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
1305
1306 // Handle constrained floating-point operations of fp128
1307 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f128, Action: Legal);
1308 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f128, Action: Legal);
1309 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f128, Action: Legal);
1310 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f128, Action: Legal);
1311 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Legal);
1312 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f128, Action: Legal);
1313 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Legal);
1314 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f64, Action: Legal);
1315 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
1316 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f128, Action: Legal);
1317 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f128, Action: Legal);
1318 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f128, Action: Legal);
1319 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f128, Action: Legal);
1320 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f128, Action: Legal);
1321 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f128, Action: Legal);
1322 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
1323 setOperationAction(Op: ISD::BSWAP, VT: MVT::v8i16, Action: Legal);
1324 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i32, Action: Legal);
1325 setOperationAction(Op: ISD::BSWAP, VT: MVT::v2i64, Action: Legal);
1326 setOperationAction(Op: ISD::BSWAP, VT: MVT::v1i128, Action: Legal);
1327 } else if (Subtarget.hasVSX()) {
1328 setOperationAction(Op: ISD::LOAD, VT: MVT::f128, Action: Promote);
1329 setOperationAction(Op: ISD::STORE, VT: MVT::f128, Action: Promote);
1330
1331 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1332 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1333
1334 // Set FADD/FSUB as libcall to avoid the legalizer to expand the
1335 // fp_to_uint and int_to_fp.
1336 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: LibCall);
1337 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: LibCall);
1338
1339 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Expand);
1340 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Expand);
1341 setOperationAction(Op: ISD::FNEG, VT: MVT::f128, Action: Expand);
1342 setOperationAction(Op: ISD::FABS, VT: MVT::f128, Action: Expand);
1343 setOperationAction(Op: ISD::FSQRT, VT: MVT::f128, Action: Expand);
1344 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
1345 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
1346
1347 // Expand the fp_extend if the target type is fp128.
1348 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Expand);
1349 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Expand);
1350
1351 // Expand the fp_round if the source type is fp128.
1352 for (MVT VT : {MVT::f32, MVT::f64}) {
1353 setOperationAction(Op: ISD::FP_ROUND, VT, Action: Custom);
1354 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Custom);
1355 }
1356
1357 setOperationAction(Op: ISD::SETCC, VT: MVT::f128, Action: Custom);
1358 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Custom);
1359 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Custom);
1360 setOperationAction(Op: ISD::BR_CC, VT: MVT::f128, Action: Expand);
1361
1362 // Lower following f128 select_cc pattern:
1363 // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE
1364 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1365
1366 // We need to handle f128 SELECT_CC with integer result type.
1367 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i32, Action: Custom);
1368 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
1369 }
1370
1371 if (Subtarget.hasP9Altivec()) {
1372 if (Subtarget.isISA3_1()) {
1373 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1374 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1375 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1376 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1377 } else {
1378 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Custom);
1379 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Custom);
1380 }
1381 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i8, Action: Legal);
1382 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i16, Action: Legal);
1383 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i32, Action: Legal);
1384 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1385 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1386 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1387 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i64, Action: Legal);
1388
1389 setOperationAction(Op: ISD::ABDU, VT: MVT::v16i8, Action: Legal);
1390 setOperationAction(Op: ISD::ABDU, VT: MVT::v8i16, Action: Legal);
1391 setOperationAction(Op: ISD::ABDU, VT: MVT::v4i32, Action: Legal);
1392 setOperationAction(Op: ISD::ABDS, VT: MVT::v4i32, Action: Legal);
1393 }
1394
1395 if (Subtarget.hasP10Vector()) {
1396 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1397 }
1398
1399 setOperationAction(Op: ISD::PARTIAL_REDUCE_UMLA, VT: MVT::v16i32, Action: Custom);
1400 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1401 Action: Legal);
1402 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1403 Action: Legal);
1404 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1405 Action: Legal);
1406 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SUMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1407 Action: Legal);
1408 }
1409
1410 if (Subtarget.pairedVectorMemops()) {
1411 addRegisterClass(VT: MVT::v256i1, RC: &PPC::VSRpRCRegClass);
1412 setOperationAction(Op: ISD::LOAD, VT: MVT::v256i1, Action: Custom);
1413 setOperationAction(Op: ISD::STORE, VT: MVT::v256i1, Action: Custom);
1414 }
1415 if (Subtarget.hasMMA()) {
1416 if (Subtarget.isISAFuture()) {
1417 addRegisterClass(VT: MVT::v512i1, RC: &PPC::WACCRCRegClass);
1418 addRegisterClass(VT: MVT::v1024i1, RC: &PPC::DMRRCRegClass);
1419 addRegisterClass(VT: MVT::v2048i1, RC: &PPC::DMRpRCRegClass);
1420 setOperationAction(Op: ISD::LOAD, VT: MVT::v1024i1, Action: Custom);
1421 setOperationAction(Op: ISD::STORE, VT: MVT::v1024i1, Action: Custom);
1422 setOperationAction(Op: ISD::LOAD, VT: MVT::v2048i1, Action: Custom);
1423 setOperationAction(Op: ISD::STORE, VT: MVT::v2048i1, Action: Custom);
1424 } else {
1425 addRegisterClass(VT: MVT::v512i1, RC: &PPC::UACCRCRegClass);
1426 }
1427 setOperationAction(Op: ISD::LOAD, VT: MVT::v512i1, Action: Custom);
1428 setOperationAction(Op: ISD::STORE, VT: MVT::v512i1, Action: Custom);
1429 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v512i1, Action: Custom);
1430 }
1431
1432 if (Subtarget.has64BitSupport())
1433 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Legal);
1434
1435 if (Subtarget.isISA3_1())
1436 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Legal);
1437
1438 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: isPPC64 ? Legal : Custom);
1439
1440 if (!isPPC64) {
1441 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i64, Action: Expand);
1442 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i64, Action: Expand);
1443 }
1444
1445 if (shouldInlineQuadwordAtomics()) {
1446 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
1447 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
1448 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i128, Action: Custom);
1449 }
1450
1451 setBooleanContents(ZeroOrOneBooleanContent);
1452
1453 if (Subtarget.hasAltivec()) {
1454 // Altivec instructions set fields to all zeros or all ones.
1455 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
1456 }
1457
1458 if (shouldInlineQuadwordAtomics())
1459 setMaxAtomicSizeInBitsSupported(128);
1460 else if (isPPC64)
1461 setMaxAtomicSizeInBitsSupported(64);
1462 else
1463 setMaxAtomicSizeInBitsSupported(32);
1464
1465 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1);
1466
1467 // We have target-specific dag combine patterns for the following nodes:
1468 setTargetDAGCombine({ISD::AND, ISD::ADD, ISD::XOR, ISD::SHL, ISD::SRA,
1469 ISD::SRL, ISD::MUL, ISD::FMA, ISD::SINT_TO_FP,
1470 ISD::BUILD_VECTOR});
1471 if (Subtarget.hasFPCVT())
1472 setTargetDAGCombine(ISD::UINT_TO_FP);
1473 setTargetDAGCombine({ISD::LOAD, ISD::STORE, ISD::BR_CC});
1474 if (Subtarget.useCRBits())
1475 setTargetDAGCombine(ISD::BRCOND);
1476 setTargetDAGCombine({ISD::BSWAP, ISD::INTRINSIC_WO_CHAIN,
1477 ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID});
1478
1479 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND, ISD::ANY_EXTEND});
1480
1481 setTargetDAGCombine({ISD::TRUNCATE, ISD::VECTOR_SHUFFLE});
1482
1483 if (Subtarget.useCRBits()) {
1484 setTargetDAGCombine({ISD::SETCC, ISD::SELECT_CC});
1485 }
1486
1487 if (Subtarget.hasP8Vector())
1488 setTargetDAGCombine(ISD::BITCAST);
1489
1490 // With 32 condition bits, we don't need to sink (and duplicate) compares
1491 // aggressively in CodeGenPrep.
1492 if (Subtarget.useCRBits()) {
1493 setJumpIsExpensive();
1494 }
1495
1496 // TODO: The default entry number is set to 64. This stops most jump table
1497 // generation on PPC. But it is good for current PPC HWs because the indirect
1498 // branch instruction mtctr to the jump table may lead to bad branch predict.
1499 // Re-evaluate this value on future HWs that can do better with mtctr.
1500 setMinimumJumpTableEntries(PPCMinimumJumpTableEntries);
1501
1502 // The default minimum of largest number in a BitTest cluster is 3.
1503 setMinimumBitTestCmps(PPCMinimumBitTestCmps);
1504
1505 setMinFunctionAlignment(Align(4));
1506 setMinCmpXchgSizeInBits(Subtarget.hasPartwordAtomics() ? 8 : 32);
1507
1508 auto CPUDirective = Subtarget.getCPUDirective();
1509 switch (CPUDirective) {
1510 default: break;
1511 case PPC::DIR_970:
1512 case PPC::DIR_A2:
1513 case PPC::DIR_E500:
1514 case PPC::DIR_E500mc:
1515 case PPC::DIR_E5500:
1516 case PPC::DIR_PWR4:
1517 case PPC::DIR_PWR5:
1518 case PPC::DIR_PWR5X:
1519 case PPC::DIR_PWR6:
1520 case PPC::DIR_PWR6X:
1521 case PPC::DIR_PWR7:
1522 case PPC::DIR_PWR8:
1523 case PPC::DIR_PWR9:
1524 case PPC::DIR_PWR10:
1525 case PPC::DIR_PWR11:
1526 case PPC::DIR_PWR_FUTURE:
1527 setPrefLoopAlignment(Align(16));
1528 setPrefFunctionAlignment(Align(16));
1529 break;
1530 }
1531
1532 if (Subtarget.enableMachineScheduler())
1533 setSchedulingPreference(Sched::Source);
1534 else
1535 setSchedulingPreference(Sched::Hybrid);
1536
1537 computeRegisterProperties(TRI: STI.getRegisterInfo());
1538
1539 // The Freescale cores do better with aggressive inlining of memcpy and
1540 // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
1541 if (CPUDirective == PPC::DIR_E500mc || CPUDirective == PPC::DIR_E5500) {
1542 MaxStoresPerMemset = 32;
1543 MaxStoresPerMemsetOptSize = 16;
1544 MaxStoresPerMemcpy = 32;
1545 MaxStoresPerMemcpyOptSize = 8;
1546 MaxStoresPerMemmove = 32;
1547 MaxStoresPerMemmoveOptSize = 8;
1548 } else if (CPUDirective == PPC::DIR_A2) {
1549 // The A2 also benefits from (very) aggressive inlining of memcpy and
1550 // friends. The overhead of a the function call, even when warm, can be
1551 // over one hundred cycles.
1552 MaxStoresPerMemset = 128;
1553 MaxStoresPerMemcpy = 128;
1554 MaxStoresPerMemmove = 128;
1555 MaxLoadsPerMemcmp = 128;
1556 } else {
1557 MaxLoadsPerMemcmp = 8;
1558 MaxLoadsPerMemcmpOptSize = 4;
1559 }
1560
1561 // Enable generation of STXVP instructions by default for mcpu=future.
1562 if (CPUDirective == PPC::DIR_PWR_FUTURE &&
1563 DisableAutoPairedVecSt.getNumOccurrences() == 0)
1564 DisableAutoPairedVecSt = false;
1565
1566 IsStrictFPEnabled = true;
1567
1568 // Let the subtarget (CPU) decide if a predictable select is more expensive
1569 // than the corresponding branch. This information is used in CGP to decide
1570 // when to convert selects into branches.
1571 PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive();
1572
1573 GatherAllAliasesMaxDepth = PPCGatherAllAliasesMaxDepth;
1574}
1575
1576// *********************************** NOTE ************************************
1577// For selecting load and store instructions, the addressing modes are defined
1578// as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD
1579// patterns to match the load the store instructions.
1580//
1581// The TD definitions for the addressing modes correspond to their respective
1582// Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely
1583// on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the
1584// address mode flags of a particular node. Afterwards, the computed address
1585// flags are passed into getAddrModeForFlags() in order to retrieve the optimal
1586// addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement
1587// accordingly, based on the preferred addressing mode.
1588//
1589// Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode.
1590// MemOpFlags contains all the possible flags that can be used to compute the
1591// optimal addressing mode for load and store instructions.
1592// AddrMode contains all the possible load and store addressing modes available
1593// on Power (such as DForm, DSForm, DQForm, XForm, etc.)
1594//
1595// When adding new load and store instructions, it is possible that new address
1596// flags may need to be added into MemOpFlags, and a new addressing mode will
1597// need to be added to AddrMode. An entry of the new addressing mode (consisting
1598// of the minimal and main distinguishing address flags for the new load/store
1599// instructions) will need to be added into initializeAddrModeMap() below.
1600// Finally, when adding new addressing modes, the getAddrModeForFlags() will
1601// need to be updated to account for selecting the optimal addressing mode.
1602// *****************************************************************************
1603/// Initialize the map that relates the different addressing modes of the load
1604/// and store instructions to a set of flags. This ensures the load/store
1605/// instruction is correctly matched during instruction selection.
1606void PPCTargetLowering::initializeAddrModeMap() {
1607 AddrModesMap[PPC::AM_DForm] = {
1608 // LWZ, STW
1609 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt,
1610 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt,
1611 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1612 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1613 // LBZ, LHZ, STB, STH
1614 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1615 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1616 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1617 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1618 // LHA
1619 PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1620 PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1621 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1622 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1623 // LFS, LFD, STFS, STFD
1624 PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1625 PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1626 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1627 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1628 };
1629 AddrModesMap[PPC::AM_DSForm] = {
1630 // LWA
1631 PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt,
1632 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1633 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1634 // LD, STD
1635 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt,
1636 PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt,
1637 PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt,
1638 // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64
1639 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1640 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1641 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1642 };
1643 AddrModesMap[PPC::AM_DQForm] = {
1644 // LXV, STXV
1645 PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1646 PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1647 PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1648 };
1649 AddrModesMap[PPC::AM_PrefixDForm] = {PPC::MOF_RPlusSImm34 |
1650 PPC::MOF_SubtargetP10};
1651 // TODO: Add mapping for quadword load/store.
1652}
1653
1654/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
1655/// the desired ByVal argument alignment.
1656static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
1657 if (MaxAlign == MaxMaxAlign)
1658 return;
1659 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
1660 if (MaxMaxAlign >= 32 &&
1661 VTy->getPrimitiveSizeInBits().getFixedValue() >= 256)
1662 MaxAlign = Align(32);
1663 else if (VTy->getPrimitiveSizeInBits().getFixedValue() >= 128 &&
1664 MaxAlign < 16)
1665 MaxAlign = Align(16);
1666 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
1667 Align EltAlign;
1668 getMaxByValAlign(Ty: ATy->getElementType(), MaxAlign&: EltAlign, MaxMaxAlign);
1669 if (EltAlign > MaxAlign)
1670 MaxAlign = EltAlign;
1671 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
1672 for (auto *EltTy : STy->elements()) {
1673 Align EltAlign;
1674 getMaxByValAlign(Ty: EltTy, MaxAlign&: EltAlign, MaxMaxAlign);
1675 if (EltAlign > MaxAlign)
1676 MaxAlign = EltAlign;
1677 if (MaxAlign == MaxMaxAlign)
1678 break;
1679 }
1680 }
1681}
1682
1683/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1684/// function arguments in the caller parameter area.
1685Align PPCTargetLowering::getByValTypeAlignment(Type *Ty,
1686 const DataLayout &DL) const {
1687 // 16byte and wider vectors are passed on 16byte boundary.
1688 // The rest is 8 on PPC64 and 4 on PPC32 boundary.
1689 Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
1690 if (Subtarget.hasAltivec())
1691 getMaxByValAlign(Ty, MaxAlign&: Alignment, MaxMaxAlign: Align(16));
1692 return Alignment;
1693}
1694
1695bool PPCTargetLowering::useSoftFloat() const {
1696 return Subtarget.useSoftFloat();
1697}
1698
1699bool PPCTargetLowering::hasSPE() const {
1700 return Subtarget.hasSPE();
1701}
1702
1703bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
1704 return VT.isScalarInteger();
1705}
1706
1707bool PPCTargetLowering::shallExtractConstSplatVectorElementToStore(
1708 Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const {
1709 if (!Subtarget.isPPC64() || !Subtarget.hasVSX())
1710 return false;
1711
1712 if (auto *VTy = dyn_cast<VectorType>(Val: VectorTy)) {
1713 if (VTy->getScalarType()->isIntegerTy()) {
1714 // ElemSizeInBits 8/16 can fit in immediate field, not needed here.
1715 if (ElemSizeInBits == 32) {
1716 Index = Subtarget.isLittleEndian() ? 2 : 1;
1717 return true;
1718 }
1719 if (ElemSizeInBits == 64) {
1720 Index = Subtarget.isLittleEndian() ? 1 : 0;
1721 return true;
1722 }
1723 }
1724 }
1725 return false;
1726}
1727
1728EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1729 EVT VT) const {
1730 if (!VT.isVector())
1731 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
1732
1733 return VT.changeVectorElementTypeToInteger();
1734}
1735
1736bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const {
1737 assert(VT.isFloatingPoint() && "Non-floating-point FMA?");
1738 return true;
1739}
1740
1741//===----------------------------------------------------------------------===//
1742// Node matching predicates, for use by the tblgen matching code.
1743//===----------------------------------------------------------------------===//
1744
1745/// isFloatingPointZero - Return true if this is 0.0 or -0.0.
1746static bool isFloatingPointZero(SDValue Op) {
1747 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
1748 return CFP->getValueAPF().isZero();
1749 else if (ISD::isEXTLoad(N: Op.getNode()) || ISD::isNON_EXTLoad(N: Op.getNode())) {
1750 // Maybe this has already been legalized into the constant pool?
1751 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val: Op.getOperand(i: 1)))
1752 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CP->getConstVal()))
1753 return CFP->getValueAPF().isZero();
1754 }
1755 return false;
1756}
1757
1758/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return
1759/// true if Op is undef or if it matches the specified value.
1760static bool isConstantOrUndef(int Op, int Val) {
1761 return Op < 0 || Op == Val;
1762}
1763
1764/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
1765/// VPKUHUM instruction.
1766/// The ShuffleKind distinguishes between big-endian operations with
1767/// two different inputs (0), either-endian operations with two identical
1768/// inputs (1), and little-endian operations with two different inputs (2).
1769/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1770bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1771 SelectionDAG &DAG) {
1772 bool IsLE = DAG.getDataLayout().isLittleEndian();
1773 if (ShuffleKind == 0) {
1774 if (IsLE)
1775 return false;
1776 for (unsigned i = 0; i != 16; ++i)
1777 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+1))
1778 return false;
1779 } else if (ShuffleKind == 2) {
1780 if (!IsLE)
1781 return false;
1782 for (unsigned i = 0; i != 16; ++i)
1783 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2))
1784 return false;
1785 } else if (ShuffleKind == 1) {
1786 unsigned j = IsLE ? 0 : 1;
1787 for (unsigned i = 0; i != 8; ++i)
1788 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+j) ||
1789 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j))
1790 return false;
1791 }
1792 return true;
1793}
1794
1795/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
1796/// VPKUWUM instruction.
1797/// The ShuffleKind distinguishes between big-endian operations with
1798/// two different inputs (0), either-endian operations with two identical
1799/// inputs (1), and little-endian operations with two different inputs (2).
1800/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1801bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1802 SelectionDAG &DAG) {
1803 bool IsLE = DAG.getDataLayout().isLittleEndian();
1804 if (ShuffleKind == 0) {
1805 if (IsLE)
1806 return false;
1807 for (unsigned i = 0; i != 16; i += 2)
1808 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+2) ||
1809 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+3))
1810 return false;
1811 } else if (ShuffleKind == 2) {
1812 if (!IsLE)
1813 return false;
1814 for (unsigned i = 0; i != 16; i += 2)
1815 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1816 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1))
1817 return false;
1818 } else if (ShuffleKind == 1) {
1819 unsigned j = IsLE ? 0 : 2;
1820 for (unsigned i = 0; i != 8; i += 2)
1821 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1822 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1823 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1824 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1))
1825 return false;
1826 }
1827 return true;
1828}
1829
1830/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
1831/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the
1832/// current subtarget.
1833///
1834/// The ShuffleKind distinguishes between big-endian operations with
1835/// two different inputs (0), either-endian operations with two identical
1836/// inputs (1), and little-endian operations with two different inputs (2).
1837/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1838bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1839 SelectionDAG &DAG) {
1840 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
1841 if (!Subtarget.hasP8Vector())
1842 return false;
1843
1844 bool IsLE = DAG.getDataLayout().isLittleEndian();
1845 if (ShuffleKind == 0) {
1846 if (IsLE)
1847 return false;
1848 for (unsigned i = 0; i != 16; i += 4)
1849 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+4) ||
1850 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+5) ||
1851 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+6) ||
1852 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+7))
1853 return false;
1854 } else if (ShuffleKind == 2) {
1855 if (!IsLE)
1856 return false;
1857 for (unsigned i = 0; i != 16; i += 4)
1858 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1859 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1) ||
1860 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+2) ||
1861 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+3))
1862 return false;
1863 } else if (ShuffleKind == 1) {
1864 unsigned j = IsLE ? 0 : 4;
1865 for (unsigned i = 0; i != 8; i += 4)
1866 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1867 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1868 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+j+2) ||
1869 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+j+3) ||
1870 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1871 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1) ||
1872 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+10), Val: i*2+j+2) ||
1873 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+11), Val: i*2+j+3))
1874 return false;
1875 }
1876 return true;
1877}
1878
1879/// isVMerge - Common function, used to match vmrg* shuffles.
1880///
1881static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
1882 unsigned LHSStart, unsigned RHSStart) {
1883 if (N->getValueType(ResNo: 0) != MVT::v16i8)
1884 return false;
1885 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&
1886 "Unsupported merge size!");
1887
1888 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units
1889 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit
1890 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+j),
1891 Val: LHSStart+j+i*UnitSize) ||
1892 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+UnitSize+j),
1893 Val: RHSStart+j+i*UnitSize))
1894 return false;
1895 }
1896 return true;
1897}
1898
1899/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
1900/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes).
1901/// The ShuffleKind distinguishes between big-endian merges with two
1902/// different inputs (0), either-endian merges with two identical inputs (1),
1903/// and little-endian merges with two different inputs (2). For the latter,
1904/// the input operands are swapped (see PPCInstrAltivec.td).
1905bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1906 unsigned ShuffleKind, SelectionDAG &DAG) {
1907 if (DAG.getDataLayout().isLittleEndian()) {
1908 if (ShuffleKind == 1) // unary
1909 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1910 else if (ShuffleKind == 2) // swapped
1911 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1912 else
1913 return false;
1914 } else {
1915 if (ShuffleKind == 1) // unary
1916 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1917 else if (ShuffleKind == 0) // normal
1918 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1919 else
1920 return false;
1921 }
1922}
1923
1924/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
1925/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes).
1926/// The ShuffleKind distinguishes between big-endian merges with two
1927/// different inputs (0), either-endian merges with two identical inputs (1),
1928/// and little-endian merges with two different inputs (2). For the latter,
1929/// the input operands are swapped (see PPCInstrAltivec.td).
1930bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1931 unsigned ShuffleKind, SelectionDAG &DAG) {
1932 if (DAG.getDataLayout().isLittleEndian()) {
1933 if (ShuffleKind == 1) // unary
1934 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1935 else if (ShuffleKind == 2) // swapped
1936 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1937 else
1938 return false;
1939 } else {
1940 if (ShuffleKind == 1) // unary
1941 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1942 else if (ShuffleKind == 0) // normal
1943 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1944 else
1945 return false;
1946 }
1947}
1948
1949/**
1950 * Common function used to match vmrgew and vmrgow shuffles
1951 *
1952 * The indexOffset determines whether to look for even or odd words in
1953 * the shuffle mask. This is based on the of the endianness of the target
1954 * machine.
1955 * - Little Endian:
1956 * - Use offset of 0 to check for odd elements
1957 * - Use offset of 4 to check for even elements
1958 * - Big Endian:
1959 * - Use offset of 0 to check for even elements
1960 * - Use offset of 4 to check for odd elements
1961 * A detailed description of the vector element ordering for little endian and
1962 * big endian can be found at
1963 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html
1964 * Targeting your applications - what little endian and big endian IBM XL C/C++
1965 * compiler differences mean to you
1966 *
1967 * The mask to the shuffle vector instruction specifies the indices of the
1968 * elements from the two input vectors to place in the result. The elements are
1969 * numbered in array-access order, starting with the first vector. These vectors
1970 * are always of type v16i8, thus each vector will contain 16 elements of size
1971 * 8. More info on the shuffle vector can be found in the
1972 * http://llvm.org/docs/LangRef.html#shufflevector-instruction
1973 * Language Reference.
1974 *
1975 * The RHSStartValue indicates whether the same input vectors are used (unary)
1976 * or two different input vectors are used, based on the following:
1977 * - If the instruction uses the same vector for both inputs, the range of the
1978 * indices will be 0 to 15. In this case, the RHSStart value passed should
1979 * be 0.
1980 * - If the instruction has two different vectors then the range of the
1981 * indices will be 0 to 31. In this case, the RHSStart value passed should
1982 * be 16 (indices 0-15 specify elements in the first vector while indices 16
1983 * to 31 specify elements in the second vector).
1984 *
1985 * \param[in] N The shuffle vector SD Node to analyze
1986 * \param[in] IndexOffset Specifies whether to look for even or odd elements
1987 * \param[in] RHSStartValue Specifies the starting index for the righthand input
1988 * vector to the shuffle_vector instruction
1989 * \return true iff this shuffle vector represents an even or odd word merge
1990 */
1991static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset,
1992 unsigned RHSStartValue) {
1993 if (N->getValueType(ResNo: 0) != MVT::v16i8)
1994 return false;
1995
1996 for (unsigned i = 0; i < 2; ++i)
1997 for (unsigned j = 0; j < 4; ++j)
1998 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j),
1999 Val: i*RHSStartValue+j+IndexOffset) ||
2000 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j+8),
2001 Val: i*RHSStartValue+j+IndexOffset+8))
2002 return false;
2003 return true;
2004}
2005
2006/**
2007 * Determine if the specified shuffle mask is suitable for the vmrgew or
2008 * vmrgow instructions.
2009 *
2010 * \param[in] N The shuffle vector SD Node to analyze
2011 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false)
2012 * \param[in] ShuffleKind Identify the type of merge:
2013 * - 0 = big-endian merge with two different inputs;
2014 * - 1 = either-endian merge with two identical inputs;
2015 * - 2 = little-endian merge with two different inputs (inputs are swapped for
2016 * little-endian merges).
2017 * \param[in] DAG The current SelectionDAG
2018 * \return true iff this shuffle mask
2019 */
2020bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
2021 unsigned ShuffleKind, SelectionDAG &DAG) {
2022 if (DAG.getDataLayout().isLittleEndian()) {
2023 unsigned indexOffset = CheckEven ? 4 : 0;
2024 if (ShuffleKind == 1) // Unary
2025 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2026 else if (ShuffleKind == 2) // swapped
2027 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2028 else
2029 return false;
2030 }
2031 else {
2032 unsigned indexOffset = CheckEven ? 0 : 4;
2033 if (ShuffleKind == 1) // Unary
2034 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2035 else if (ShuffleKind == 0) // Normal
2036 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2037 else
2038 return false;
2039 }
2040 return false;
2041}
2042
2043/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
2044/// amount, otherwise return -1.
2045/// The ShuffleKind distinguishes between big-endian operations with two
2046/// different inputs (0), either-endian operations with two identical inputs
2047/// (1), and little-endian operations with two different inputs (2). For the
2048/// latter, the input operands are swapped (see PPCInstrAltivec.td).
2049int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
2050 SelectionDAG &DAG) {
2051 if (N->getValueType(ResNo: 0) != MVT::v16i8)
2052 return -1;
2053
2054 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2055
2056 // Find the first non-undef value in the shuffle mask.
2057 unsigned i;
2058 for (i = 0; i != 16 && SVOp->getMaskElt(Idx: i) < 0; ++i)
2059 /*search*/;
2060
2061 if (i == 16) return -1; // all undef.
2062
2063 // Otherwise, check to see if the rest of the elements are consecutively
2064 // numbered from this value.
2065 unsigned ShiftAmt = SVOp->getMaskElt(Idx: i);
2066 if (ShiftAmt < i) return -1;
2067
2068 ShiftAmt -= i;
2069 bool isLE = DAG.getDataLayout().isLittleEndian();
2070
2071 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
2072 // Check the rest of the elements to see if they are consecutive.
2073 for (++i; i != 16; ++i)
2074 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: ShiftAmt+i))
2075 return -1;
2076 } else if (ShuffleKind == 1) {
2077 // Check the rest of the elements to see if they are consecutive.
2078 for (++i; i != 16; ++i)
2079 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: (ShiftAmt+i) & 15))
2080 return -1;
2081 } else
2082 return -1;
2083
2084 if (isLE)
2085 ShiftAmt = 16 - ShiftAmt;
2086
2087 return ShiftAmt;
2088}
2089
2090/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
2091/// specifies a splat of a single element that is suitable for input to
2092/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.).
2093bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
2094 EVT VT = N->getValueType(ResNo: 0);
2095 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2096 return EltSize == 8 && N->getMaskElt(Idx: 0) == N->getMaskElt(Idx: 1);
2097
2098 assert(VT == MVT::v16i8 && isPowerOf2_32(EltSize) &&
2099 EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes");
2100
2101 // The consecutive indices need to specify an element, not part of two
2102 // different elements. So abandon ship early if this isn't the case.
2103 if (N->getMaskElt(Idx: 0) % EltSize != 0)
2104 return false;
2105
2106 // This is a splat operation if each element of the permute is the same, and
2107 // if the value doesn't reference the second vector.
2108 unsigned ElementBase = N->getMaskElt(Idx: 0);
2109
2110 // FIXME: Handle UNDEF elements too!
2111 if (ElementBase >= 16)
2112 return false;
2113
2114 // Check that the indices are consecutive, in the case of a multi-byte element
2115 // splatted with a v16i8 mask.
2116 for (unsigned i = 1; i != EltSize; ++i)
2117 if (N->getMaskElt(Idx: i) < 0 || N->getMaskElt(Idx: i) != (int)(i+ElementBase))
2118 return false;
2119
2120 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
2121 // An UNDEF element is a sequence of UNDEF bytes.
2122 if (N->getMaskElt(Idx: i) < 0) {
2123 for (unsigned j = 1; j != EltSize; ++j)
2124 if (N->getMaskElt(Idx: i + j) >= 0)
2125 return false;
2126 } else
2127 for (unsigned j = 0; j != EltSize; ++j)
2128 if (N->getMaskElt(Idx: i + j) != N->getMaskElt(Idx: j))
2129 return false;
2130 }
2131 return true;
2132}
2133
2134/// Check that the mask is shuffling N byte elements. Within each N byte
2135/// element of the mask, the indices could be either in increasing or
2136/// decreasing order as long as they are consecutive.
2137/// \param[in] N the shuffle vector SD Node to analyze
2138/// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/
2139/// Word/DoubleWord/QuadWord).
2140/// \param[in] StepLen the delta indices number among the N byte element, if
2141/// the mask is in increasing/decreasing order then it is 1/-1.
2142/// \return true iff the mask is shuffling N byte elements.
2143static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width,
2144 int StepLen) {
2145 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) &&
2146 "Unexpected element width.");
2147 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width.");
2148
2149 unsigned NumOfElem = 16 / Width;
2150 unsigned MaskVal[16]; // Width is never greater than 16
2151 for (unsigned i = 0; i < NumOfElem; ++i) {
2152 MaskVal[0] = N->getMaskElt(Idx: i * Width);
2153 if ((StepLen == 1) && (MaskVal[0] % Width)) {
2154 return false;
2155 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) {
2156 return false;
2157 }
2158
2159 for (unsigned int j = 1; j < Width; ++j) {
2160 MaskVal[j] = N->getMaskElt(Idx: i * Width + j);
2161 if (MaskVal[j] != MaskVal[j-1] + StepLen) {
2162 return false;
2163 }
2164 }
2165 }
2166
2167 return true;
2168}
2169
2170bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2171 unsigned &InsertAtByte, bool &Swap, bool IsLE) {
2172 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2173 return false;
2174
2175 // Now we look at mask elements 0,4,8,12
2176 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2177 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2178 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2179 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2180 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 };
2181 unsigned BigEndianShifts[] = { 3, 0, 1, 2 };
2182
2183 // Below, let H and L be arbitrary elements of the shuffle mask
2184 // where H is in the range [4,7] and L is in the range [0,3].
2185 // H, 1, 2, 3 or L, 5, 6, 7
2186 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) ||
2187 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) {
2188 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3];
2189 InsertAtByte = IsLE ? 12 : 0;
2190 Swap = M0 < 4;
2191 return true;
2192 }
2193 // 0, H, 2, 3 or 4, L, 6, 7
2194 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) ||
2195 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) {
2196 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3];
2197 InsertAtByte = IsLE ? 8 : 4;
2198 Swap = M1 < 4;
2199 return true;
2200 }
2201 // 0, 1, H, 3 or 4, 5, L, 7
2202 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) ||
2203 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) {
2204 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3];
2205 InsertAtByte = IsLE ? 4 : 8;
2206 Swap = M2 < 4;
2207 return true;
2208 }
2209 // 0, 1, 2, H or 4, 5, 6, L
2210 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) ||
2211 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) {
2212 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3];
2213 InsertAtByte = IsLE ? 0 : 12;
2214 Swap = M3 < 4;
2215 return true;
2216 }
2217
2218 // If both vector operands for the shuffle are the same vector, the mask will
2219 // contain only elements from the first one and the second one will be undef.
2220 if (N->getOperand(Num: 1).isUndef()) {
2221 ShiftElts = 0;
2222 Swap = true;
2223 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1;
2224 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) {
2225 InsertAtByte = IsLE ? 12 : 0;
2226 return true;
2227 }
2228 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) {
2229 InsertAtByte = IsLE ? 8 : 4;
2230 return true;
2231 }
2232 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) {
2233 InsertAtByte = IsLE ? 4 : 8;
2234 return true;
2235 }
2236 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) {
2237 InsertAtByte = IsLE ? 0 : 12;
2238 return true;
2239 }
2240 }
2241
2242 return false;
2243}
2244
2245bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2246 bool &Swap, bool IsLE) {
2247 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2248 // Ensure each byte index of the word is consecutive.
2249 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2250 return false;
2251
2252 // Now we look at mask elements 0,4,8,12, which are the beginning of words.
2253 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2254 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2255 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2256 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2257
2258 // If both vector operands for the shuffle are the same vector, the mask will
2259 // contain only elements from the first one and the second one will be undef.
2260 if (N->getOperand(Num: 1).isUndef()) {
2261 assert(M0 < 4 && "Indexing into an undef vector?");
2262 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4)
2263 return false;
2264
2265 ShiftElts = IsLE ? (4 - M0) % 4 : M0;
2266 Swap = false;
2267 return true;
2268 }
2269
2270 // Ensure each word index of the ShuffleVector Mask is consecutive.
2271 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8)
2272 return false;
2273
2274 if (IsLE) {
2275 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) {
2276 // Input vectors don't need to be swapped if the leading element
2277 // of the result is one of the 3 left elements of the second vector
2278 // (or if there is no shift to be done at all).
2279 Swap = false;
2280 ShiftElts = (8 - M0) % 8;
2281 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) {
2282 // Input vectors need to be swapped if the leading element
2283 // of the result is one of the 3 left elements of the first vector
2284 // (or if we're shifting by 4 - thereby simply swapping the vectors).
2285 Swap = true;
2286 ShiftElts = (4 - M0) % 4;
2287 }
2288
2289 return true;
2290 } else { // BE
2291 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) {
2292 // Input vectors don't need to be swapped if the leading element
2293 // of the result is one of the 4 elements of the first vector.
2294 Swap = false;
2295 ShiftElts = M0;
2296 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) {
2297 // Input vectors need to be swapped if the leading element
2298 // of the result is one of the 4 elements of the right vector.
2299 Swap = true;
2300 ShiftElts = M0 - 4;
2301 }
2302
2303 return true;
2304 }
2305}
2306
2307bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) {
2308 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2309
2310 if (!isNByteElemShuffleMask(N, Width, StepLen: -1))
2311 return false;
2312
2313 for (int i = 0; i < 16; i += Width)
2314 if (N->getMaskElt(Idx: i) != i + Width - 1)
2315 return false;
2316
2317 return true;
2318}
2319
2320bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) {
2321 return isXXBRShuffleMaskHelper(N, Width: 2);
2322}
2323
2324bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) {
2325 return isXXBRShuffleMaskHelper(N, Width: 4);
2326}
2327
2328bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) {
2329 return isXXBRShuffleMaskHelper(N, Width: 8);
2330}
2331
2332bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) {
2333 return isXXBRShuffleMaskHelper(N, Width: 16);
2334}
2335
2336/// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap
2337/// if the inputs to the instruction should be swapped and set \p DM to the
2338/// value for the immediate.
2339/// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI
2340/// AND element 0 of the result comes from the first input (LE) or second input
2341/// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered.
2342/// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle
2343/// mask.
2344bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM,
2345 bool &Swap, bool IsLE) {
2346 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2347
2348 // Ensure each byte index of the double word is consecutive.
2349 if (!isNByteElemShuffleMask(N, Width: 8, StepLen: 1))
2350 return false;
2351
2352 unsigned M0 = N->getMaskElt(Idx: 0) / 8;
2353 unsigned M1 = N->getMaskElt(Idx: 8) / 8;
2354 assert(((M0 | M1) < 4) && "A mask element out of bounds?");
2355
2356 // If both vector operands for the shuffle are the same vector, the mask will
2357 // contain only elements from the first one and the second one will be undef.
2358 if (N->getOperand(Num: 1).isUndef()) {
2359 if ((M0 | M1) < 2) {
2360 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1);
2361 Swap = false;
2362 return true;
2363 } else
2364 return false;
2365 }
2366
2367 if (IsLE) {
2368 if (M0 > 1 && M1 < 2) {
2369 Swap = false;
2370 } else if (M0 < 2 && M1 > 1) {
2371 M0 = (M0 + 2) % 4;
2372 M1 = (M1 + 2) % 4;
2373 Swap = true;
2374 } else
2375 return false;
2376
2377 // Note: if control flow comes here that means Swap is already set above
2378 DM = (((~M1) & 1) << 1) + ((~M0) & 1);
2379 return true;
2380 } else { // BE
2381 if (M0 < 2 && M1 > 1) {
2382 Swap = false;
2383 } else if (M0 > 1 && M1 < 2) {
2384 M0 = (M0 + 2) % 4;
2385 M1 = (M1 + 2) % 4;
2386 Swap = true;
2387 } else
2388 return false;
2389
2390 // Note: if control flow comes here that means Swap is already set above
2391 DM = (M0 << 1) + (M1 & 1);
2392 return true;
2393 }
2394}
2395
2396
2397/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
2398/// appropriate for PPC mnemonics (which have a big endian bias - namely
2399/// elements are counted from the left of the vector register).
2400unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
2401 SelectionDAG &DAG) {
2402 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2403 assert(isSplatShuffleMask(SVOp, EltSize));
2404 EVT VT = SVOp->getValueType(ResNo: 0);
2405
2406 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2407 return DAG.getDataLayout().isLittleEndian() ? 1 - SVOp->getMaskElt(Idx: 0)
2408 : SVOp->getMaskElt(Idx: 0);
2409
2410 if (DAG.getDataLayout().isLittleEndian())
2411 return (16 / EltSize) - 1 - (SVOp->getMaskElt(Idx: 0) / EltSize);
2412 else
2413 return SVOp->getMaskElt(Idx: 0) / EltSize;
2414}
2415
2416/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
2417/// by using a vspltis[bhw] instruction of the specified element size, return
2418/// the constant being splatted. The ByteSize field indicates the number of
2419/// bytes of each element [124] -> [bhw].
2420SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
2421 SDValue OpVal;
2422
2423 // If ByteSize of the splat is bigger than the element size of the
2424 // build_vector, then we have a case where we are checking for a splat where
2425 // multiple elements of the buildvector are folded together into a single
2426 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8).
2427 unsigned EltSize = 16/N->getNumOperands();
2428 if (EltSize < ByteSize) {
2429 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval.
2430 SDValue UniquedVals[4];
2431 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?");
2432
2433 // See if all of the elements in the buildvector agree across.
2434 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2435 if (N->getOperand(Num: i).isUndef()) continue;
2436 // If the element isn't a constant, bail fully out.
2437 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: i))) return SDValue();
2438
2439 if (!UniquedVals[i&(Multiple-1)].getNode())
2440 UniquedVals[i&(Multiple-1)] = N->getOperand(Num: i);
2441 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(Num: i))
2442 return SDValue(); // no match.
2443 }
2444
2445 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains
2446 // either constant or undef values that are identical for each chunk. See
2447 // if these chunks can form into a larger vspltis*.
2448
2449 // Check to see if all of the leading entries are either 0 or -1. If
2450 // neither, then this won't fit into the immediate field.
2451 bool LeadingZero = true;
2452 bool LeadingOnes = true;
2453 for (unsigned i = 0; i != Multiple-1; ++i) {
2454 if (!UniquedVals[i].getNode()) continue; // Must have been undefs.
2455
2456 LeadingZero &= isNullConstant(V: UniquedVals[i]);
2457 LeadingOnes &= isAllOnesConstant(V: UniquedVals[i]);
2458 }
2459 // Finally, check the least significant entry.
2460 if (LeadingZero) {
2461 if (!UniquedVals[Multiple-1].getNode())
2462 return DAG.getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32); // 0,0,0,undef
2463 int Val = UniquedVals[Multiple - 1]->getAsZExtVal();
2464 if (Val < 16) // 0,0,0,4 -> vspltisw(4)
2465 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2466 }
2467 if (LeadingOnes) {
2468 if (!UniquedVals[Multiple-1].getNode())
2469 return DAG.getTargetConstant(Val: ~0U, DL: SDLoc(N), VT: MVT::i32); // -1,-1,-1,undef
2470 int Val =cast<ConstantSDNode>(Val&: UniquedVals[Multiple-1])->getSExtValue();
2471 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2)
2472 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2473 }
2474
2475 return SDValue();
2476 }
2477
2478 // Check to see if this buildvec has a single non-undef value in its elements.
2479 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2480 if (N->getOperand(Num: i).isUndef()) continue;
2481 if (!OpVal.getNode())
2482 OpVal = N->getOperand(Num: i);
2483 else if (OpVal != N->getOperand(Num: i))
2484 return SDValue();
2485 }
2486
2487 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def.
2488
2489 unsigned ValSizeInBytes = EltSize;
2490 uint64_t Value = 0;
2491 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: OpVal)) {
2492 Value = CN->getZExtValue();
2493 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal)) {
2494 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!");
2495 Value = llvm::bit_cast<uint32_t>(from: CN->getValueAPF().convertToFloat());
2496 }
2497
2498 // If the splat value is larger than the element value, then we can never do
2499 // this splat. The only case that we could fit the replicated bits into our
2500 // immediate field for would be zero, and we prefer to use vxor for it.
2501 if (ValSizeInBytes < ByteSize) return SDValue();
2502
2503 // If the element value is larger than the splat value, check if it consists
2504 // of a repeated bit pattern of size ByteSize.
2505 if (!APInt(ValSizeInBytes * 8, Value).isSplat(SplatSizeInBits: ByteSize * 8))
2506 return SDValue();
2507
2508 // Properly sign extend the value.
2509 int MaskVal = SignExtend32(X: Value, B: ByteSize * 8);
2510
2511 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
2512 if (MaskVal == 0) return SDValue();
2513
2514 // Finally, if this value fits in a 5 bit sext field, return it
2515 if (SignExtend32<5>(X: MaskVal) == MaskVal)
2516 return DAG.getSignedTargetConstant(Val: MaskVal, DL: SDLoc(N), VT: MVT::i32);
2517 return SDValue();
2518}
2519
2520//===----------------------------------------------------------------------===//
2521// Addressing Mode Selection
2522//===----------------------------------------------------------------------===//
2523
2524/// isIntS16Immediate - This method tests to see if the node is either a 32-bit
2525/// or 64-bit immediate, and if the value can be accurately represented as a
2526/// sign extension from a 16-bit value. If so, this returns true and the
2527/// immediate.
2528bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) {
2529 if (!isa<ConstantSDNode>(Val: N))
2530 return false;
2531
2532 Imm = (int16_t)N->getAsZExtVal();
2533 if (N->getValueType(ResNo: 0) == MVT::i32)
2534 return Imm == (int32_t)N->getAsZExtVal();
2535 else
2536 return Imm == (int64_t)N->getAsZExtVal();
2537}
2538bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) {
2539 return isIntS16Immediate(N: Op.getNode(), Imm);
2540}
2541
2542/// Used when computing address flags for selecting loads and stores.
2543/// If we have an OR, check if the LHS and RHS are provably disjoint.
2544/// An OR of two provably disjoint values is equivalent to an ADD.
2545/// Most PPC load/store instructions compute the effective address as a sum,
2546/// so doing this conversion is useful.
2547static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) {
2548 if (N.getOpcode() != ISD::OR)
2549 return false;
2550 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2551 if (!LHSKnown.Zero.getBoolValue())
2552 return false;
2553 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2554 return (~(LHSKnown.Zero | RHSKnown.Zero) == 0);
2555}
2556
2557/// SelectAddressEVXRegReg - Given the specified address, check to see if it can
2558/// be represented as an indexed [r+r] operation.
2559bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
2560 SDValue &Index,
2561 SelectionDAG &DAG) const {
2562 for (SDNode *U : N->users()) {
2563 if (MemSDNode *Memop = dyn_cast<MemSDNode>(Val: U)) {
2564 if (Memop->getMemoryVT() == MVT::f64) {
2565 Base = N.getOperand(i: 0);
2566 Index = N.getOperand(i: 1);
2567 return true;
2568 }
2569 }
2570 }
2571 return false;
2572}
2573
2574/// isIntS34Immediate - This method tests if value of node given can be
2575/// accurately represented as a sign extension from a 34-bit value. If so,
2576/// this returns true and the immediate.
2577bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) {
2578 if (!isa<ConstantSDNode>(Val: N))
2579 return false;
2580
2581 Imm = cast<ConstantSDNode>(Val: N)->getSExtValue();
2582 return isInt<34>(x: Imm);
2583}
2584bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) {
2585 return isIntS34Immediate(N: Op.getNode(), Imm);
2586}
2587
2588/// SelectAddressRegReg - Given the specified addressed, check to see if it
2589/// can be represented as an indexed [r+r] operation. Returns false if it
2590/// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is
2591/// non-zero and N can be represented by a base register plus a signed 16-bit
2592/// displacement, make a more precise judgement by checking (displacement % \p
2593/// EncodingAlignment).
2594bool PPCTargetLowering::SelectAddressRegReg(
2595 SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG,
2596 MaybeAlign EncodingAlignment) const {
2597 // If we have a PC Relative target flag don't select as [reg+reg]. It will be
2598 // a [pc+imm].
2599 if (SelectAddressPCRel(N, Base))
2600 return false;
2601
2602 int16_t Imm = 0;
2603 if (N.getOpcode() == ISD::ADD) {
2604 // Is there any SPE load/store (f64), which can't handle 16bit offset?
2605 // SPE load/store can only handle 8-bit offsets.
2606 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG))
2607 return true;
2608 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2609 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2610 return false; // r+i
2611 if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo)
2612 return false; // r+i
2613
2614 Base = N.getOperand(i: 0);
2615 Index = N.getOperand(i: 1);
2616 return true;
2617 } else if (N.getOpcode() == ISD::OR) {
2618 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2619 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2620 return false; // r+i can fold it if we can.
2621
2622 // If this is an or of disjoint bitfields, we can codegen this as an add
2623 // (for better address arithmetic) if the LHS and RHS of the OR are provably
2624 // disjoint.
2625 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2626
2627 if (LHSKnown.Zero.getBoolValue()) {
2628 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2629 // If all of the bits are known zero on the LHS or RHS, the add won't
2630 // carry.
2631 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) {
2632 Base = N.getOperand(i: 0);
2633 Index = N.getOperand(i: 1);
2634 return true;
2635 }
2636 }
2637 }
2638
2639 return false;
2640}
2641
2642// If we happen to be doing an i64 load or store into a stack slot that has
2643// less than a 4-byte alignment, then the frame-index elimination may need to
2644// use an indexed load or store instruction (because the offset may not be a
2645// multiple of 4). The extra register needed to hold the offset comes from the
2646// register scavenger, and it is possible that the scavenger will need to use
2647// an emergency spill slot. As a result, we need to make sure that a spill slot
2648// is allocated when doing an i64 load/store into a less-than-4-byte-aligned
2649// stack slot.
2650static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
2651 // FIXME: This does not handle the LWA case.
2652 if (VT != MVT::i64)
2653 return;
2654
2655 // NOTE: We'll exclude negative FIs here, which come from argument
2656 // lowering, because there are no known test cases triggering this problem
2657 // using packed structures (or similar). We can remove this exclusion if
2658 // we find such a test case. The reason why this is so test-case driven is
2659 // because this entire 'fixup' is only to prevent crashes (from the
2660 // register scavenger) on not-really-valid inputs. For example, if we have:
2661 // %a = alloca i1
2662 // %b = bitcast i1* %a to i64*
2663 // store i64* a, i64 b
2664 // then the store should really be marked as 'align 1', but is not. If it
2665 // were marked as 'align 1' then the indexed form would have been
2666 // instruction-selected initially, and the problem this 'fixup' is preventing
2667 // won't happen regardless.
2668 if (FrameIdx < 0)
2669 return;
2670
2671 MachineFunction &MF = DAG.getMachineFunction();
2672 MachineFrameInfo &MFI = MF.getFrameInfo();
2673
2674 if (MFI.getObjectAlign(ObjectIdx: FrameIdx) >= Align(4))
2675 return;
2676
2677 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2678 FuncInfo->setHasNonRISpills();
2679}
2680
2681/// Returns true if the address N can be represented by a base register plus
2682/// a signed 16-bit displacement [r+imm], and if it is not better
2683/// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept
2684/// displacements that are multiples of that value.
2685bool PPCTargetLowering::SelectAddressRegImm(
2686 SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG,
2687 MaybeAlign EncodingAlignment) const {
2688 // FIXME dl should come from parent load or store, not from address
2689 SDLoc dl(N);
2690
2691 // If we have a PC Relative target flag don't select as [reg+imm]. It will be
2692 // a [pc+imm].
2693 if (SelectAddressPCRel(N, Base))
2694 return false;
2695
2696 // If this can be more profitably realized as r+r, fail.
2697 if (SelectAddressRegReg(N, Base&: Disp, Index&: Base, DAG, EncodingAlignment))
2698 return false;
2699
2700 if (N.getOpcode() == ISD::ADD) {
2701 int16_t imm = 0;
2702 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2703 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2704 Disp = DAG.getSignedTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2705 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2706 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2707 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2708 } else {
2709 Base = N.getOperand(i: 0);
2710 }
2711 return true; // [r+i]
2712 } else if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo) {
2713 // Match LOAD (ADD (X, Lo(G))).
2714 assert(!N.getOperand(1).getConstantOperandVal(1) &&
2715 "Cannot handle constant offsets yet!");
2716 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
2717 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
2718 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
2719 Disp.getOpcode() == ISD::TargetConstantPool ||
2720 Disp.getOpcode() == ISD::TargetJumpTable);
2721 Base = N.getOperand(i: 0);
2722 return true; // [&g+r]
2723 }
2724 } else if (N.getOpcode() == ISD::OR) {
2725 int16_t imm = 0;
2726 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2727 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2728 // If this is an or of disjoint bitfields, we can codegen this as an add
2729 // (for better address arithmetic) if the LHS and RHS of the OR are
2730 // provably disjoint.
2731 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2732
2733 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
2734 // If all of the bits are known zero on the LHS or RHS, the add won't
2735 // carry.
2736 if (FrameIndexSDNode *FI =
2737 dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2738 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2739 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2740 } else {
2741 Base = N.getOperand(i: 0);
2742 }
2743 Disp = DAG.getTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2744 return true;
2745 }
2746 }
2747 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
2748 // Loading from a constant address.
2749
2750 // If this address fits entirely in a 16-bit sext immediate field, codegen
2751 // this as "d, 0"
2752 int16_t Imm;
2753 if (isIntS16Immediate(N: CN, Imm) &&
2754 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm))) {
2755 Disp = DAG.getTargetConstant(Val: Imm, DL: dl, VT: CN->getValueType(ResNo: 0));
2756 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2757 VT: CN->getValueType(ResNo: 0));
2758 return true;
2759 }
2760
2761 // Handle 32-bit sext immediates with LIS + addr mode.
2762 if ((CN->getValueType(ResNo: 0) == MVT::i32 ||
2763 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
2764 (!EncodingAlignment ||
2765 isAligned(Lhs: *EncodingAlignment, SizeInBytes: CN->getZExtValue()))) {
2766 int Addr = (int)CN->getZExtValue();
2767
2768 // Otherwise, break this down into an LIS + disp.
2769 Disp = DAG.getTargetConstant(Val: (short)Addr, DL: dl, VT: MVT::i32);
2770
2771 Base = DAG.getTargetConstant(Val: (Addr - (signed short)Addr) >> 16, DL: dl,
2772 VT: MVT::i32);
2773 unsigned Opc = CN->getValueType(ResNo: 0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
2774 Base = SDValue(DAG.getMachineNode(Opcode: Opc, dl, VT: CN->getValueType(ResNo: 0), Op1: Base), 0);
2775 return true;
2776 }
2777 }
2778
2779 Disp = DAG.getTargetConstant(Val: 0, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()));
2780 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
2781 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2782 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2783 } else
2784 Base = N;
2785 return true; // [r+0]
2786}
2787
2788/// Similar to the 16-bit case but for instructions that take a 34-bit
2789/// displacement field (prefixed loads/stores).
2790bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
2791 SDValue &Base,
2792 SelectionDAG &DAG) const {
2793 // Only on 64-bit targets.
2794 if (N.getValueType() != MVT::i64)
2795 return false;
2796
2797 SDLoc dl(N);
2798 int64_t Imm = 0;
2799
2800 if (N.getOpcode() == ISD::ADD) {
2801 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2802 return false;
2803 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2804 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2805 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2806 else
2807 Base = N.getOperand(i: 0);
2808 return true;
2809 }
2810
2811 if (N.getOpcode() == ISD::OR) {
2812 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2813 return false;
2814 // If this is an or of disjoint bitfields, we can codegen this as an add
2815 // (for better address arithmetic) if the LHS and RHS of the OR are
2816 // provably disjoint.
2817 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2818 if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL)
2819 return false;
2820 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2821 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2822 else
2823 Base = N.getOperand(i: 0);
2824 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2825 return true;
2826 }
2827
2828 if (isIntS34Immediate(Op: N, Imm)) { // If the address is a 34-bit const.
2829 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2830 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
2831 return true;
2832 }
2833
2834 return false;
2835}
2836
2837/// SelectAddressRegRegOnly - Given the specified addressed, force it to be
2838/// represented as an indexed [r+r] operation.
2839bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
2840 SDValue &Index,
2841 SelectionDAG &DAG) const {
2842 // Check to see if we can easily represent this as an [r+r] address. This
2843 // will fail if it thinks that the address is more profitably represented as
2844 // reg+imm, e.g. where imm = 0.
2845 if (SelectAddressRegReg(N, Base, Index, DAG))
2846 return true;
2847
2848 // If the address is the result of an add, we will utilize the fact that the
2849 // address calculation includes an implicit add. However, we can reduce
2850 // register pressure if we do not materialize a constant just for use as the
2851 // index register. We only get rid of the add if it is not an add of a
2852 // value and a 16-bit signed constant and both have a single use.
2853 int16_t imm = 0;
2854 if (N.getOpcode() == ISD::ADD &&
2855 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) ||
2856 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
2857 Base = N.getOperand(i: 0);
2858 Index = N.getOperand(i: 1);
2859 return true;
2860 }
2861
2862 // Otherwise, do it the hard way, using R0 as the base register.
2863 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2864 VT: N.getValueType());
2865 Index = N;
2866 return true;
2867}
2868
2869template <typename Ty> static bool isValidPCRelNode(SDValue N) {
2870 Ty *PCRelCand = dyn_cast<Ty>(N);
2871 return PCRelCand && (PPCInstrInfo::hasPCRelFlag(TF: PCRelCand->getTargetFlags()));
2872}
2873
2874/// Returns true if this address is a PC Relative address.
2875/// PC Relative addresses are marked with the flag PPCII::MO_PCREL_FLAG
2876/// or if the node opcode is PPCISD::MAT_PCREL_ADDR.
2877bool PPCTargetLowering::SelectAddressPCRel(SDValue N, SDValue &Base) const {
2878 // This is a materialize PC Relative node. Always select this as PC Relative.
2879 Base = N;
2880 if (N.getOpcode() == PPCISD::MAT_PCREL_ADDR)
2881 return true;
2882 if (isValidPCRelNode<ConstantPoolSDNode>(N) ||
2883 isValidPCRelNode<GlobalAddressSDNode>(N) ||
2884 isValidPCRelNode<JumpTableSDNode>(N) ||
2885 isValidPCRelNode<BlockAddressSDNode>(N))
2886 return true;
2887 return false;
2888}
2889
2890/// Returns true if we should use a direct load into vector instruction
2891/// (such as lxsd or lfd), instead of a load into gpr + direct move sequence.
2892static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) {
2893
2894 // If there are any other uses other than scalar to vector, then we should
2895 // keep it as a scalar load -> direct move pattern to prevent multiple
2896 // loads.
2897 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N);
2898 if (!LD)
2899 return false;
2900
2901 EVT MemVT = LD->getMemoryVT();
2902 if (!MemVT.isSimple())
2903 return false;
2904 switch(MemVT.getSimpleVT().SimpleTy) {
2905 case MVT::i64:
2906 break;
2907 case MVT::i32:
2908 if (!ST.hasP8Vector())
2909 return false;
2910 break;
2911 case MVT::i16:
2912 case MVT::i8:
2913 if (!ST.hasP9Vector())
2914 return false;
2915 break;
2916 default:
2917 return false;
2918 }
2919
2920 SDValue LoadedVal(N, 0);
2921 if (!LoadedVal.hasOneUse())
2922 return false;
2923
2924 for (SDUse &Use : LD->uses())
2925 if (Use.getResNo() == 0 &&
2926 Use.getUser()->getOpcode() != ISD::SCALAR_TO_VECTOR &&
2927 Use.getUser()->getOpcode() != PPCISD::SCALAR_TO_VECTOR_PERMUTED)
2928 return false;
2929
2930 return true;
2931}
2932
2933/// getPreIndexedAddressParts - returns true by value, base pointer and
2934/// offset pointer and addressing mode by reference if the node's address
2935/// can be legally represented as pre-indexed load / store address.
2936bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
2937 SDValue &Offset,
2938 ISD::MemIndexedMode &AM,
2939 SelectionDAG &DAG) const {
2940 if (DisablePPCPreinc) return false;
2941
2942 bool isLoad = true;
2943 SDValue Ptr;
2944 EVT VT;
2945 Align Alignment;
2946 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
2947 Ptr = LD->getBasePtr();
2948 VT = LD->getMemoryVT();
2949 Alignment = LD->getAlign();
2950 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
2951 Ptr = ST->getBasePtr();
2952 VT = ST->getMemoryVT();
2953 Alignment = ST->getAlign();
2954 isLoad = false;
2955 } else
2956 return false;
2957
2958 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector
2959 // instructions because we can fold these into a more efficient instruction
2960 // instead, (such as LXSD).
2961 if (isLoad && usePartialVectorLoads(N, ST: Subtarget)) {
2962 return false;
2963 }
2964
2965 // PowerPC doesn't have preinc load/store instructions for vectors
2966 if (VT.isVector())
2967 return false;
2968
2969 if (SelectAddressRegReg(N: Ptr, Base, Index&: Offset, DAG)) {
2970 // Common code will reject creating a pre-inc form if the base pointer
2971 // is a frame index, or if N is a store and the base pointer is either
2972 // the same as or a predecessor of the value being stored. Check for
2973 // those situations here, and try with swapped Base/Offset instead.
2974 bool Swap = false;
2975
2976 if (isa<FrameIndexSDNode>(Val: Base) || isa<RegisterSDNode>(Val: Base))
2977 Swap = true;
2978 else if (!isLoad) {
2979 SDValue Val = cast<StoreSDNode>(Val: N)->getValue();
2980 if (Val == Base || Base.getNode()->isPredecessorOf(N: Val.getNode()))
2981 Swap = true;
2982 }
2983
2984 if (Swap)
2985 std::swap(a&: Base, b&: Offset);
2986
2987 AM = ISD::PRE_INC;
2988 return true;
2989 }
2990
2991 // LDU/STU can only handle immediates that are a multiple of 4.
2992 if (VT != MVT::i64) {
2993 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: std::nullopt))
2994 return false;
2995 } else {
2996 // LDU/STU need an address with at least 4-byte alignment.
2997 if (Alignment < Align(4))
2998 return false;
2999
3000 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: Align(4)))
3001 return false;
3002 }
3003
3004 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
3005 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of
3006 // sext i32 to i64 when addr mode is r+i.
3007 if (LD->getValueType(ResNo: 0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 &&
3008 LD->getExtensionType() == ISD::SEXTLOAD &&
3009 isa<ConstantSDNode>(Val: Offset))
3010 return false;
3011 }
3012
3013 AM = ISD::PRE_INC;
3014 return true;
3015}
3016
3017//===----------------------------------------------------------------------===//
3018// LowerOperation implementation
3019//===----------------------------------------------------------------------===//
3020
3021/// Return true if we should reference labels using a PICBase, set the HiOpFlags
3022/// and LoOpFlags to the target MO flags.
3023static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget,
3024 unsigned &HiOpFlags, unsigned &LoOpFlags,
3025 const GlobalValue *GV = nullptr) {
3026 HiOpFlags = PPCII::MO_HA;
3027 LoOpFlags = PPCII::MO_LO;
3028
3029 // Don't use the pic base if not in PIC relocation model.
3030 if (IsPIC) {
3031 HiOpFlags = PPCII::MO_PIC_HA_FLAG;
3032 LoOpFlags = PPCII::MO_PIC_LO_FLAG;
3033 }
3034}
3035
3036static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
3037 SelectionDAG &DAG) {
3038 SDLoc DL(HiPart);
3039 EVT PtrVT = HiPart.getValueType();
3040 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
3041
3042 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL, VT: PtrVT, N1: HiPart, N2: Zero);
3043 SDValue Lo = DAG.getNode(Opcode: PPCISD::Lo, DL, VT: PtrVT, N1: LoPart, N2: Zero);
3044
3045 // With PIC, the first instruction is actually "GR+hi(&G)".
3046 if (isPIC)
3047 Hi = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
3048 N1: DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL, VT: PtrVT), N2: Hi);
3049
3050 // Generate non-pic code that has direct accesses to the constant pool.
3051 // The address of the global is just (hi(&g)+lo(&g)).
3052 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Hi, N2: Lo);
3053}
3054
3055static void setUsesTOCBasePtr(MachineFunction &MF) {
3056 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3057 FuncInfo->setUsesTOCBasePtr();
3058}
3059
3060static void setUsesTOCBasePtr(SelectionDAG &DAG) {
3061 setUsesTOCBasePtr(DAG.getMachineFunction());
3062}
3063
3064SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
3065 SDValue GA) const {
3066 EVT VT = Subtarget.getScalarIntVT();
3067 SDValue Reg = Subtarget.isPPC64() ? DAG.getRegister(Reg: PPC::X2, VT)
3068 : Subtarget.isAIXABI()
3069 ? DAG.getRegister(Reg: PPC::R2, VT)
3070 : DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT);
3071 SDValue Ops[] = { GA, Reg };
3072 return DAG.getMemIntrinsicNode(
3073 Opcode: PPCISD::TOC_ENTRY, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT: VT,
3074 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: std::nullopt,
3075 Flags: MachineMemOperand::MOLoad);
3076}
3077
3078SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
3079 SelectionDAG &DAG) const {
3080 EVT PtrVT = Op.getValueType();
3081 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val&: Op);
3082 const Constant *C = CP->getConstVal();
3083
3084 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3085 // The actual address of the GlobalValue is stored in the TOC.
3086 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3087 if (Subtarget.isUsingPCRelativeCalls()) {
3088 SDLoc DL(CP);
3089 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3090 SDValue ConstPool = DAG.getTargetConstantPool(
3091 C, VT: Ty, Align: CP->getAlign(), Offset: CP->getOffset(), TargetFlags: PPCII::MO_PCREL_FLAG);
3092 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: ConstPool);
3093 }
3094 setUsesTOCBasePtr(DAG);
3095 SDValue GA = DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0);
3096 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3097 }
3098
3099 unsigned MOHiFlag, MOLoFlag;
3100 bool IsPIC = isPositionIndependent();
3101 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3102
3103 if (IsPIC && Subtarget.isSVR4ABI()) {
3104 SDValue GA =
3105 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: PPCII::MO_PIC_FLAG);
3106 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3107 }
3108
3109 SDValue CPIHi =
3110 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOHiFlag);
3111 SDValue CPILo =
3112 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOLoFlag);
3113 return LowerLabelRef(HiPart: CPIHi, LoPart: CPILo, isPIC: IsPIC, DAG);
3114}
3115
3116// For 64-bit PowerPC, prefer the more compact relative encodings.
3117// This trades 32 bits per jump table entry for one or two instructions
3118// on the jump site.
3119unsigned PPCTargetLowering::getJumpTableEncoding() const {
3120 if (isJumpTableRelative())
3121 return MachineJumpTableInfo::EK_LabelDifference32;
3122
3123 return TargetLowering::getJumpTableEncoding();
3124}
3125
3126bool PPCTargetLowering::isJumpTableRelative() const {
3127 if (UseAbsoluteJumpTables)
3128 return false;
3129 if (Subtarget.isPPC64() || Subtarget.isAIXABI())
3130 return true;
3131 return TargetLowering::isJumpTableRelative();
3132}
3133
3134SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3135 SelectionDAG &DAG) const {
3136 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3137 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3138
3139 switch (getTargetMachine().getCodeModel()) {
3140 case CodeModel::Small:
3141 case CodeModel::Medium:
3142 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3143 default:
3144 return DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: SDLoc(),
3145 VT: getPointerTy(DL: DAG.getDataLayout()));
3146 }
3147}
3148
3149const MCExpr *
3150PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
3151 unsigned JTI,
3152 MCContext &Ctx) const {
3153 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3154 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3155
3156 switch (getTargetMachine().getCodeModel()) {
3157 case CodeModel::Small:
3158 case CodeModel::Medium:
3159 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3160 default:
3161 return MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx);
3162 }
3163}
3164
3165SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
3166 EVT PtrVT = Op.getValueType();
3167 JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Op);
3168
3169 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3170 if (Subtarget.isUsingPCRelativeCalls()) {
3171 SDLoc DL(JT);
3172 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3173 SDValue GA =
3174 DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: Ty, TargetFlags: PPCII::MO_PCREL_FLAG);
3175 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3176 return MatAddr;
3177 }
3178
3179 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3180 // The actual address of the GlobalValue is stored in the TOC.
3181 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3182 setUsesTOCBasePtr(DAG);
3183 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
3184 return getTOCEntry(DAG, dl: SDLoc(JT), GA);
3185 }
3186
3187 unsigned MOHiFlag, MOLoFlag;
3188 bool IsPIC = isPositionIndependent();
3189 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3190
3191 if (IsPIC && Subtarget.isSVR4ABI()) {
3192 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT,
3193 TargetFlags: PPCII::MO_PIC_FLAG);
3194 return getTOCEntry(DAG, dl: SDLoc(GA), GA);
3195 }
3196
3197 SDValue JTIHi = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOHiFlag);
3198 SDValue JTILo = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOLoFlag);
3199 return LowerLabelRef(HiPart: JTIHi, LoPart: JTILo, isPIC: IsPIC, DAG);
3200}
3201
3202SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
3203 SelectionDAG &DAG) const {
3204 EVT PtrVT = Op.getValueType();
3205 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Val&: Op);
3206 const BlockAddress *BA = BASDN->getBlockAddress();
3207
3208 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3209 if (Subtarget.isUsingPCRelativeCalls()) {
3210 SDLoc DL(BASDN);
3211 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3212 SDValue GA = DAG.getTargetBlockAddress(BA, VT: Ty, Offset: BASDN->getOffset(),
3213 TargetFlags: PPCII::MO_PCREL_FLAG);
3214 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3215 return MatAddr;
3216 }
3217
3218 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3219 // The actual BlockAddress is stored in the TOC.
3220 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3221 setUsesTOCBasePtr(DAG);
3222 SDValue GA = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset());
3223 return getTOCEntry(DAG, dl: SDLoc(BASDN), GA);
3224 }
3225
3226 // 32-bit position-independent ELF stores the BlockAddress in the .got.
3227 if (Subtarget.is32BitELFABI() && isPositionIndependent())
3228 return getTOCEntry(
3229 DAG, dl: SDLoc(BASDN),
3230 GA: DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset()));
3231
3232 unsigned MOHiFlag, MOLoFlag;
3233 bool IsPIC = isPositionIndependent();
3234 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3235 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOHiFlag);
3236 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOLoFlag);
3237 return LowerLabelRef(HiPart: TgtBAHi, LoPart: TgtBALo, isPIC: IsPIC, DAG);
3238}
3239
3240SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
3241 SelectionDAG &DAG) const {
3242 if (Subtarget.isAIXABI())
3243 return LowerGlobalTLSAddressAIX(Op, DAG);
3244
3245 return LowerGlobalTLSAddressLinux(Op, DAG);
3246}
3247
3248/// updateForAIXShLibTLSModelOpt - Helper to initialize TLS model opt settings,
3249/// and then apply the update.
3250static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
3251 SelectionDAG &DAG,
3252 const TargetMachine &TM) {
3253 // Initialize TLS model opt setting lazily:
3254 // (1) Use initial-exec for single TLS var references within current function.
3255 // (2) Use local-dynamic for multiple TLS var references within current
3256 // function.
3257 PPCFunctionInfo *FuncInfo =
3258 DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
3259 if (!FuncInfo->isAIXFuncTLSModelOptInitDone()) {
3260 SmallPtrSet<const GlobalValue *, 8> TLSGV;
3261 // Iterate over all instructions within current function, collect all TLS
3262 // global variables (global variables taken as the first parameter to
3263 // Intrinsic::threadlocal_address).
3264 const Function &Func = DAG.getMachineFunction().getFunction();
3265 for (const BasicBlock &BB : Func)
3266 for (const Instruction &I : BB)
3267 if (I.getOpcode() == Instruction::Call)
3268 if (const CallInst *CI = dyn_cast<const CallInst>(Val: &I))
3269 if (Function *CF = CI->getCalledFunction())
3270 if (CF->isDeclaration() &&
3271 CF->getIntrinsicID() == Intrinsic::threadlocal_address)
3272 if (const GlobalValue *GV =
3273 dyn_cast<GlobalValue>(Val: I.getOperand(i: 0))) {
3274 TLSModel::Model GVModel = TM.getTLSModel(GV);
3275 if (GVModel == TLSModel::LocalDynamic)
3276 TLSGV.insert(Ptr: GV);
3277 }
3278
3279 unsigned TLSGVCnt = TLSGV.size();
3280 LLVM_DEBUG(dbgs() << format("LocalDynamic TLSGV count:%d\n", TLSGVCnt));
3281 if (TLSGVCnt <= PPCAIXTLSModelOptUseIEForLDLimit)
3282 FuncInfo->setAIXFuncUseTLSIEForLD();
3283 FuncInfo->setAIXFuncTLSModelOptInitDone();
3284 }
3285
3286 if (FuncInfo->isAIXFuncUseTLSIEForLD()) {
3287 LLVM_DEBUG(
3288 dbgs() << DAG.getMachineFunction().getName()
3289 << " function is using the TLS-IE model for TLS-LD access.\n");
3290 Model = TLSModel::InitialExec;
3291 }
3292}
3293
3294SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
3295 SelectionDAG &DAG) const {
3296 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3297
3298 if (DAG.getTarget().useEmulatedTLS())
3299 report_fatal_error(reason: "Emulated TLS is not yet supported on AIX");
3300
3301 SDLoc dl(GA);
3302 const GlobalValue *GV = GA->getGlobal();
3303 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3304 bool Is64Bit = Subtarget.isPPC64();
3305 TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
3306
3307 // Apply update to the TLS model.
3308 if (Subtarget.hasAIXShLibTLSModelOpt())
3309 updateForAIXShLibTLSModelOpt(Model, DAG, TM: getTargetMachine());
3310
3311 // TLS variables are accessed through TOC entries.
3312 // To support this, set the DAG to use the TOC base pointer.
3313 setUsesTOCBasePtr(DAG);
3314
3315 bool IsTLSLocalExecModel = Model == TLSModel::LocalExec;
3316
3317 if (IsTLSLocalExecModel || Model == TLSModel::InitialExec) {
3318 bool HasAIXSmallLocalExecTLS = Subtarget.hasAIXSmallLocalExecTLS();
3319 bool HasAIXSmallTLSGlobalAttr = false;
3320 SDValue VariableOffsetTGA =
3321 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TPREL_FLAG);
3322 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3323 SDValue TLSReg;
3324
3325 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(Val: GV))
3326 if (GVar->hasAttribute(Kind: "aix-small-tls"))
3327 HasAIXSmallTLSGlobalAttr = true;
3328
3329 if (Is64Bit) {
3330 // For local-exec and initial-exec on AIX (64-bit), the sequence generated
3331 // involves a load of the variable offset (from the TOC), followed by an
3332 // add of the loaded variable offset to R13 (the thread pointer).
3333 // This code sequence looks like:
3334 // ld reg1,var[TC](2)
3335 // add reg2, reg1, r13 // r13 contains the thread pointer
3336 TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3337
3338 // With the -maix-small-local-exec-tls option, or with the "aix-small-tls"
3339 // global variable attribute, produce a faster access sequence for
3340 // local-exec TLS variables where the offset from the TLS base is encoded
3341 // as an immediate operand.
3342 //
3343 // We only utilize the faster local-exec access sequence when the TLS
3344 // variable has a size within the policy limit. We treat types that are
3345 // not sized or are empty as being over the policy size limit.
3346 if ((HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr) &&
3347 IsTLSLocalExecModel) {
3348 Type *GVType = GV->getValueType();
3349 if (GVType->isSized() && !GVType->isEmptyTy() &&
3350 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3351 AIXSmallTlsPolicySizeLimit)
3352 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA, N2: TLSReg);
3353 }
3354 } else {
3355 // For local-exec and initial-exec on AIX (32-bit), the sequence generated
3356 // involves loading the variable offset from the TOC, generating a call to
3357 // .__get_tpointer to get the thread pointer (which will be in R3), and
3358 // adding the two together:
3359 // lwz reg1,var[TC](2)
3360 // bla .__get_tpointer
3361 // add reg2, reg1, r3
3362 TLSReg = DAG.getNode(Opcode: PPCISD::GET_TPOINTER, DL: dl, VT: PtrVT);
3363
3364 // We do not implement the 32-bit version of the faster access sequence
3365 // for local-exec that is controlled by the -maix-small-local-exec-tls
3366 // option, or the "aix-small-tls" global variable attribute.
3367 if (HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr)
3368 report_fatal_error(reason: "The small-local-exec TLS access sequence is "
3369 "currently only supported on AIX (64-bit mode).");
3370 }
3371 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: VariableOffset);
3372 }
3373
3374 if (Model == TLSModel::LocalDynamic) {
3375 bool HasAIXSmallLocalDynamicTLS = Subtarget.hasAIXSmallLocalDynamicTLS();
3376
3377 // We do not implement the 32-bit version of the faster access sequence
3378 // for local-dynamic that is controlled by -maix-small-local-dynamic-tls.
3379 if (!Is64Bit && HasAIXSmallLocalDynamicTLS)
3380 report_fatal_error(reason: "The small-local-dynamic TLS access sequence is "
3381 "currently only supported on AIX (64-bit mode).");
3382
3383 // For local-dynamic on AIX, we need to generate one TOC entry for each
3384 // variable offset, and a single module-handle TOC entry for the entire
3385 // file.
3386
3387 SDValue VariableOffsetTGA =
3388 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLD_FLAG);
3389 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3390
3391 Module *M = DAG.getMachineFunction().getFunction().getParent();
3392 GlobalVariable *TLSGV =
3393 dyn_cast_or_null<GlobalVariable>(Val: M->getOrInsertGlobal(
3394 Name: StringRef("_$TLSML"), Ty: PointerType::getUnqual(C&: *DAG.getContext())));
3395 TLSGV->setThreadLocalMode(GlobalVariable::LocalDynamicTLSModel);
3396 assert(TLSGV && "Not able to create GV for _$TLSML.");
3397 SDValue ModuleHandleTGA =
3398 DAG.getTargetGlobalAddress(GV: TLSGV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLDM_FLAG);
3399 SDValue ModuleHandleTOC = getTOCEntry(DAG, dl, GA: ModuleHandleTGA);
3400 SDValue ModuleHandle =
3401 DAG.getNode(Opcode: PPCISD::TLSLD_AIX, DL: dl, VT: PtrVT, Operand: ModuleHandleTOC);
3402
3403 // With the -maix-small-local-dynamic-tls option, produce a faster access
3404 // sequence for local-dynamic TLS variables where the offset from the
3405 // module-handle is encoded as an immediate operand.
3406 //
3407 // We only utilize the faster local-dynamic access sequence when the TLS
3408 // variable has a size within the policy limit. We treat types that are
3409 // not sized or are empty as being over the policy size limit.
3410 if (HasAIXSmallLocalDynamicTLS) {
3411 Type *GVType = GV->getValueType();
3412 if (GVType->isSized() && !GVType->isEmptyTy() &&
3413 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3414 AIXSmallTlsPolicySizeLimit)
3415 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA,
3416 N2: ModuleHandle);
3417 }
3418
3419 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ModuleHandle, N2: VariableOffset);
3420 }
3421
3422 // If Local- or Initial-exec or Local-dynamic is not possible or specified,
3423 // all GlobalTLSAddress nodes are lowered using the general-dynamic model. We
3424 // need to generate two TOC entries, one for the variable offset, one for the
3425 // region handle. The global address for the TOC entry of the region handle is
3426 // created with the MO_TLSGDM_FLAG flag and the global address for the TOC
3427 // entry of the variable offset is created with MO_TLSGD_FLAG.
3428 SDValue VariableOffsetTGA =
3429 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGD_FLAG);
3430 SDValue RegionHandleTGA =
3431 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGDM_FLAG);
3432 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3433 SDValue RegionHandle = getTOCEntry(DAG, dl, GA: RegionHandleTGA);
3434 return DAG.getNode(Opcode: PPCISD::TLSGD_AIX, DL: dl, VT: PtrVT, N1: VariableOffset,
3435 N2: RegionHandle);
3436}
3437
3438SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op,
3439 SelectionDAG &DAG) const {
3440 // FIXME: TLS addresses currently use medium model code sequences,
3441 // which is the most useful form. Eventually support for small and
3442 // large models could be added if users need it, at the cost of
3443 // additional complexity.
3444 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3445 if (DAG.getTarget().useEmulatedTLS())
3446 return LowerToTLSEmulatedModel(GA, DAG);
3447
3448 SDLoc dl(GA);
3449 const GlobalValue *GV = GA->getGlobal();
3450 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3451 bool is64bit = Subtarget.isPPC64();
3452 const Module *M = DAG.getMachineFunction().getFunction().getParent();
3453 PICLevel::Level picLevel = M->getPICLevel();
3454
3455 const TargetMachine &TM = getTargetMachine();
3456 TLSModel::Model Model = TM.getTLSModel(GV);
3457
3458 if (Model == TLSModel::LocalExec) {
3459 if (Subtarget.isUsingPCRelativeCalls()) {
3460 SDValue TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3461 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3462 TargetFlags: PPCII::MO_TPREL_PCREL_FLAG);
3463 SDValue MatAddr =
3464 DAG.getNode(Opcode: PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3465 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: MatAddr);
3466 }
3467
3468 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3469 TargetFlags: PPCII::MO_TPREL_HA);
3470 SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3471 TargetFlags: PPCII::MO_TPREL_LO);
3472 SDValue TLSReg = is64bit ? DAG.getRegister(Reg: PPC::X13, VT: MVT::i64)
3473 : DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
3474
3475 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL: dl, VT: PtrVT, N1: TGAHi, N2: TLSReg);
3476 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: TGALo, N2: Hi);
3477 }
3478
3479 if (Model == TLSModel::InitialExec) {
3480 bool IsPCRel = Subtarget.isUsingPCRelativeCalls();
3481 SDValue TGA = DAG.getTargetGlobalAddress(
3482 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0);
3483 SDValue TGATLS = DAG.getTargetGlobalAddress(
3484 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_TLS_PCREL_FLAG : PPCII::MO_TLS);
3485 SDValue TPOffset;
3486 if (IsPCRel) {
3487 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3488 TPOffset = DAG.getLoad(VT: MVT::i64, dl, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3489 PtrInfo: MachinePointerInfo());
3490 } else {
3491 SDValue GOTPtr;
3492 if (is64bit) {
3493 setUsesTOCBasePtr(DAG);
3494 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3495 GOTPtr =
3496 DAG.getNode(Opcode: PPCISD::ADDIS_GOT_TPREL_HA, DL: dl, VT: PtrVT, N1: GOTReg, N2: TGA);
3497 } else {
3498 if (!TM.isPositionIndependent())
3499 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_GOT, DL: dl, VT: PtrVT);
3500 else if (picLevel == PICLevel::SmallPIC)
3501 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3502 else
3503 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3504 }
3505 TPOffset = DAG.getNode(Opcode: PPCISD::LD_GOT_TPREL_L, DL: dl, VT: PtrVT, N1: TGA, N2: GOTPtr);
3506 }
3507 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TPOffset, N2: TGATLS);
3508 }
3509
3510 if (Model == TLSModel::GeneralDynamic) {
3511 if (Subtarget.isUsingPCRelativeCalls()) {
3512 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3513 TargetFlags: PPCII::MO_GOT_TLSGD_PCREL_FLAG);
3514 return DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3515 }
3516
3517 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3518 SDValue GOTPtr;
3519 if (is64bit) {
3520 setUsesTOCBasePtr(DAG);
3521 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3522 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSGD_HA, DL: dl, VT: PtrVT,
3523 N1: GOTReg, N2: TGA);
3524 } else {
3525 if (picLevel == PICLevel::SmallPIC)
3526 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3527 else
3528 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3529 }
3530 return DAG.getNode(Opcode: PPCISD::ADDI_TLSGD_L_ADDR, DL: dl, VT: PtrVT,
3531 N1: GOTPtr, N2: TGA, N3: TGA);
3532 }
3533
3534 if (Model == TLSModel::LocalDynamic) {
3535 if (Subtarget.isUsingPCRelativeCalls()) {
3536 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3537 TargetFlags: PPCII::MO_GOT_TLSLD_PCREL_FLAG);
3538 SDValue MatPCRel =
3539 DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3540 return DAG.getNode(Opcode: PPCISD::PADDI_DTPREL, DL: dl, VT: PtrVT, N1: MatPCRel, N2: TGA);
3541 }
3542
3543 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3544 SDValue GOTPtr;
3545 if (is64bit) {
3546 setUsesTOCBasePtr(DAG);
3547 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3548 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSLD_HA, DL: dl, VT: PtrVT,
3549 N1: GOTReg, N2: TGA);
3550 } else {
3551 if (picLevel == PICLevel::SmallPIC)
3552 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3553 else
3554 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3555 }
3556 SDValue TLSAddr = DAG.getNode(Opcode: PPCISD::ADDI_TLSLD_L_ADDR, DL: dl,
3557 VT: PtrVT, N1: GOTPtr, N2: TGA, N3: TGA);
3558 SDValue DtvOffsetHi = DAG.getNode(Opcode: PPCISD::ADDIS_DTPREL_HA, DL: dl,
3559 VT: PtrVT, N1: TLSAddr, N2: TGA);
3560 return DAG.getNode(Opcode: PPCISD::ADDI_DTPREL_L, DL: dl, VT: PtrVT, N1: DtvOffsetHi, N2: TGA);
3561 }
3562
3563 llvm_unreachable("Unknown TLS model!");
3564}
3565
3566SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
3567 SelectionDAG &DAG) const {
3568 EVT PtrVT = Op.getValueType();
3569 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Val&: Op);
3570 SDLoc DL(GSDN);
3571 const GlobalValue *GV = GSDN->getGlobal();
3572
3573 // 64-bit SVR4 ABI & AIX ABI code is always position-independent.
3574 // The actual address of the GlobalValue is stored in the TOC.
3575 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3576 if (Subtarget.isUsingPCRelativeCalls()) {
3577 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3578 if (isAccessedAsGotIndirect(N: Op)) {
3579 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3580 TargetFlags: PPCII::MO_GOT_PCREL_FLAG);
3581 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3582 SDValue Load = DAG.getLoad(VT: MVT::i64, dl: DL, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3583 PtrInfo: MachinePointerInfo());
3584 return Load;
3585 } else {
3586 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3587 TargetFlags: PPCII::MO_PCREL_FLAG);
3588 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3589 }
3590 }
3591 setUsesTOCBasePtr(DAG);
3592 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset());
3593 return getTOCEntry(DAG, dl: DL, GA);
3594 }
3595
3596 unsigned MOHiFlag, MOLoFlag;
3597 bool IsPIC = isPositionIndependent();
3598 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag, GV);
3599
3600 if (IsPIC && Subtarget.isSVR4ABI()) {
3601 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT,
3602 offset: GSDN->getOffset(),
3603 TargetFlags: PPCII::MO_PIC_FLAG);
3604 return getTOCEntry(DAG, dl: DL, GA);
3605 }
3606
3607 SDValue GAHi =
3608 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOHiFlag);
3609 SDValue GALo =
3610 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOLoFlag);
3611
3612 return LowerLabelRef(HiPart: GAHi, LoPart: GALo, isPIC: IsPIC, DAG);
3613}
3614
3615SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3616 bool IsStrict = Op->isStrictFPOpcode();
3617 const SDNodeFlags Flags = Op.getNode()->getFlags();
3618 ISD::CondCode CC =
3619 cast<CondCodeSDNode>(Val: Op.getOperand(i: IsStrict ? 3 : 2))->get();
3620 SDValue LHS = Op.getOperand(i: IsStrict ? 1 : 0);
3621 SDValue RHS = Op.getOperand(i: IsStrict ? 2 : 1);
3622 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
3623 EVT LHSVT = LHS.getValueType();
3624 SDLoc dl(Op);
3625
3626 // Soften the setcc with libcall if it is fp128 or it is SPE and fp32/fp64.
3627 if (LHSVT == MVT::f128 ||
3628 (Subtarget.hasSPE() && (LHSVT == MVT::f32 || LHSVT == MVT::f64) &&
3629 (!Flags.hasNoNaNs() || !Flags.hasNoInfs()))) {
3630 assert(!Subtarget.hasP9Vector() &&
3631 "SETCC for f128 is already legal under Power9!");
3632 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS, Chain,
3633 IsSignaling: Op->getOpcode() == ISD::STRICT_FSETCCS);
3634 if (RHS.getNode())
3635 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS,
3636 N3: DAG.getCondCode(Cond: CC));
3637 if (IsStrict)
3638 return DAG.getMergeValues(Ops: {LHS, Chain}, dl);
3639 return LHS;
3640 } else if (LHSVT == MVT::f32 || LHSVT == MVT::f64) {
3641 return Op;
3642 }
3643
3644 assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!");
3645
3646 if (Op.getValueType() == MVT::v2i64) {
3647 // When the operands themselves are v2i64 values, we need to do something
3648 // special because VSX has no underlying comparison operations for these.
3649 if (LHS.getValueType() == MVT::v2i64) {
3650 // Equality can be handled by casting to the legal type for Altivec
3651 // comparisons, everything else needs to be expanded.
3652 if (CC != ISD::SETEQ && CC != ISD::SETNE)
3653 return SDValue();
3654 SDValue SetCC32 = DAG.getSetCC(
3655 DL: dl, VT: MVT::v4i32, LHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: LHS),
3656 RHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: RHS), Cond: CC);
3657 int ShuffV[] = {1, 0, 3, 2};
3658 SDValue Shuff =
3659 DAG.getVectorShuffle(VT: MVT::v4i32, dl, N1: SetCC32, N2: SetCC32, Mask: ShuffV);
3660 return DAG.getBitcast(VT: MVT::v2i64,
3661 V: DAG.getNode(Opcode: CC == ISD::SETEQ ? ISD::AND : ISD::OR,
3662 DL: dl, VT: MVT::v4i32, N1: Shuff, N2: SetCC32));
3663 }
3664
3665 // We handle most of these in the usual way.
3666 return Op;
3667 }
3668
3669 // If we're comparing for equality to zero, expose the fact that this is
3670 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can
3671 // fold the new nodes.
3672 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG))
3673 return V;
3674
3675 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3676 // Leave comparisons against 0 and -1 alone for now, since they're usually
3677 // optimized. FIXME: revisit this when we can custom lower all setcc
3678 // optimizations.
3679 if (C->isAllOnes() || C->isZero())
3680 return SDValue();
3681 }
3682
3683 // If we have an integer seteq/setne, turn it into a compare against zero
3684 // by xor'ing the rhs with the lhs, which is faster than setting a
3685 // condition register, reading it back out, and masking the correct bit. The
3686 // normal approach here uses sub to do this instead of xor. Using xor exposes
3687 // the result to other bit-twiddling opportunities.
3688 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
3689 EVT VT = Op.getValueType();
3690 SDValue Sub = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSVT, N1: LHS, N2: RHS);
3691 return DAG.getSetCC(DL: dl, VT, LHS: Sub, RHS: DAG.getConstant(Val: 0, DL: dl, VT: LHSVT), Cond: CC);
3692 }
3693 return SDValue();
3694}
3695
3696SDValue PPCTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3697 const SDNodeFlags Flags = Op->getFlags();
3698 SDValue Chain = Op.getOperand(i: 0);
3699 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 1))->get();
3700 SDValue LHS = Op.getOperand(i: 2);
3701 SDValue RHS = Op.getOperand(i: 3);
3702 SDValue Dest = Op.getOperand(i: 4);
3703 EVT LHSVT = LHS.getValueType();
3704 SDLoc dl(Op);
3705
3706 assert(Subtarget.hasSPE() && "LowerBR_CC used only for targets with SPE");
3707
3708 if ((LHSVT == MVT::f32 || LHSVT == MVT::f64) && Flags.hasNoNaNs() &&
3709 Flags.hasNoInfs())
3710 return Op;
3711
3712 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
3713
3714 // If softenSetCCOperands returned a scalar, we need to compare the result
3715 // against zero to select between true and false values.
3716 if (!RHS) {
3717 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHSVT);
3718 CC = ISD::SETNE;
3719 }
3720
3721 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Op.getValueType(), N1: Chain,
3722 N2: DAG.getCondCode(Cond: CC), N3: LHS, N4: RHS, N5: Dest);
3723}
3724
3725SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3726 SDNode *Node = Op.getNode();
3727 EVT VT = Node->getValueType(ResNo: 0);
3728 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3729 SDValue InChain = Node->getOperand(Num: 0);
3730 SDValue VAListPtr = Node->getOperand(Num: 1);
3731 const Value *SV = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
3732 SDLoc dl(Node);
3733
3734 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only");
3735
3736 // gpr_index
3737 SDValue GprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3738 Ptr: VAListPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3739 InChain = GprIndex.getValue(R: 1);
3740
3741 if (VT == MVT::i64) {
3742 // Check if GprIndex is even
3743 SDValue GprAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: GprIndex,
3744 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3745 SDValue CC64 = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: GprAnd,
3746 RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), Cond: ISD::SETNE);
3747 SDValue GprIndexPlusOne = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: GprIndex,
3748 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3749 // Align GprIndex to be even if it isn't
3750 GprIndex = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC64, N2: GprIndexPlusOne,
3751 N3: GprIndex);
3752 }
3753
3754 // fpr index is 1 byte after gpr
3755 SDValue FprPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3756 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3757
3758 // fpr
3759 SDValue FprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3760 Ptr: FprPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3761 InChain = FprIndex.getValue(R: 1);
3762
3763 SDValue RegSaveAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3764 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
3765
3766 SDValue OverflowAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3767 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
3768
3769 // areas
3770 SDValue OverflowArea =
3771 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: OverflowAreaPtr, PtrInfo: MachinePointerInfo());
3772 InChain = OverflowArea.getValue(R: 1);
3773
3774 SDValue RegSaveArea =
3775 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: RegSaveAreaPtr, PtrInfo: MachinePointerInfo());
3776 InChain = RegSaveArea.getValue(R: 1);
3777
3778 // select overflow_area if index > 8
3779 SDValue CC = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: VT.isInteger() ? GprIndex : FprIndex,
3780 RHS: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32), Cond: ISD::SETLT);
3781
3782 // adjustment constant gpr_index * 4/8
3783 SDValue RegConstant = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32,
3784 N1: VT.isInteger() ? GprIndex : FprIndex,
3785 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8, DL: dl,
3786 VT: MVT::i32));
3787
3788 // OurReg = RegSaveArea + RegConstant
3789 SDValue OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: RegSaveArea,
3790 N2: RegConstant);
3791
3792 // Floating types are 32 bytes into RegSaveArea
3793 if (VT.isFloatingPoint())
3794 OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OurReg,
3795 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
3796
3797 // increase {f,g}pr_index by 1 (or 2 if VT is i64)
3798 SDValue IndexPlus1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32,
3799 N1: VT.isInteger() ? GprIndex : FprIndex,
3800 N2: DAG.getConstant(Val: VT == MVT::i64 ? 2 : 1, DL: dl,
3801 VT: MVT::i32));
3802
3803 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: IndexPlus1,
3804 Ptr: VT.isInteger() ? VAListPtr : FprPtr,
3805 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
3806
3807 // determine if we should load from reg_save_area or overflow_area
3808 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: PtrVT, N1: CC, N2: OurReg, N3: OverflowArea);
3809
3810 // increase overflow_area by 4/8 if gpr/fpr > 8
3811 SDValue OverflowAreaPlusN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OverflowArea,
3812 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8,
3813 DL: dl, VT: MVT::i32));
3814
3815 OverflowArea = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC, N2: OverflowArea,
3816 N3: OverflowAreaPlusN);
3817
3818 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: OverflowArea, Ptr: OverflowAreaPtr,
3819 PtrInfo: MachinePointerInfo(), SVT: MVT::i32);
3820
3821 return DAG.getLoad(VT, dl, Chain: InChain, Ptr: Result, PtrInfo: MachinePointerInfo());
3822}
3823
3824SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
3825 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only");
3826
3827 // We have to copy the entire va_list struct:
3828 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
3829 return DAG.getMemcpy(Chain: Op.getOperand(i: 0), dl: Op, Dst: Op.getOperand(i: 1), Src: Op.getOperand(i: 2),
3830 Size: DAG.getConstant(Val: 12, DL: SDLoc(Op), VT: MVT::i32), DstAlign: Align(8),
3831 SrcAlign: Align(8), isVol: false, AlwaysInline: true, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
3832 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
3833}
3834
3835SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
3836 SelectionDAG &DAG) const {
3837 return Op.getOperand(i: 0);
3838}
3839
3840SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
3841 MachineFunction &MF = DAG.getMachineFunction();
3842 PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>();
3843
3844 assert((Op.getOpcode() == ISD::INLINEASM ||
3845 Op.getOpcode() == ISD::INLINEASM_BR) &&
3846 "Expecting Inline ASM node.");
3847
3848 // If an LR store is already known to be required then there is not point in
3849 // checking this ASM as well.
3850 if (MFI.isLRStoreRequired())
3851 return Op;
3852
3853 // Inline ASM nodes have an optional last operand that is an incoming Flag of
3854 // type MVT::Glue. We want to ignore this last operand if that is the case.
3855 unsigned NumOps = Op.getNumOperands();
3856 if (Op.getOperand(i: NumOps - 1).getValueType() == MVT::Glue)
3857 --NumOps;
3858
3859 // Check all operands that may contain the LR.
3860 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
3861 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
3862 unsigned NumVals = Flags.getNumOperandRegisters();
3863 ++i; // Skip the ID value.
3864
3865 switch (Flags.getKind()) {
3866 default:
3867 llvm_unreachable("Bad flags!");
3868 case InlineAsm::Kind::RegUse:
3869 case InlineAsm::Kind::Imm:
3870 case InlineAsm::Kind::Mem:
3871 i += NumVals;
3872 break;
3873 case InlineAsm::Kind::Clobber:
3874 case InlineAsm::Kind::RegDef:
3875 case InlineAsm::Kind::RegDefEarlyClobber: {
3876 for (; NumVals; --NumVals, ++i) {
3877 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
3878 if (Reg != PPC::LR && Reg != PPC::LR8)
3879 continue;
3880 MFI.setLRStoreRequired();
3881 return Op;
3882 }
3883 break;
3884 }
3885 }
3886 }
3887
3888 return Op;
3889}
3890
3891SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
3892 SelectionDAG &DAG) const {
3893 SDValue Chain = Op.getOperand(i: 0);
3894 SDValue Trmp = Op.getOperand(i: 1); // trampoline
3895 SDValue FPtr = Op.getOperand(i: 2); // nested function
3896 SDValue Nest = Op.getOperand(i: 3); // 'nest' parameter value
3897 SDLoc dl(Op);
3898
3899 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3900
3901 if (Subtarget.isAIXABI()) {
3902 // On AIX we create a trampoline descriptor by combining the
3903 // entry point and TOC from the global descriptor (FPtr) with the
3904 // nest argument as the environment pointer.
3905 uint64_t PointerSize = Subtarget.isPPC64() ? 8 : 4;
3906 MaybeAlign PointerAlign(PointerSize);
3907 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
3908 ? (MachineMemOperand::MODereferenceable |
3909 MachineMemOperand::MOInvariant)
3910 : MachineMemOperand::MONone;
3911
3912 uint64_t TOCPointerOffset = 1 * PointerSize;
3913 uint64_t EnvPointerOffset = 2 * PointerSize;
3914 SDValue SDTOCPtrOffset = DAG.getConstant(Val: TOCPointerOffset, DL: dl, VT: PtrVT);
3915 SDValue SDEnvPtrOffset = DAG.getConstant(Val: EnvPointerOffset, DL: dl, VT: PtrVT);
3916
3917 const Value *TrampolineAddr =
3918 cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
3919 const Function *Func =
3920 cast<Function>(Val: cast<SrcValueSDNode>(Val: Op.getOperand(i: 5))->getValue());
3921
3922 SDValue OutChains[3];
3923
3924 // Copy the entry point address from the global descriptor to the
3925 // trampoline buffer.
3926 SDValue LoadEntryPoint =
3927 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: FPtr, PtrInfo: MachinePointerInfo(Func, 0),
3928 Alignment: PointerAlign, MMOFlags);
3929 SDValue EPLoadChain = LoadEntryPoint.getValue(R: 1);
3930 OutChains[0] = DAG.getStore(Chain: EPLoadChain, dl, Val: LoadEntryPoint, Ptr: Trmp,
3931 PtrInfo: MachinePointerInfo(TrampolineAddr, 0));
3932
3933 // Copy the TOC pointer from the global descriptor to the trampoline
3934 // buffer.
3935 SDValue TOCFromDescriptorPtr =
3936 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FPtr, N2: SDTOCPtrOffset);
3937 SDValue TOCReg = DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: TOCFromDescriptorPtr,
3938 PtrInfo: MachinePointerInfo(Func, TOCPointerOffset),
3939 Alignment: PointerAlign, MMOFlags);
3940 SDValue TrampolineTOCPointer =
3941 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDTOCPtrOffset);
3942 SDValue TOCLoadChain = TOCReg.getValue(R: 1);
3943 OutChains[1] =
3944 DAG.getStore(Chain: TOCLoadChain, dl, Val: TOCReg, Ptr: TrampolineTOCPointer,
3945 PtrInfo: MachinePointerInfo(TrampolineAddr, TOCPointerOffset));
3946
3947 // Store the nest argument into the environment pointer in the trampoline
3948 // buffer.
3949 SDValue EnvPointer = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDEnvPtrOffset);
3950 OutChains[2] =
3951 DAG.getStore(Chain, dl, Val: Nest, Ptr: EnvPointer,
3952 PtrInfo: MachinePointerInfo(TrampolineAddr, EnvPointerOffset));
3953
3954 SDValue TokenFactor =
3955 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
3956 return TokenFactor;
3957 }
3958
3959 bool isPPC64 = (PtrVT == MVT::i64);
3960 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(C&: *DAG.getContext());
3961
3962 TargetLowering::ArgListTy Args;
3963 Args.emplace_back(args&: Trmp, args&: IntPtrTy);
3964 // TrampSize == (isPPC64 ? 48 : 40);
3965 Args.emplace_back(
3966 args: DAG.getConstant(Val: isPPC64 ? 48 : 40, DL: dl, VT: Subtarget.getScalarIntVT()),
3967 args&: IntPtrTy);
3968 Args.emplace_back(args&: FPtr, args&: IntPtrTy);
3969 Args.emplace_back(args&: Nest, args&: IntPtrTy);
3970
3971 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
3972 TargetLowering::CallLoweringInfo CLI(DAG);
3973 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3974 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
3975 Target: DAG.getExternalSymbol(Sym: "__trampoline_setup", VT: PtrVT), ArgsList: std::move(Args));
3976
3977 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3978 return CallResult.second;
3979}
3980
3981SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3982 MachineFunction &MF = DAG.getMachineFunction();
3983 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3984 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
3985
3986 SDLoc dl(Op);
3987
3988 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) {
3989 // vastart just stores the address of the VarArgsFrameIndex slot into the
3990 // memory location argument.
3991 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
3992 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
3993 return DAG.getStore(Chain: Op.getOperand(i: 0), dl, Val: FR, Ptr: Op.getOperand(i: 1),
3994 PtrInfo: MachinePointerInfo(SV));
3995 }
3996
3997 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct.
3998 // We suppose the given va_list is already allocated.
3999 //
4000 // typedef struct {
4001 // char gpr; /* index into the array of 8 GPRs
4002 // * stored in the register save area
4003 // * gpr=0 corresponds to r3,
4004 // * gpr=1 to r4, etc.
4005 // */
4006 // char fpr; /* index into the array of 8 FPRs
4007 // * stored in the register save area
4008 // * fpr=0 corresponds to f1,
4009 // * fpr=1 to f2, etc.
4010 // */
4011 // char *overflow_arg_area;
4012 // /* location on stack that holds
4013 // * the next overflow argument
4014 // */
4015 // char *reg_save_area;
4016 // /* where r3:r10 and f1:f8 (if saved)
4017 // * are stored
4018 // */
4019 // } va_list[1];
4020
4021 SDValue ArgGPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumGPR(), DL: dl, VT: MVT::i32);
4022 SDValue ArgFPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumFPR(), DL: dl, VT: MVT::i32);
4023 SDValue StackOffsetFI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsStackOffset(),
4024 VT: PtrVT);
4025 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4026 VT: PtrVT);
4027
4028 uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
4029 SDValue ConstFrameOffset = DAG.getConstant(Val: FrameOffset, DL: dl, VT: PtrVT);
4030
4031 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
4032 SDValue ConstStackOffset = DAG.getConstant(Val: StackOffset, DL: dl, VT: PtrVT);
4033
4034 uint64_t FPROffset = 1;
4035 SDValue ConstFPROffset = DAG.getConstant(Val: FPROffset, DL: dl, VT: PtrVT);
4036
4037 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4038
4039 // Store first byte : number of int regs
4040 SDValue firstStore =
4041 DAG.getTruncStore(Chain: Op.getOperand(i: 0), dl, Val: ArgGPR, Ptr: Op.getOperand(i: 1),
4042 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
4043 uint64_t nextOffset = FPROffset;
4044 SDValue nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Op.getOperand(i: 1),
4045 N2: ConstFPROffset);
4046
4047 // Store second byte : number of float regs
4048 SDValue secondStore =
4049 DAG.getTruncStore(Chain: firstStore, dl, Val: ArgFPR, Ptr: nextPtr,
4050 PtrInfo: MachinePointerInfo(SV, nextOffset), SVT: MVT::i8);
4051 nextOffset += StackOffset;
4052 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstStackOffset);
4053
4054 // Store second word : arguments given on stack
4055 SDValue thirdStore = DAG.getStore(Chain: secondStore, dl, Val: StackOffsetFI, Ptr: nextPtr,
4056 PtrInfo: MachinePointerInfo(SV, nextOffset));
4057 nextOffset += FrameOffset;
4058 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstFrameOffset);
4059
4060 // Store third word : arguments given in registers
4061 return DAG.getStore(Chain: thirdStore, dl, Val: FR, Ptr: nextPtr,
4062 PtrInfo: MachinePointerInfo(SV, nextOffset));
4063}
4064
4065/// FPR - The set of FP registers that should be allocated for arguments
4066/// on Darwin and AIX.
4067static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5,
4068 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10,
4069 PPC::F11, PPC::F12, PPC::F13};
4070
4071/// CalculateStackSlotSize - Calculates the size reserved for this argument on
4072/// the stack.
4073static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
4074 unsigned PtrByteSize) {
4075 unsigned ArgSize = ArgVT.getStoreSize();
4076 if (Flags.isByVal())
4077 ArgSize = Flags.getByValSize();
4078
4079 // Round up to multiples of the pointer size, except for array members,
4080 // which are always packed.
4081 if (!Flags.isInConsecutiveRegs())
4082 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4083
4084 return ArgSize;
4085}
4086
4087/// CalculateStackSlotAlignment - Calculates the alignment of this argument
4088/// on the stack.
4089static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
4090 ISD::ArgFlagsTy Flags,
4091 unsigned PtrByteSize) {
4092 Align Alignment(PtrByteSize);
4093
4094 // Altivec parameters are padded to a 16 byte boundary.
4095 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4096 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4097 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4098 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4099 Alignment = Align(16);
4100
4101 // ByVal parameters are aligned as requested.
4102 if (Flags.isByVal()) {
4103 auto BVAlign = Flags.getNonZeroByValAlign();
4104 if (BVAlign > PtrByteSize) {
4105 if (BVAlign.value() % PtrByteSize != 0)
4106 llvm_unreachable(
4107 "ByVal alignment is not a multiple of the pointer size");
4108
4109 Alignment = BVAlign;
4110 }
4111 }
4112
4113 // Array members are always packed to their original alignment.
4114 if (Flags.isInConsecutiveRegs()) {
4115 // If the array member was split into multiple registers, the first
4116 // needs to be aligned to the size of the full type. (Except for
4117 // ppcf128, which is only aligned as its f64 components.)
4118 if (Flags.isSplit() && OrigVT != MVT::ppcf128)
4119 Alignment = Align(OrigVT.getStoreSize());
4120 else
4121 Alignment = Align(ArgVT.getStoreSize());
4122 }
4123
4124 return Alignment;
4125}
4126
4127/// CalculateStackSlotUsed - Return whether this argument will use its
4128/// stack slot (instead of being passed in registers). ArgOffset,
4129/// AvailableFPRs, and AvailableVRs must hold the current argument
4130/// position, and will be updated to account for this argument.
4131static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
4132 unsigned PtrByteSize, unsigned LinkageSize,
4133 unsigned ParamAreaSize, unsigned &ArgOffset,
4134 unsigned &AvailableFPRs,
4135 unsigned &AvailableVRs) {
4136 bool UseMemory = false;
4137
4138 // Respect alignment of argument on the stack.
4139 Align Alignment =
4140 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
4141 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4142 // If there's no space left in the argument save area, we must
4143 // use memory (this check also catches zero-sized arguments).
4144 if (ArgOffset >= LinkageSize + ParamAreaSize)
4145 UseMemory = true;
4146
4147 // Allocate argument on the stack.
4148 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
4149 if (Flags.isInConsecutiveRegsLast())
4150 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4151 // If we overran the argument save area, we must use memory
4152 // (this check catches arguments passed partially in memory)
4153 if (ArgOffset > LinkageSize + ParamAreaSize)
4154 UseMemory = true;
4155
4156 // However, if the argument is actually passed in an FPR or a VR,
4157 // we don't use memory after all.
4158 if (!Flags.isByVal()) {
4159 if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
4160 if (AvailableFPRs > 0) {
4161 --AvailableFPRs;
4162 return false;
4163 }
4164 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4165 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4166 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4167 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4168 if (AvailableVRs > 0) {
4169 --AvailableVRs;
4170 return false;
4171 }
4172 }
4173
4174 return UseMemory;
4175}
4176
4177/// EnsureStackAlignment - Round stack frame size up from NumBytes to
4178/// ensure minimum alignment required for target.
4179static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
4180 unsigned NumBytes) {
4181 return alignTo(Size: NumBytes, A: Lowering->getStackAlign());
4182}
4183
4184SDValue PPCTargetLowering::LowerFormalArguments(
4185 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4186 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4187 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4188 if (Subtarget.isAIXABI())
4189 return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG,
4190 InVals);
4191 if (Subtarget.is64BitELFABI())
4192 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4193 InVals);
4194 assert(Subtarget.is32BitELFABI());
4195 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4196 InVals);
4197}
4198
4199SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
4200 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4201 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4202 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4203
4204 // 32-bit SVR4 ABI Stack Frame Layout:
4205 // +-----------------------------------+
4206 // +--> | Back chain |
4207 // | +-----------------------------------+
4208 // | | Floating-point register save area |
4209 // | +-----------------------------------+
4210 // | | General register save area |
4211 // | +-----------------------------------+
4212 // | | CR save word |
4213 // | +-----------------------------------+
4214 // | | VRSAVE save word |
4215 // | +-----------------------------------+
4216 // | | Alignment padding |
4217 // | +-----------------------------------+
4218 // | | Vector register save area |
4219 // | +-----------------------------------+
4220 // | | Local variable space |
4221 // | +-----------------------------------+
4222 // | | Parameter list area |
4223 // | +-----------------------------------+
4224 // | | LR save word |
4225 // | +-----------------------------------+
4226 // SP--> +--- | Back chain |
4227 // +-----------------------------------+
4228 //
4229 // Specifications:
4230 // System V Application Binary Interface PowerPC Processor Supplement
4231 // AltiVec Technology Programming Interface Manual
4232
4233 MachineFunction &MF = DAG.getMachineFunction();
4234 MachineFrameInfo &MFI = MF.getFrameInfo();
4235 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4236
4237 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4238 // Potential tail calls could cause overwriting of argument stack slots.
4239 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4240 (CallConv == CallingConv::Fast));
4241 const Align PtrAlign(4);
4242
4243 // Assign locations to all of the incoming arguments.
4244 SmallVector<CCValAssign, 16> ArgLocs;
4245 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4246 *DAG.getContext());
4247
4248 // Reserve space for the linkage area on the stack.
4249 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4250 CCInfo.AllocateStack(Size: LinkageSize, Alignment: PtrAlign);
4251 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4);
4252
4253 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4254 CCValAssign &VA = ArgLocs[i];
4255
4256 // Arguments stored in registers.
4257 if (VA.isRegLoc()) {
4258 const TargetRegisterClass *RC;
4259 EVT ValVT = VA.getValVT();
4260
4261 switch (ValVT.getSimpleVT().SimpleTy) {
4262 default:
4263 llvm_unreachable("ValVT not supported by formal arguments Lowering");
4264 case MVT::i1:
4265 case MVT::i32:
4266 RC = &PPC::GPRCRegClass;
4267 break;
4268 case MVT::f32:
4269 if (Subtarget.hasP8Vector())
4270 RC = &PPC::VSSRCRegClass;
4271 else if (Subtarget.hasSPE())
4272 RC = &PPC::GPRCRegClass;
4273 else
4274 RC = &PPC::F4RCRegClass;
4275 break;
4276 case MVT::f64:
4277 if (Subtarget.hasVSX())
4278 RC = &PPC::VSFRCRegClass;
4279 else if (Subtarget.hasSPE())
4280 // SPE passes doubles in GPR pairs.
4281 RC = &PPC::GPRCRegClass;
4282 else
4283 RC = &PPC::F8RCRegClass;
4284 break;
4285 case MVT::v16i8:
4286 case MVT::v8i16:
4287 case MVT::v4i32:
4288 case MVT::v4f32:
4289 case MVT::v2f64:
4290 case MVT::v2i64:
4291 case MVT::f128:
4292 RC = &PPC::VRRCRegClass;
4293 break;
4294 }
4295
4296 SDValue ArgValue;
4297 // Transform the arguments stored in physical registers into
4298 // virtual ones.
4299 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) {
4300 assert(i + 1 < e && "No second half of double precision argument");
4301 Register RegLo = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4302 Register RegHi = MF.addLiveIn(PReg: ArgLocs[++i].getLocReg(), RC);
4303 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, Reg: RegLo, VT: MVT::i32);
4304 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, Reg: RegHi, VT: MVT::i32);
4305 if (!Subtarget.isLittleEndian())
4306 std::swap (a&: ArgValueLo, b&: ArgValueHi);
4307 ArgValue = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: ArgValueLo,
4308 N2: ArgValueHi);
4309 } else {
4310 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4311 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
4312 VT: ValVT == MVT::i1 ? MVT::i32 : ValVT);
4313 if (ValVT == MVT::i1)
4314 ArgValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: ArgValue);
4315 }
4316
4317 InVals.push_back(Elt: ArgValue);
4318 } else {
4319 // Argument stored in memory.
4320 assert(VA.isMemLoc());
4321
4322 // Get the extended size of the argument type in stack
4323 unsigned ArgSize = VA.getLocVT().getStoreSize();
4324 // Get the actual size of the argument type
4325 unsigned ObjSize = VA.getValVT().getStoreSize();
4326 unsigned ArgOffset = VA.getLocMemOffset();
4327 // Stack objects in PPC32 are right justified.
4328 ArgOffset += ArgSize - ObjSize;
4329 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: isImmutable);
4330
4331 // Create load nodes to retrieve arguments from the stack.
4332 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4333 InVals.push_back(
4334 Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo()));
4335 }
4336 }
4337
4338 // Assign locations to all of the incoming aggregate by value arguments.
4339 // Aggregates passed by value are stored in the local variable space of the
4340 // caller's stack frame, right above the parameter list area.
4341 SmallVector<CCValAssign, 16> ByValArgLocs;
4342 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
4343 ByValArgLocs, *DAG.getContext());
4344
4345 // Reserve stack space for the allocations in CCInfo.
4346 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
4347
4348 CCByValInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4_ByVal);
4349
4350 // Area that is at least reserved in the caller of this function.
4351 unsigned MinReservedArea = CCByValInfo.getStackSize();
4352 MinReservedArea = std::max(a: MinReservedArea, b: LinkageSize);
4353
4354 // Set the size that is at least reserved in caller of this function. Tail
4355 // call optimized function's reserved stack space needs to be aligned so that
4356 // taking the difference between two stack areas will result in an aligned
4357 // stack.
4358 MinReservedArea =
4359 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4360 FuncInfo->setMinReservedArea(MinReservedArea);
4361
4362 SmallVector<SDValue, 8> MemOps;
4363
4364 // If the function takes variable number of arguments, make a frame index for
4365 // the start of the first vararg value... for expansion of llvm.va_start.
4366 if (isVarArg) {
4367 static const MCPhysReg GPArgRegs[] = {
4368 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
4369 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
4370 };
4371 const unsigned NumGPArgRegs = std::size(GPArgRegs);
4372
4373 static const MCPhysReg FPArgRegs[] = {
4374 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
4375 PPC::F8
4376 };
4377 unsigned NumFPArgRegs = std::size(FPArgRegs);
4378
4379 if (useSoftFloat() || hasSPE())
4380 NumFPArgRegs = 0;
4381
4382 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(Regs: GPArgRegs));
4383 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(Regs: FPArgRegs));
4384
4385 // Make room for NumGPArgRegs and NumFPArgRegs.
4386 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 +
4387 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8;
4388
4389 FuncInfo->setVarArgsStackOffset(MFI.CreateFixedObject(
4390 Size: PtrVT.getSizeInBits() / 8, SPOffset: CCInfo.getStackSize(), IsImmutable: true));
4391
4392 FuncInfo->setVarArgsFrameIndex(
4393 MFI.CreateStackObject(Size: Depth, Alignment: Align(8), isSpillSlot: false));
4394 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4395
4396 // The fixed integer arguments of a variadic function are stored to the
4397 // VarArgsFrameIndex on the stack so that they may be loaded by
4398 // dereferencing the result of va_next.
4399 for (MCPhysReg GPArgReg : GPArgRegs) {
4400 // Get an existing live-in vreg, or add a new one.
4401 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: GPArgReg);
4402 if (!VReg)
4403 VReg = MF.addLiveIn(PReg: GPArgReg, RC: &PPC::GPRCRegClass);
4404
4405 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4406 SDValue Store =
4407 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4408 MemOps.push_back(Elt: Store);
4409 // Increment the address by four for the next argument to store
4410 SDValue PtrOff = DAG.getConstant(Val: PtrVT.getSizeInBits()/8, DL: dl, VT: PtrVT);
4411 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4412 }
4413
4414 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6
4415 // is set.
4416 // The double arguments are stored to the VarArgsFrameIndex
4417 // on the stack.
4418 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) {
4419 // Get an existing live-in vreg, or add a new one.
4420 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: FPArgRegs[FPRIndex]);
4421 if (!VReg)
4422 VReg = MF.addLiveIn(PReg: FPArgRegs[FPRIndex], RC: &PPC::F8RCRegClass);
4423
4424 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::f64);
4425 SDValue Store =
4426 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4427 MemOps.push_back(Elt: Store);
4428 // Increment the address by eight for the next argument to store
4429 SDValue PtrOff = DAG.getConstant(Val: MVT(MVT::f64).getSizeInBits()/8, DL: dl,
4430 VT: PtrVT);
4431 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4432 }
4433 }
4434
4435 if (!MemOps.empty())
4436 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4437
4438 return Chain;
4439}
4440
4441// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4442// value to MVT::i64 and then truncate to the correct register size.
4443SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags,
4444 EVT ObjectVT, SelectionDAG &DAG,
4445 SDValue ArgVal,
4446 const SDLoc &dl) const {
4447 if (Flags.isSExt())
4448 ArgVal = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: MVT::i64, N1: ArgVal,
4449 N2: DAG.getValueType(ObjectVT));
4450 else if (Flags.isZExt())
4451 ArgVal = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i64, N1: ArgVal,
4452 N2: DAG.getValueType(ObjectVT));
4453
4454 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ObjectVT, Operand: ArgVal);
4455}
4456
4457SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
4458 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4459 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4460 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4461 // TODO: add description of PPC stack frame format, or at least some docs.
4462 //
4463 bool isELFv2ABI = Subtarget.isELFv2ABI();
4464 bool isLittleEndian = Subtarget.isLittleEndian();
4465 MachineFunction &MF = DAG.getMachineFunction();
4466 MachineFrameInfo &MFI = MF.getFrameInfo();
4467 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4468
4469 assert(!(CallConv == CallingConv::Fast && isVarArg) &&
4470 "fastcc not supported on varargs functions");
4471
4472 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4473 // Potential tail calls could cause overwriting of argument stack slots.
4474 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4475 (CallConv == CallingConv::Fast));
4476 unsigned PtrByteSize = 8;
4477 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4478
4479 static const MCPhysReg GPR[] = {
4480 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4481 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4482 };
4483 static const MCPhysReg VR[] = {
4484 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4485 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4486 };
4487
4488 const unsigned Num_GPR_Regs = std::size(GPR);
4489 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
4490 const unsigned Num_VR_Regs = std::size(VR);
4491
4492 // Do a first pass over the arguments to determine whether the ABI
4493 // guarantees that our caller has allocated the parameter save area
4494 // on its stack frame. In the ELFv1 ABI, this is always the case;
4495 // in the ELFv2 ABI, it is true if this is a vararg function or if
4496 // any parameter is located in a stack slot.
4497
4498 bool HasParameterArea = !isELFv2ABI || isVarArg;
4499 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize;
4500 unsigned NumBytes = LinkageSize;
4501 unsigned AvailableFPRs = Num_FPR_Regs;
4502 unsigned AvailableVRs = Num_VR_Regs;
4503 for (const ISD::InputArg &In : Ins) {
4504 if (In.Flags.isNest())
4505 continue;
4506
4507 if (CalculateStackSlotUsed(ArgVT: In.VT, OrigVT: In.ArgVT, Flags: In.Flags, PtrByteSize,
4508 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4509 AvailableFPRs, AvailableVRs))
4510 HasParameterArea = true;
4511 }
4512
4513 // Add DAG nodes to load the arguments or copy them out of registers. On
4514 // entry to a function on PPC, the arguments start after the linkage area,
4515 // although the first ones are often in registers.
4516
4517 unsigned ArgOffset = LinkageSize;
4518 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
4519 SmallVector<SDValue, 8> MemOps;
4520 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
4521 unsigned CurArgIdx = 0;
4522 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
4523 SDValue ArgVal;
4524 bool needsLoad = false;
4525 EVT ObjectVT = Ins[ArgNo].VT;
4526 EVT OrigVT = Ins[ArgNo].ArgVT;
4527 unsigned ObjSize = ObjectVT.getStoreSize();
4528 unsigned ArgSize = ObjSize;
4529 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
4530 if (Ins[ArgNo].isOrigArg()) {
4531 std::advance(i&: FuncArg, n: Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
4532 CurArgIdx = Ins[ArgNo].getOrigArgIndex();
4533 }
4534 // We re-align the argument offset for each argument, except when using the
4535 // fast calling convention, when we need to make sure we do that only when
4536 // we'll actually use a stack slot.
4537 unsigned CurArgOffset;
4538 Align Alignment;
4539 auto ComputeArgOffset = [&]() {
4540 /* Respect alignment of argument on the stack. */
4541 Alignment =
4542 CalculateStackSlotAlignment(ArgVT: ObjectVT, OrigVT, Flags, PtrByteSize);
4543 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4544 CurArgOffset = ArgOffset;
4545 };
4546
4547 if (CallConv != CallingConv::Fast) {
4548 ComputeArgOffset();
4549
4550 /* Compute GPR index associated with argument offset. */
4551 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4552 GPR_idx = std::min(a: GPR_idx, b: Num_GPR_Regs);
4553 }
4554
4555 // FIXME the codegen can be much improved in some cases.
4556 // We do not have to keep everything in memory.
4557 if (Flags.isByVal()) {
4558 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit");
4559
4560 if (CallConv == CallingConv::Fast)
4561 ComputeArgOffset();
4562
4563 // ObjSize is the true size, ArgSize rounded up to multiple of registers.
4564 ObjSize = Flags.getByValSize();
4565 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4566 // Empty aggregate parameters do not take up registers. Examples:
4567 // struct { } a;
4568 // union { } b;
4569 // int c[0];
4570 // etc. However, we have to provide a place-holder in InVals, so
4571 // pretend we have an 8-byte item at the current address for that
4572 // purpose.
4573 if (!ObjSize) {
4574 int FI = MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: ArgOffset, IsImmutable: true);
4575 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4576 InVals.push_back(Elt: FIN);
4577 continue;
4578 }
4579
4580 // Create a stack object covering all stack doublewords occupied
4581 // by the argument. If the argument is (fully or partially) on
4582 // the stack, or if the argument is fully in registers but the
4583 // caller has allocated the parameter save anyway, we can refer
4584 // directly to the caller's stack frame. Otherwise, create a
4585 // local copy in our own frame.
4586 int FI;
4587 if (HasParameterArea ||
4588 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize)
4589 FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: false, isAliased: true);
4590 else
4591 FI = MFI.CreateStackObject(Size: ArgSize, Alignment, isSpillSlot: false);
4592 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4593
4594 // Handle aggregates smaller than 8 bytes.
4595 if (ObjSize < PtrByteSize) {
4596 // The value of the object is its address, which differs from the
4597 // address of the enclosing doubleword on big-endian systems.
4598 SDValue Arg = FIN;
4599 if (!isLittleEndian) {
4600 SDValue ArgOff = DAG.getConstant(Val: PtrByteSize - ObjSize, DL: dl, VT: PtrVT);
4601 Arg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ArgOff.getValueType(), N1: Arg, N2: ArgOff);
4602 }
4603 InVals.push_back(Elt: Arg);
4604
4605 if (GPR_idx != Num_GPR_Regs) {
4606 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4607 FuncInfo->addLiveInAttr(VReg, Flags);
4608 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4609 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ObjSize * 8);
4610 SDValue Store =
4611 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Arg,
4612 PtrInfo: MachinePointerInfo(&*FuncArg), SVT: ObjType);
4613 MemOps.push_back(Elt: Store);
4614 }
4615 // Whether we copied from a register or not, advance the offset
4616 // into the parameter save area by a full doubleword.
4617 ArgOffset += PtrByteSize;
4618 continue;
4619 }
4620
4621 // The value of the object is its address, which is the address of
4622 // its first stack doubleword.
4623 InVals.push_back(Elt: FIN);
4624
4625 // Store whatever pieces of the object are in registers to memory.
4626 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
4627 if (GPR_idx == Num_GPR_Regs)
4628 break;
4629
4630 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4631 FuncInfo->addLiveInAttr(VReg, Flags);
4632 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4633 SDValue Addr = FIN;
4634 if (j) {
4635 SDValue Off = DAG.getConstant(Val: j, DL: dl, VT: PtrVT);
4636 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Off.getValueType(), N1: Addr, N2: Off);
4637 }
4638 unsigned StoreSizeInBits = std::min(a: PtrByteSize, b: (ObjSize - j)) * 8;
4639 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreSizeInBits);
4640 SDValue Store =
4641 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Addr,
4642 PtrInfo: MachinePointerInfo(&*FuncArg, j), SVT: ObjType);
4643 MemOps.push_back(Elt: Store);
4644 ++GPR_idx;
4645 }
4646 ArgOffset += ArgSize;
4647 continue;
4648 }
4649
4650 switch (ObjectVT.getSimpleVT().SimpleTy) {
4651 default: llvm_unreachable("Unhandled argument type!");
4652 case MVT::i1:
4653 case MVT::i32:
4654 case MVT::i64:
4655 if (Flags.isNest()) {
4656 // The 'nest' parameter, if any, is passed in R11.
4657 Register VReg = MF.addLiveIn(PReg: PPC::X11, RC: &PPC::G8RCRegClass);
4658 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4659
4660 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4661 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4662
4663 break;
4664 }
4665
4666 // These can be scalar arguments or elements of an integer array type
4667 // passed directly. Clang may use those instead of "byval" aggregate
4668 // types to avoid forcing arguments to memory unnecessarily.
4669 if (GPR_idx != Num_GPR_Regs) {
4670 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4671 FuncInfo->addLiveInAttr(VReg, Flags);
4672 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4673
4674 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4675 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4676 // value to MVT::i64 and then truncate to the correct register size.
4677 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4678 } else {
4679 if (CallConv == CallingConv::Fast)
4680 ComputeArgOffset();
4681
4682 needsLoad = true;
4683 ArgSize = PtrByteSize;
4684 }
4685 if (CallConv != CallingConv::Fast || needsLoad)
4686 ArgOffset += 8;
4687 break;
4688
4689 case MVT::f32:
4690 case MVT::f64:
4691 // These can be scalar arguments or elements of a float array type
4692 // passed directly. The latter are used to implement ELFv2 homogenous
4693 // float aggregates.
4694 if (FPR_idx != Num_FPR_Regs) {
4695 unsigned VReg;
4696
4697 if (ObjectVT == MVT::f32)
4698 VReg = MF.addLiveIn(PReg: FPR[FPR_idx],
4699 RC: Subtarget.hasP8Vector()
4700 ? &PPC::VSSRCRegClass
4701 : &PPC::F4RCRegClass);
4702 else
4703 VReg = MF.addLiveIn(PReg: FPR[FPR_idx], RC: Subtarget.hasVSX()
4704 ? &PPC::VSFRCRegClass
4705 : &PPC::F8RCRegClass);
4706
4707 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4708 ++FPR_idx;
4709 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) {
4710 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
4711 // once we support fp <-> gpr moves.
4712
4713 // This can only ever happen in the presence of f32 array types,
4714 // since otherwise we never run out of FPRs before running out
4715 // of GPRs.
4716 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4717 FuncInfo->addLiveInAttr(VReg, Flags);
4718 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4719
4720 if (ObjectVT == MVT::f32) {
4721 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
4722 ArgVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgVal,
4723 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
4724 ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: ArgVal);
4725 }
4726
4727 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ObjectVT, Operand: ArgVal);
4728 } else {
4729 if (CallConv == CallingConv::Fast)
4730 ComputeArgOffset();
4731
4732 needsLoad = true;
4733 }
4734
4735 // When passing an array of floats, the array occupies consecutive
4736 // space in the argument area; only round up to the next doubleword
4737 // at the end of the array. Otherwise, each float takes 8 bytes.
4738 if (CallConv != CallingConv::Fast || needsLoad) {
4739 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize;
4740 ArgOffset += ArgSize;
4741 if (Flags.isInConsecutiveRegsLast())
4742 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4743 }
4744 break;
4745 case MVT::v4f32:
4746 case MVT::v4i32:
4747 case MVT::v8i16:
4748 case MVT::v16i8:
4749 case MVT::v2f64:
4750 case MVT::v2i64:
4751 case MVT::v1i128:
4752 case MVT::f128:
4753 // These can be scalar arguments or elements of a vector array type
4754 // passed directly. The latter are used to implement ELFv2 homogenous
4755 // vector aggregates.
4756 if (VR_idx != Num_VR_Regs) {
4757 Register VReg = MF.addLiveIn(PReg: VR[VR_idx], RC: &PPC::VRRCRegClass);
4758 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4759 ++VR_idx;
4760 } else {
4761 if (CallConv == CallingConv::Fast)
4762 ComputeArgOffset();
4763 needsLoad = true;
4764 }
4765 if (CallConv != CallingConv::Fast || needsLoad)
4766 ArgOffset += 16;
4767 break;
4768 }
4769
4770 // We need to load the argument to a virtual register if we determined
4771 // above that we ran out of physical registers of the appropriate type.
4772 if (needsLoad) {
4773 if (ObjSize < ArgSize && !isLittleEndian)
4774 CurArgOffset += ArgSize - ObjSize;
4775 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: CurArgOffset, IsImmutable: isImmutable);
4776 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4777 ArgVal = DAG.getLoad(VT: ObjectVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
4778 }
4779
4780 InVals.push_back(Elt: ArgVal);
4781 }
4782
4783 // Area that is at least reserved in the caller of this function.
4784 unsigned MinReservedArea;
4785 if (HasParameterArea)
4786 MinReservedArea = std::max(a: ArgOffset, b: LinkageSize + 8 * PtrByteSize);
4787 else
4788 MinReservedArea = LinkageSize;
4789
4790 // Set the size that is at least reserved in caller of this function. Tail
4791 // call optimized functions' reserved stack space needs to be aligned so that
4792 // taking the difference between two stack areas will result in an aligned
4793 // stack.
4794 MinReservedArea =
4795 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4796 FuncInfo->setMinReservedArea(MinReservedArea);
4797
4798 // If the function takes variable number of arguments, make a frame index for
4799 // the start of the first vararg value... for expansion of llvm.va_start.
4800 // On ELFv2ABI spec, it writes:
4801 // C programs that are intended to be *portable* across different compilers
4802 // and architectures must use the header file <stdarg.h> to deal with variable
4803 // argument lists.
4804 if (isVarArg && MFI.hasVAStart()) {
4805 int Depth = ArgOffset;
4806
4807 FuncInfo->setVarArgsFrameIndex(
4808 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: Depth, IsImmutable: true));
4809 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4810
4811 // If this function is vararg, store any remaining integer argument regs
4812 // to their spots on the stack so that they may be loaded by dereferencing
4813 // the result of va_next.
4814 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4815 GPR_idx < Num_GPR_Regs; ++GPR_idx) {
4816 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4817 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4818 SDValue Store =
4819 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4820 MemOps.push_back(Elt: Store);
4821 // Increment the address by four for the next argument to store
4822 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
4823 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4824 }
4825 }
4826
4827 if (!MemOps.empty())
4828 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4829
4830 return Chain;
4831}
4832
4833/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be
4834/// adjusted to accommodate the arguments for the tailcall.
4835static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
4836 unsigned ParamSize) {
4837
4838 if (!isTailCall) return 0;
4839
4840 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
4841 unsigned CallerMinReservedArea = FI->getMinReservedArea();
4842 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize;
4843 // Remember only if the new adjustment is bigger.
4844 if (SPDiff < FI->getTailCallSPDelta())
4845 FI->setTailCallSPDelta(SPDiff);
4846
4847 return SPDiff;
4848}
4849
4850static bool isFunctionGlobalAddress(const GlobalValue *CalleeGV);
4851
4852static bool callsShareTOCBase(const Function *Caller,
4853 const GlobalValue *CalleeGV,
4854 const TargetMachine &TM) {
4855 // It does not make sense to call callsShareTOCBase() with a caller that
4856 // is PC Relative since PC Relative callers do not have a TOC.
4857#ifndef NDEBUG
4858 const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller);
4859 assert(!STICaller->isUsingPCRelativeCalls() &&
4860 "PC Relative callers do not have a TOC and cannot share a TOC Base");
4861#endif
4862
4863 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
4864 // don't have enough information to determine if the caller and callee share
4865 // the same TOC base, so we have to pessimistically assume they don't for
4866 // correctness.
4867 if (!CalleeGV)
4868 return false;
4869
4870 // If the callee is preemptable, then the static linker will use a plt-stub
4871 // which saves the toc to the stack, and needs a nop after the call
4872 // instruction to convert to a toc-restore.
4873 if (!TM.shouldAssumeDSOLocal(GV: CalleeGV))
4874 return false;
4875
4876 // Functions with PC Relative enabled may clobber the TOC in the same DSO.
4877 // We may need a TOC restore in the situation where the caller requires a
4878 // valid TOC but the callee is PC Relative and does not.
4879 const Function *F = dyn_cast<Function>(Val: CalleeGV);
4880 const GlobalAlias *Alias = dyn_cast<GlobalAlias>(Val: CalleeGV);
4881
4882 // If we have an Alias we can try to get the function from there.
4883 if (Alias) {
4884 const GlobalObject *GlobalObj = Alias->getAliaseeObject();
4885 F = dyn_cast<Function>(Val: GlobalObj);
4886 }
4887
4888 // If we still have no valid function pointer we do not have enough
4889 // information to determine if the callee uses PC Relative calls so we must
4890 // assume that it does.
4891 if (!F)
4892 return false;
4893
4894 // If the callee uses PC Relative we cannot guarantee that the callee won't
4895 // clobber the TOC of the caller and so we must assume that the two
4896 // functions do not share a TOC base.
4897 const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(F: *F);
4898 if (STICallee->isUsingPCRelativeCalls())
4899 return false;
4900
4901 // If the GV is not a strong definition then we need to assume it can be
4902 // replaced by another function at link time. The function that replaces
4903 // it may not share the same TOC as the caller since the callee may be
4904 // replaced by a PC Relative version of the same function.
4905 if (!CalleeGV->isStrongDefinitionForLinker())
4906 return false;
4907
4908 // The medium and large code models are expected to provide a sufficiently
4909 // large TOC to provide all data addressing needs of a module with a
4910 // single TOC.
4911 if (CodeModel::Medium == TM.getCodeModel() ||
4912 CodeModel::Large == TM.getCodeModel())
4913 return true;
4914
4915 // Any explicitly-specified sections and section prefixes must also match.
4916 // Also, if we're using -ffunction-sections, then each function is always in
4917 // a different section (the same is true for COMDAT functions).
4918 if (TM.getFunctionSections() || CalleeGV->hasComdat() ||
4919 Caller->hasComdat() || CalleeGV->getSection() != Caller->getSection())
4920 return false;
4921 if (const auto *F = dyn_cast<Function>(Val: CalleeGV)) {
4922 if (F->getSectionPrefix() != Caller->getSectionPrefix())
4923 return false;
4924 }
4925
4926 return true;
4927}
4928
4929static bool
4930needStackSlotPassParameters(const PPCSubtarget &Subtarget,
4931 const SmallVectorImpl<ISD::OutputArg> &Outs) {
4932 assert(Subtarget.is64BitELFABI());
4933
4934 const unsigned PtrByteSize = 8;
4935 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4936
4937 static const MCPhysReg GPR[] = {
4938 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4939 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4940 };
4941 static const MCPhysReg VR[] = {
4942 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4943 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4944 };
4945
4946 const unsigned NumGPRs = std::size(GPR);
4947 const unsigned NumFPRs = 13;
4948 const unsigned NumVRs = std::size(VR);
4949 const unsigned ParamAreaSize = NumGPRs * PtrByteSize;
4950
4951 unsigned NumBytes = LinkageSize;
4952 unsigned AvailableFPRs = NumFPRs;
4953 unsigned AvailableVRs = NumVRs;
4954
4955 for (const ISD::OutputArg& Param : Outs) {
4956 if (Param.Flags.isNest()) continue;
4957
4958 if (CalculateStackSlotUsed(ArgVT: Param.VT, OrigVT: Param.ArgVT, Flags: Param.Flags, PtrByteSize,
4959 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4960 AvailableFPRs, AvailableVRs))
4961 return true;
4962 }
4963 return false;
4964}
4965
4966static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) {
4967 if (CB.arg_size() != CallerFn->arg_size())
4968 return false;
4969
4970 auto CalleeArgIter = CB.arg_begin();
4971 auto CalleeArgEnd = CB.arg_end();
4972 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin();
4973
4974 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) {
4975 const Value* CalleeArg = *CalleeArgIter;
4976 const Value* CallerArg = &(*CallerArgIter);
4977 if (CalleeArg == CallerArg)
4978 continue;
4979
4980 // e.g. @caller([4 x i64] %a, [4 x i64] %b) {
4981 // tail call @callee([4 x i64] undef, [4 x i64] %b)
4982 // }
4983 // 1st argument of callee is undef and has the same type as caller.
4984 if (CalleeArg->getType() == CallerArg->getType() &&
4985 isa<UndefValue>(Val: CalleeArg))
4986 continue;
4987
4988 return false;
4989 }
4990
4991 return true;
4992}
4993
4994// Returns true if TCO is possible between the callers and callees
4995// calling conventions.
4996static bool
4997areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC,
4998 CallingConv::ID CalleeCC) {
4999 // Tail calls are possible with fastcc and ccc.
5000 auto isTailCallableCC = [] (CallingConv::ID CC){
5001 return CC == CallingConv::C || CC == CallingConv::Fast;
5002 };
5003 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC))
5004 return false;
5005
5006 // We can safely tail call both fastcc and ccc callees from a c calling
5007 // convention caller. If the caller is fastcc, we may have less stack space
5008 // than a non-fastcc caller with the same signature so disable tail-calls in
5009 // that case.
5010 return CallerCC == CallingConv::C || CallerCC == CalleeCC;
5011}
5012
5013bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
5014 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5015 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5016 const SmallVectorImpl<ISD::OutputArg> &Outs,
5017 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5018 bool isCalleeExternalSymbol) const {
5019 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
5020
5021 if (DisableSCO && !TailCallOpt) return false;
5022
5023 // Variadic argument functions are not supported.
5024 if (isVarArg) return false;
5025
5026 // Check that the calling conventions are compatible for tco.
5027 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC, CalleeCC))
5028 return false;
5029
5030 // Caller contains any byval parameter is not supported.
5031 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5032 return false;
5033
5034 // Callee contains any byval parameter is not supported, too.
5035 // Note: This is a quick work around, because in some cases, e.g.
5036 // caller's stack size > callee's stack size, we are still able to apply
5037 // sibling call optimization. For example, gcc is able to do SCO for caller1
5038 // in the following example, but not for caller2.
5039 // struct test {
5040 // long int a;
5041 // char ary[56];
5042 // } gTest;
5043 // __attribute__((noinline)) int callee(struct test v, struct test *b) {
5044 // b->a = v.a;
5045 // return 0;
5046 // }
5047 // void caller1(struct test a, struct test c, struct test *b) {
5048 // callee(gTest, b); }
5049 // void caller2(struct test *b) { callee(gTest, b); }
5050 if (any_of(Range: Outs, P: [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); }))
5051 return false;
5052
5053 // If callee and caller use different calling conventions, we cannot pass
5054 // parameters on stack since offsets for the parameter area may be different.
5055 if (CallerCC != CalleeCC && needStackSlotPassParameters(Subtarget, Outs))
5056 return false;
5057
5058 // All variants of 64-bit ELF ABIs without PC-Relative addressing require that
5059 // the caller and callee share the same TOC for TCO/SCO. If the caller and
5060 // callee potentially have different TOC bases then we cannot tail call since
5061 // we need to restore the TOC pointer after the call.
5062 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977
5063 // We cannot guarantee this for indirect calls or calls to external functions.
5064 // When PC-Relative addressing is used, the concept of the TOC is no longer
5065 // applicable so this check is not required.
5066 // Check first for indirect calls.
5067 if (!Subtarget.isUsingPCRelativeCalls() &&
5068 !isFunctionGlobalAddress(CalleeGV) && !isCalleeExternalSymbol)
5069 return false;
5070
5071 // Check if we share the TOC base.
5072 if (!Subtarget.isUsingPCRelativeCalls() &&
5073 !callsShareTOCBase(Caller: CallerFunc, CalleeGV, TM: getTargetMachine()))
5074 return false;
5075
5076 // TCO allows altering callee ABI, so we don't have to check further.
5077 if (CalleeCC == CallingConv::Fast && TailCallOpt)
5078 return true;
5079
5080 if (DisableSCO) return false;
5081
5082 // If callee use the same argument list that caller is using, then we can
5083 // apply SCO on this case. If it is not, then we need to check if callee needs
5084 // stack for passing arguments.
5085 // PC Relative tail calls may not have a CallBase.
5086 // If there is no CallBase we cannot verify if we have the same argument
5087 // list so assume that we don't have the same argument list.
5088 if (CB && !hasSameArgumentList(CallerFn: CallerFunc, CB: *CB) &&
5089 needStackSlotPassParameters(Subtarget, Outs))
5090 return false;
5091 else if (!CB && needStackSlotPassParameters(Subtarget, Outs))
5092 return false;
5093
5094 return true;
5095}
5096
5097/// IsEligibleForTailCallOptimization - Check whether the call is eligible
5098/// for tail call optimization. Targets which want to do tail call
5099/// optimization should implement this function.
5100bool PPCTargetLowering::IsEligibleForTailCallOptimization(
5101 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5102 CallingConv::ID CallerCC, bool isVarArg,
5103 const SmallVectorImpl<ISD::InputArg> &Ins) const {
5104 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5105 return false;
5106
5107 // Variable argument functions are not supported.
5108 if (isVarArg)
5109 return false;
5110
5111 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) {
5112 // Functions containing by val parameters are not supported.
5113 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5114 return false;
5115
5116 // Non-PIC/GOT tail calls are supported.
5117 if (getTargetMachine().getRelocationModel() != Reloc::PIC_)
5118 return true;
5119
5120 // At the moment we can only do local tail calls (in same module, hidden
5121 // or protected) if we are generating PIC.
5122 if (CalleeGV)
5123 return CalleeGV->hasHiddenVisibility() ||
5124 CalleeGV->hasProtectedVisibility();
5125 }
5126
5127 return false;
5128}
5129
5130/// isCallCompatibleAddress - Return the immediate to use if the specified
5131/// 32-bit value is representable in the immediate field of a BxA instruction.
5132static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
5133 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op);
5134 if (!C) return nullptr;
5135
5136 int Addr = C->getZExtValue();
5137 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
5138 SignExtend32<26>(X: Addr) != Addr)
5139 return nullptr; // Top 6 bits have to be sext of immediate.
5140
5141 return DAG
5142 .getSignedConstant(
5143 Val: (int)C->getZExtValue() >> 2, DL: SDLoc(Op),
5144 VT: DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout()))
5145 .getNode();
5146}
5147
5148namespace {
5149
5150struct TailCallArgumentInfo {
5151 SDValue Arg;
5152 SDValue FrameIdxOp;
5153 int FrameIdx = 0;
5154
5155 TailCallArgumentInfo() = default;
5156};
5157
5158} // end anonymous namespace
5159
5160/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot.
5161static void StoreTailCallArgumentsToStackSlot(
5162 SelectionDAG &DAG, SDValue Chain,
5163 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs,
5164 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) {
5165 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) {
5166 SDValue Arg = TailCallArgs[i].Arg;
5167 SDValue FIN = TailCallArgs[i].FrameIdxOp;
5168 int FI = TailCallArgs[i].FrameIdx;
5169 // Store relative to framepointer.
5170 MemOpChains.push_back(Elt: DAG.getStore(
5171 Chain, dl, Val: Arg, Ptr: FIN,
5172 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI)));
5173 }
5174}
5175
5176/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to
5177/// the appropriate stack slot for the tail call optimized function call.
5178static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
5179 SDValue OldRetAddr, SDValue OldFP,
5180 int SPDiff, const SDLoc &dl) {
5181 if (SPDiff) {
5182 // Calculate the new stack slot for the return address.
5183 MachineFunction &MF = DAG.getMachineFunction();
5184 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
5185 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
5186 int SlotSize = Subtarget.isPPC64() ? 8 : 4;
5187 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset();
5188 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(Size: SlotSize,
5189 SPOffset: NewRetAddrLoc, IsImmutable: true);
5190 SDValue NewRetAddrFrIdx =
5191 DAG.getFrameIndex(FI: NewRetAddr, VT: Subtarget.getScalarIntVT());
5192 Chain = DAG.getStore(Chain, dl, Val: OldRetAddr, Ptr: NewRetAddrFrIdx,
5193 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: NewRetAddr));
5194 }
5195 return Chain;
5196}
5197
5198/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate
5199/// the position of the argument.
5200static void CalculateTailCallArgDest(
5201 SelectionDAG &DAG, MachineFunction &MF, bool IsPPC64, SDValue Arg,
5202 int SPDiff, unsigned ArgOffset,
5203 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5204 int Offset = ArgOffset + SPDiff;
5205 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8;
5206 int FI = MF.getFrameInfo().CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
5207 EVT VT = IsPPC64 ? MVT::i64 : MVT::i32;
5208 SDValue FIN = DAG.getFrameIndex(FI, VT);
5209 TailCallArgumentInfo Info;
5210 Info.Arg = Arg;
5211 Info.FrameIdxOp = FIN;
5212 Info.FrameIdx = FI;
5213 TailCallArguments.push_back(Elt: Info);
5214}
5215
5216/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address
5217/// stack slot. Returns the chain as result and the loaded frame pointers in
5218/// LROpOut/FPOpout. Used when tail calling.
5219SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(
5220 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
5221 SDValue &FPOpOut, const SDLoc &dl) const {
5222 if (SPDiff) {
5223 // Load the LR and FP stack slot for later adjusting.
5224 LROpOut = getReturnAddrFrameIndex(DAG);
5225 LROpOut = DAG.getLoad(VT: Subtarget.getScalarIntVT(), dl, Chain, Ptr: LROpOut,
5226 PtrInfo: MachinePointerInfo());
5227 Chain = SDValue(LROpOut.getNode(), 1);
5228 }
5229 return Chain;
5230}
5231
5232/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
5233/// by "Src" to address "Dst" of size "Size". Alignment information is
5234/// specified by the specific parameter attribute. The copy will be passed as
5235/// a byval function parameter.
5236/// Sometimes what we are copying is the end of a larger object, the part that
5237/// does not fit in registers.
5238static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
5239 SDValue Chain, ISD::ArgFlagsTy Flags,
5240 SelectionDAG &DAG, const SDLoc &dl) {
5241 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
5242 Align Alignment = Flags.getNonZeroByValAlign();
5243 return DAG.getMemcpy(
5244 Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment, isVol: false, AlwaysInline: false,
5245 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
5246}
5247
5248/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
5249/// tail calls.
5250static void LowerMemOpCallTo(
5251 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
5252 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64,
5253 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
5254 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) {
5255 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5256 if (!isTailCall) {
5257 if (isVector) {
5258 SDValue StackPtr;
5259 if (isPPC64)
5260 StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
5261 else
5262 StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
5263 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr,
5264 N2: DAG.getConstant(Val: ArgOffset, DL: dl, VT: PtrVT));
5265 }
5266 MemOpChains.push_back(
5267 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
5268 // Calculate and remember argument location.
5269 } else
5270 CalculateTailCallArgDest(DAG, MF, IsPPC64: isPPC64, Arg, SPDiff, ArgOffset,
5271 TailCallArguments);
5272}
5273
5274static void
5275PrepareTailCall(SelectionDAG &DAG, SDValue &InGlue, SDValue &Chain,
5276 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp,
5277 SDValue FPOp,
5278 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5279 // Emit a sequence of copyto/copyfrom virtual registers for arguments that
5280 // might overwrite each other in case of tail call optimization.
5281 SmallVector<SDValue, 8> MemOpChains2;
5282 // Do not flag preceding copytoreg stuff together with the following stuff.
5283 InGlue = SDValue();
5284 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArgs: TailCallArguments,
5285 MemOpChains&: MemOpChains2, dl);
5286 if (!MemOpChains2.empty())
5287 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains2);
5288
5289 // Store the return address to the appropriate stack slot.
5290 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, OldRetAddr: LROp, OldFP: FPOp, SPDiff, dl);
5291
5292 // Emit callseq_end just before tailcall node.
5293 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL: dl);
5294 InGlue = Chain.getValue(R: 1);
5295}
5296
5297// Is this global address that of a function that can be called by name? (as
5298// opposed to something that must hold a descriptor for an indirect call).
5299static bool isFunctionGlobalAddress(const GlobalValue *GV) {
5300 if (GV) {
5301 if (GV->isThreadLocal())
5302 return false;
5303
5304 return GV->getValueType()->isFunctionTy();
5305 }
5306
5307 return false;
5308}
5309
5310SDValue PPCTargetLowering::LowerCallResult(
5311 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
5312 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5313 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
5314 SmallVector<CCValAssign, 16> RVLocs;
5315 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5316 *DAG.getContext());
5317
5318 CCRetInfo.AnalyzeCallResult(
5319 Ins, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
5320 ? RetCC_PPC_Cold
5321 : RetCC_PPC);
5322
5323 // Copy all of the result registers out of their specified physreg.
5324 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
5325 CCValAssign &VA = RVLocs[i];
5326 assert(VA.isRegLoc() && "Can only return in registers!");
5327
5328 SDValue Val;
5329
5330 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
5331 SDValue Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5332 Glue: InGlue);
5333 Chain = Lo.getValue(R: 1);
5334 InGlue = Lo.getValue(R: 2);
5335 VA = RVLocs[++i]; // skip ahead to next loc
5336 SDValue Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5337 Glue: InGlue);
5338 Chain = Hi.getValue(R: 1);
5339 InGlue = Hi.getValue(R: 2);
5340 if (!Subtarget.isLittleEndian())
5341 std::swap (a&: Lo, b&: Hi);
5342 Val = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
5343 } else {
5344 Val = DAG.getCopyFromReg(Chain, dl,
5345 Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
5346 Chain = Val.getValue(R: 1);
5347 InGlue = Val.getValue(R: 2);
5348 }
5349
5350 switch (VA.getLocInfo()) {
5351 default: llvm_unreachable("Unknown loc info!");
5352 case CCValAssign::Full: break;
5353 case CCValAssign::AExt:
5354 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5355 break;
5356 case CCValAssign::ZExt:
5357 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: VA.getLocVT(), N1: Val,
5358 N2: DAG.getValueType(VA.getValVT()));
5359 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5360 break;
5361 case CCValAssign::SExt:
5362 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: VA.getLocVT(), N1: Val,
5363 N2: DAG.getValueType(VA.getValVT()));
5364 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5365 break;
5366 }
5367
5368 InVals.push_back(Elt: Val);
5369 }
5370
5371 return Chain;
5372}
5373
5374static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG,
5375 const PPCSubtarget &Subtarget, bool isPatchPoint) {
5376 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5377 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5378
5379 // PatchPoint calls are not indirect.
5380 if (isPatchPoint)
5381 return false;
5382
5383 if (isFunctionGlobalAddress(GV) || isa<ExternalSymbolSDNode>(Val: Callee))
5384 return false;
5385
5386 // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not
5387 // becuase the immediate function pointer points to a descriptor instead of
5388 // a function entry point. The ELFv2 ABI cannot use a BLA because the function
5389 // pointer immediate points to the global entry point, while the BLA would
5390 // need to jump to the local entry point (see rL211174).
5391 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() &&
5392 isBLACompatibleAddress(Op: Callee, DAG))
5393 return false;
5394
5395 return true;
5396}
5397
5398// AIX and 64-bit ELF ABIs w/o PCRel require a TOC save/restore around calls.
5399static inline bool isTOCSaveRestoreRequired(const PPCSubtarget &Subtarget) {
5400 return Subtarget.isAIXABI() ||
5401 (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls());
5402}
5403
5404static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags,
5405 const Function &Caller, const SDValue &Callee,
5406 const PPCSubtarget &Subtarget,
5407 const TargetMachine &TM,
5408 bool IsStrictFPCall = false) {
5409 if (CFlags.IsTailCall)
5410 return PPCISD::TC_RETURN;
5411
5412 unsigned RetOpc = 0;
5413 // This is a call through a function pointer.
5414 if (CFlags.IsIndirect) {
5415 // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross
5416 // indirect calls. The save of the caller's TOC pointer to the stack will be
5417 // inserted into the DAG as part of call lowering. The restore of the TOC
5418 // pointer is modeled by using a pseudo instruction for the call opcode that
5419 // represents the 2 instruction sequence of an indirect branch and link,
5420 // immediately followed by a load of the TOC pointer from the stack save
5421 // slot into gpr2. For 64-bit ELFv2 ABI with PCRel, do not restore the TOC
5422 // as it is not saved or used.
5423 if (Subtarget.usePointerGlueHelper())
5424 RetOpc = PPCISD::BL_LOAD_TOC;
5425 else
5426 RetOpc = isTOCSaveRestoreRequired(Subtarget) ? PPCISD::BCTRL_LOAD_TOC
5427 : PPCISD::BCTRL;
5428 } else if (Subtarget.isUsingPCRelativeCalls()) {
5429 assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI.");
5430 RetOpc = PPCISD::CALL_NOTOC;
5431 } else if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) {
5432 // The ABIs that maintain a TOC pointer accross calls need to have a nop
5433 // immediately following the call instruction if the caller and callee may
5434 // have different TOC bases. At link time if the linker determines the calls
5435 // may not share a TOC base, the call is redirected to a trampoline inserted
5436 // by the linker. The trampoline will (among other things) save the callers
5437 // TOC pointer at an ABI designated offset in the linkage area and the
5438 // linker will rewrite the nop to be a load of the TOC pointer from the
5439 // linkage area into gpr2.
5440 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5441 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5442 RetOpc =
5443 callsShareTOCBase(Caller: &Caller, CalleeGV: GV, TM) ? PPCISD::CALL : PPCISD::CALL_NOP;
5444 } else
5445 RetOpc = PPCISD::CALL;
5446 if (IsStrictFPCall) {
5447 switch (RetOpc) {
5448 default:
5449 llvm_unreachable("Unknown call opcode");
5450 case PPCISD::BCTRL_LOAD_TOC:
5451 RetOpc = PPCISD::BCTRL_LOAD_TOC_RM;
5452 break;
5453 case PPCISD::BCTRL:
5454 RetOpc = PPCISD::BCTRL_RM;
5455 break;
5456 case PPCISD::BL_LOAD_TOC:
5457 RetOpc = PPCISD::BL_LOAD_TOC_RM;
5458 break;
5459 case PPCISD::CALL_NOTOC:
5460 RetOpc = PPCISD::CALL_NOTOC_RM;
5461 break;
5462 case PPCISD::CALL:
5463 RetOpc = PPCISD::CALL_RM;
5464 break;
5465 case PPCISD::CALL_NOP:
5466 RetOpc = PPCISD::CALL_NOP_RM;
5467 break;
5468 }
5469 }
5470 return RetOpc;
5471}
5472
5473static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
5474 const SDLoc &dl, const PPCSubtarget &Subtarget) {
5475 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI())
5476 if (SDNode *Dest = isBLACompatibleAddress(Op: Callee, DAG))
5477 return SDValue(Dest, 0);
5478
5479 // Returns true if the callee is local, and false otherwise.
5480 auto isLocalCallee = [&]() {
5481 const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5482 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5483
5484 return DAG.getTarget().shouldAssumeDSOLocal(GV) &&
5485 !isa_and_nonnull<GlobalIFunc>(Val: GV);
5486 };
5487
5488 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in
5489 // a static relocation model causes some versions of GNU LD (2.17.50, at
5490 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are
5491 // built with secure-PLT.
5492 bool UsePlt =
5493 Subtarget.is32BitELFABI() && !isLocalCallee() &&
5494 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_;
5495
5496 const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) {
5497 const TargetMachine &TM = Subtarget.getTargetMachine();
5498 const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering();
5499 auto *S =
5500 static_cast<MCSymbolXCOFF *>(TLOF->getFunctionEntryPointSymbol(Func: GV, TM));
5501
5502 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5503 return DAG.getMCSymbol(Sym: S, VT: PtrVT);
5504 };
5505
5506 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5507 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5508 if (isFunctionGlobalAddress(GV)) {
5509 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val: Callee)->getGlobal();
5510
5511 if (Subtarget.isAIXABI()) {
5512 return getAIXFuncEntryPointSymbolSDNode(GV);
5513 }
5514 return DAG.getTargetGlobalAddress(GV, DL: dl, VT: Callee.getValueType(), offset: 0,
5515 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5516 }
5517
5518 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val: Callee)) {
5519 const char *SymName = S->getSymbol();
5520 if (Subtarget.isAIXABI()) {
5521 // If there exists a user-declared function whose name is the same as the
5522 // ExternalSymbol's, then we pick up the user-declared version.
5523 const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
5524 if (const Function *F =
5525 dyn_cast_or_null<Function>(Val: Mod->getNamedValue(Name: SymName)))
5526 return getAIXFuncEntryPointSymbolSDNode(F);
5527
5528 // On AIX, direct function calls reference the symbol for the function's
5529 // entry point, which is named by prepending a "." before the function's
5530 // C-linkage name. A Qualname is returned here because an external
5531 // function entry point is a csect with XTY_ER property.
5532 const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) {
5533 auto &Context = DAG.getMachineFunction().getContext();
5534 MCSectionXCOFF *Sec = Context.getXCOFFSection(
5535 Section: (Twine(".") + Twine(SymName)).str(), K: SectionKind::getMetadata(),
5536 CsectProp: XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER));
5537 return Sec->getQualNameSymbol();
5538 };
5539
5540 SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data();
5541 }
5542 return DAG.getTargetExternalSymbol(Sym: SymName, VT: Callee.getValueType(),
5543 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5544 }
5545
5546 // No transformation needed.
5547 assert(Callee.getNode() && "What no callee?");
5548 return Callee;
5549}
5550
5551static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) {
5552 assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START &&
5553 "Expected a CALLSEQ_STARTSDNode.");
5554
5555 // The last operand is the chain, except when the node has glue. If the node
5556 // has glue, then the last operand is the glue, and the chain is the second
5557 // last operand.
5558 SDValue LastValue = CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 1);
5559 if (LastValue.getValueType() != MVT::Glue)
5560 return LastValue;
5561
5562 return CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 2);
5563}
5564
5565// Creates the node that moves a functions address into the count register
5566// to prepare for an indirect call instruction.
5567static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5568 SDValue &Glue, SDValue &Chain,
5569 const SDLoc &dl) {
5570 SDValue MTCTROps[] = {Chain, Callee, Glue};
5571 EVT ReturnTypes[] = {MVT::Other, MVT::Glue};
5572 Chain = DAG.getNode(Opcode: PPCISD::MTCTR, DL: dl, ResultTys: ReturnTypes,
5573 Ops: ArrayRef(MTCTROps, Glue.getNode() ? 3 : 2));
5574 // The glue is the second value produced.
5575 Glue = Chain.getValue(R: 1);
5576}
5577
5578static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5579 SDValue &Glue, SDValue &Chain,
5580 SDValue CallSeqStart,
5581 const CallBase *CB, const SDLoc &dl,
5582 bool hasNest,
5583 const PPCSubtarget &Subtarget) {
5584 // Function pointers in the 64-bit SVR4 ABI do not point to the function
5585 // entry point, but to the function descriptor (the function entry point
5586 // address is part of the function descriptor though).
5587 // The function descriptor is a three doubleword structure with the
5588 // following fields: function entry point, TOC base address and
5589 // environment pointer.
5590 // Thus for a call through a function pointer, the following actions need
5591 // to be performed:
5592 // 1. Save the TOC of the caller in the TOC save area of its stack
5593 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
5594 // 2. Load the address of the function entry point from the function
5595 // descriptor.
5596 // 3. Load the TOC of the callee from the function descriptor into r2.
5597 // 4. Load the environment pointer from the function descriptor into
5598 // r11.
5599 // 5. Branch to the function entry point address.
5600 // 6. On return of the callee, the TOC of the caller needs to be
5601 // restored (this is done in FinishCall()).
5602 //
5603 // The loads are scheduled at the beginning of the call sequence, and the
5604 // register copies are flagged together to ensure that no other
5605 // operations can be scheduled in between. E.g. without flagging the
5606 // copies together, a TOC access in the caller could be scheduled between
5607 // the assignment of the callee TOC and the branch to the callee, which leads
5608 // to incorrect code.
5609
5610 // Start by loading the function address from the descriptor.
5611 SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart);
5612 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
5613 ? (MachineMemOperand::MODereferenceable |
5614 MachineMemOperand::MOInvariant)
5615 : MachineMemOperand::MONone;
5616
5617 MachinePointerInfo MPI(CB ? CB->getCalledOperand() : nullptr);
5618
5619 // Registers used in building the DAG.
5620 const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister();
5621 const MCRegister TOCReg = Subtarget.getTOCPointerRegister();
5622
5623 // Offsets of descriptor members.
5624 const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset();
5625 const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset();
5626
5627 const MVT RegVT = Subtarget.getScalarIntVT();
5628 const Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
5629
5630 // One load for the functions entry point address.
5631 SDValue LoadFuncPtr = DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: Callee, PtrInfo: MPI,
5632 Alignment, MMOFlags);
5633
5634 // One for loading the TOC anchor for the module that contains the called
5635 // function.
5636 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCAnchorOffset, DL: dl);
5637 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: TOCOff);
5638 SDValue TOCPtr =
5639 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddTOC,
5640 PtrInfo: MPI.getWithOffset(O: TOCAnchorOffset), Alignment, MMOFlags);
5641
5642 // One for loading the environment pointer.
5643 SDValue PtrOff = DAG.getIntPtrConstant(Val: EnvPtrOffset, DL: dl);
5644 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: PtrOff);
5645 SDValue LoadEnvPtr =
5646 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddPtr,
5647 PtrInfo: MPI.getWithOffset(O: EnvPtrOffset), Alignment, MMOFlags);
5648
5649
5650 // Then copy the newly loaded TOC anchor to the TOC pointer.
5651 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, Reg: TOCReg, N: TOCPtr, Glue);
5652 Chain = TOCVal.getValue(R: 0);
5653 Glue = TOCVal.getValue(R: 1);
5654
5655 // If the function call has an explicit 'nest' parameter, it takes the
5656 // place of the environment pointer.
5657 assert((!hasNest || !Subtarget.isAIXABI()) &&
5658 "Nest parameter is not supported on AIX.");
5659 if (!hasNest) {
5660 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, Reg: EnvPtrReg, N: LoadEnvPtr, Glue);
5661 Chain = EnvVal.getValue(R: 0);
5662 Glue = EnvVal.getValue(R: 1);
5663 }
5664
5665 // The rest of the indirect call sequence is the same as the non-descriptor
5666 // DAG.
5667 prepareIndirectCall(DAG, Callee&: LoadFuncPtr, Glue, Chain, dl);
5668}
5669
5670static void prepareOutOfLineGlueCall(SelectionDAG &DAG, SDValue &Callee,
5671 SDValue &Glue, SDValue &Chain,
5672 SDValue CallSeqStart, const CallBase *CB,
5673 const SDLoc &dl, bool hasNest,
5674 const PPCSubtarget &Subtarget) {
5675 // On AIX there is a feature ("out of line glue code") which uses a special
5676 // trampoline function ._ptrgl to do the indirect call. If this option is
5677 // enabled we instead simply load the address of the descriptor into gpr11,
5678 // with the arguments in the 'normal' registers and branch to the ._ptrgl
5679 // stub.
5680 const MCRegister PtrGlueReg = Subtarget.getGlueCodeDescriptorRegister();
5681 SDValue MoveToPhysicalReg =
5682 DAG.getCopyToReg(Chain, dl, Reg: PtrGlueReg, N: Callee, Glue);
5683 Chain = MoveToPhysicalReg.getValue(R: 0);
5684 Glue = MoveToPhysicalReg.getValue(R: 1);
5685}
5686
5687static void
5688buildCallOperands(SmallVectorImpl<SDValue> &Ops,
5689 PPCTargetLowering::CallFlags CFlags, const SDLoc &dl,
5690 SelectionDAG &DAG,
5691 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass,
5692 SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff,
5693 const PPCSubtarget &Subtarget) {
5694 const bool IsPPC64 = Subtarget.isPPC64();
5695 // MVT for a general purpose register.
5696 const MVT RegVT = Subtarget.getScalarIntVT();
5697
5698 // First operand is always the chain.
5699 Ops.push_back(Elt: Chain);
5700
5701 // If it's a direct call pass the callee as the second operand.
5702 if (!CFlags.IsIndirect)
5703 Ops.push_back(Elt: Callee);
5704 else if (Subtarget.usePointerGlueHelper()) {
5705 Ops.push_back(Elt: Callee);
5706 // Add the register used to pass the descriptor address.
5707 Ops.push_back(
5708 Elt: DAG.getRegister(Reg: Subtarget.getGlueCodeDescriptorRegister(), VT: RegVT));
5709 } else {
5710 assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect.");
5711
5712 // For the TOC based ABIs, we have saved the TOC pointer to the linkage area
5713 // on the stack (this would have been done in `LowerCall_64SVR4` or
5714 // `LowerCall_AIX`). The call instruction is a pseudo instruction that
5715 // represents both the indirect branch and a load that restores the TOC
5716 // pointer from the linkage area. The operand for the TOC restore is an add
5717 // of the TOC save offset to the stack pointer. This must be the second
5718 // operand: after the chain input but before any other variadic arguments.
5719 // For 64-bit ELFv2 ABI with PCRel, do not restore the TOC as it is not
5720 // saved or used.
5721 if (isTOCSaveRestoreRequired(Subtarget)) {
5722 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
5723
5724 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: RegVT);
5725 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5726 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
5727 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: StackPtr, N2: TOCOff);
5728 Ops.push_back(Elt: AddTOC);
5729 }
5730
5731 // Add the register used for the environment pointer.
5732 if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest)
5733 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getEnvironmentPointerRegister(),
5734 VT: RegVT));
5735
5736
5737 // Add CTR register as callee so a bctr can be emitted later.
5738 if (CFlags.IsTailCall)
5739 Ops.push_back(Elt: DAG.getRegister(Reg: IsPPC64 ? PPC::CTR8 : PPC::CTR, VT: RegVT));
5740 }
5741
5742 // If this is a tail call add stack pointer delta.
5743 if (CFlags.IsTailCall)
5744 Ops.push_back(Elt: DAG.getConstant(Val: SPDiff, DL: dl, VT: MVT::i32));
5745
5746 // Add argument registers to the end of the list so that they are known live
5747 // into the call.
5748 for (const auto &[Reg, N] : RegsToPass)
5749 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
5750
5751 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
5752 // no way to mark dependencies as implicit here.
5753 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter.
5754 if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) &&
5755 !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls())
5756 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getTOCPointerRegister(), VT: RegVT));
5757
5758 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
5759 if (CFlags.IsVarArg && Subtarget.is32BitELFABI())
5760 Ops.push_back(Elt: DAG.getRegister(Reg: PPC::CR1EQ, VT: MVT::i32));
5761
5762 // Add a register mask operand representing the call-preserved registers.
5763 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
5764 const uint32_t *Mask =
5765 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CFlags.CallConv);
5766 assert(Mask && "Missing call preserved mask for calling convention");
5767 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
5768
5769 // If the glue is valid, it is the last operand.
5770 if (Glue.getNode())
5771 Ops.push_back(Elt: Glue);
5772}
5773
5774SDValue PPCTargetLowering::FinishCall(
5775 CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG,
5776 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue,
5777 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5778 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins,
5779 SmallVectorImpl<SDValue> &InVals, const CallBase *CB) const {
5780
5781 if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) ||
5782 Subtarget.isAIXABI())
5783 setUsesTOCBasePtr(DAG);
5784
5785 unsigned CallOpc =
5786 getCallOpcode(CFlags, Caller: DAG.getMachineFunction().getFunction(), Callee,
5787 Subtarget, TM: DAG.getTarget(), IsStrictFPCall: CB ? CB->isStrictFP() : false);
5788
5789 if (!CFlags.IsIndirect)
5790 Callee = transformCallee(Callee, DAG, dl, Subtarget);
5791 else if (Subtarget.usesFunctionDescriptors()) {
5792 if (Subtarget.usePointerGlueHelper()) {
5793 prepareOutOfLineGlueCall(DAG, Callee, Glue, Chain, CallSeqStart, CB, dl,
5794 hasNest: CFlags.HasNest, Subtarget);
5795 SDValue PtrGlueCallee =
5796 DAG.getExternalSymbol(Sym: "_ptrgl", VT: getPointerTy(DL: DAG.getDataLayout()));
5797 Callee = transformCallee(Callee: PtrGlueCallee, DAG, dl, Subtarget);
5798 } else {
5799 prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CB,
5800 dl, hasNest: CFlags.HasNest, Subtarget);
5801 }
5802 } else {
5803 prepareIndirectCall(DAG, Callee, Glue, Chain, dl);
5804 }
5805
5806 // Build the operand list for the call instruction.
5807 SmallVector<SDValue, 8> Ops;
5808 buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee,
5809 SPDiff, Subtarget);
5810
5811 // Emit tail call.
5812 if (CFlags.IsTailCall) {
5813 // Indirect tail call when using PC Relative calls do not have the same
5814 // constraints.
5815 assert(((Callee.getOpcode() == ISD::Register &&
5816 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) ||
5817 Callee.getOpcode() == ISD::TargetExternalSymbol ||
5818 Callee.getOpcode() == ISD::TargetGlobalAddress ||
5819 isa<ConstantSDNode>(Callee) ||
5820 (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) &&
5821 "Expecting a global address, external symbol, absolute value, "
5822 "register or an indirect tail call when PC Relative calls are "
5823 "used.");
5824 // PC Relative calls also use TC_RETURN as the way to mark tail calls.
5825 assert(CallOpc == PPCISD::TC_RETURN &&
5826 "Unexpected call opcode for a tail call.");
5827 DAG.getMachineFunction().getFrameInfo().setHasTailCall();
5828 SDValue Ret = DAG.getNode(Opcode: CallOpc, DL: dl, VT: MVT::Other, Ops);
5829 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CFlags.NoMerge);
5830 return Ret;
5831 }
5832
5833 std::array<EVT, 2> ReturnTypes = {._M_elems: {MVT::Other, MVT::Glue}};
5834 Chain = DAG.getNode(Opcode: CallOpc, DL: dl, ResultTys: ReturnTypes, Ops);
5835 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CFlags.NoMerge);
5836 Glue = Chain.getValue(R: 1);
5837
5838 // When performing tail call optimization the callee pops its arguments off
5839 // the stack. Account for this here so these bytes can be pushed back on in
5840 // PPCFrameLowering::eliminateCallFramePseudoInstr.
5841 int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast &&
5842 getTargetMachine().Options.GuaranteedTailCallOpt)
5843 ? NumBytes
5844 : 0;
5845
5846 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: BytesCalleePops, Glue, DL: dl);
5847 Glue = Chain.getValue(R: 1);
5848
5849 return LowerCallResult(Chain, InGlue: Glue, CallConv: CFlags.CallConv, isVarArg: CFlags.IsVarArg, Ins, dl,
5850 DAG, InVals);
5851}
5852
5853bool PPCTargetLowering::supportsTailCallFor(const CallBase *CB) const {
5854 CallingConv::ID CalleeCC = CB->getCallingConv();
5855 const Function *CallerFunc = CB->getCaller();
5856 CallingConv::ID CallerCC = CallerFunc->getCallingConv();
5857 const Function *CalleeFunc = CB->getCalledFunction();
5858 if (!CalleeFunc)
5859 return false;
5860 const GlobalValue *CalleeGV = dyn_cast<GlobalValue>(Val: CalleeFunc);
5861
5862 SmallVector<ISD::OutputArg, 2> Outs;
5863 SmallVector<ISD::InputArg, 2> Ins;
5864
5865 GetReturnInfo(CC: CalleeCC, ReturnType: CalleeFunc->getReturnType(),
5866 attr: CalleeFunc->getAttributes(), Outs, TLI: *this,
5867 DL: CalleeFunc->getDataLayout());
5868
5869 return isEligibleForTCO(CalleeGV, CalleeCC, CallerCC, CB,
5870 isVarArg: CalleeFunc->isVarArg(), Outs, Ins, CallerFunc,
5871 isCalleeExternalSymbol: false /*isCalleeExternalSymbol*/);
5872}
5873
5874bool PPCTargetLowering::isEligibleForTCO(
5875 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5876 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5877 const SmallVectorImpl<ISD::OutputArg> &Outs,
5878 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5879 bool isCalleeExternalSymbol) const {
5880 if (Subtarget.useLongCalls() && !(CB && CB->isMustTailCall()))
5881 return false;
5882
5883 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64())
5884 return IsEligibleForTailCallOptimization_64SVR4(
5885 CalleeGV, CalleeCC, CallerCC, CB, isVarArg, Outs, Ins, CallerFunc,
5886 isCalleeExternalSymbol);
5887 else
5888 return IsEligibleForTailCallOptimization(CalleeGV, CalleeCC, CallerCC,
5889 isVarArg, Ins);
5890}
5891
5892SDValue
5893PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
5894 SmallVectorImpl<SDValue> &InVals) const {
5895 SelectionDAG &DAG = CLI.DAG;
5896 SDLoc &dl = CLI.DL;
5897 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
5898 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
5899 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
5900 SDValue Chain = CLI.Chain;
5901 SDValue Callee = CLI.Callee;
5902 bool &isTailCall = CLI.IsTailCall;
5903 CallingConv::ID CallConv = CLI.CallConv;
5904 bool isVarArg = CLI.IsVarArg;
5905 bool isPatchPoint = CLI.IsPatchPoint;
5906 const CallBase *CB = CLI.CB;
5907
5908 if (isTailCall) {
5909 MachineFunction &MF = DAG.getMachineFunction();
5910 CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
5911 auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee);
5912 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5913 bool IsCalleeExternalSymbol = isa<ExternalSymbolSDNode>(Val: Callee);
5914
5915 isTailCall =
5916 isEligibleForTCO(CalleeGV: GV, CalleeCC: CallConv, CallerCC, CB, isVarArg, Outs, Ins,
5917 CallerFunc: &(MF.getFunction()), isCalleeExternalSymbol: IsCalleeExternalSymbol);
5918 if (isTailCall) {
5919 ++NumTailCalls;
5920 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5921 ++NumSiblingCalls;
5922
5923 // PC Relative calls no longer guarantee that the callee is a Global
5924 // Address Node. The callee could be an indirect tail call in which
5925 // case the SDValue for the callee could be a load (to load the address
5926 // of a function pointer) or it may be a register copy (to move the
5927 // address of the callee from a function parameter into a virtual
5928 // register). It may also be an ExternalSymbolSDNode (ex memcopy).
5929 assert((Subtarget.isUsingPCRelativeCalls() ||
5930 isa<GlobalAddressSDNode>(Callee)) &&
5931 "Callee should be an llvm::Function object.");
5932
5933 LLVM_DEBUG(dbgs() << "TCO caller: " << DAG.getMachineFunction().getName()
5934 << "\nTCO callee: ");
5935 LLVM_DEBUG(Callee.dump());
5936 }
5937 }
5938
5939 if (!isTailCall && CB && CB->isMustTailCall())
5940 report_fatal_error(reason: "failed to perform tail call elimination on a call "
5941 "site marked musttail");
5942
5943 // When long calls (i.e. indirect calls) are always used, calls are always
5944 // made via function pointer. If we have a function name, first translate it
5945 // into a pointer.
5946 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Val: Callee) &&
5947 !isTailCall)
5948 Callee = LowerGlobalAddress(Op: Callee, DAG);
5949
5950 CallFlags CFlags(
5951 CallConv, isTailCall, isVarArg, isPatchPoint,
5952 isIndirectCall(Callee, DAG, Subtarget, isPatchPoint),
5953 // hasNest
5954 Subtarget.is64BitELFABI() &&
5955 any_of(Range&: Outs, P: [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }),
5956 CLI.NoMerge);
5957
5958 if (Subtarget.isAIXABI())
5959 return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5960 InVals, CB);
5961
5962 assert(Subtarget.isSVR4ABI());
5963 if (Subtarget.isPPC64())
5964 return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5965 InVals, CB);
5966 return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5967 InVals, CB);
5968}
5969
5970SDValue PPCTargetLowering::LowerCall_32SVR4(
5971 SDValue Chain, SDValue Callee, CallFlags CFlags,
5972 const SmallVectorImpl<ISD::OutputArg> &Outs,
5973 const SmallVectorImpl<SDValue> &OutVals,
5974 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5975 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5976 const CallBase *CB) const {
5977 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
5978 // of the 32-bit SVR4 ABI stack frame layout.
5979
5980 const CallingConv::ID CallConv = CFlags.CallConv;
5981 const bool IsVarArg = CFlags.IsVarArg;
5982 const bool IsTailCall = CFlags.IsTailCall;
5983
5984 assert((CallConv == CallingConv::C ||
5985 CallConv == CallingConv::Cold ||
5986 CallConv == CallingConv::Fast) && "Unknown calling convention!");
5987
5988 const Align PtrAlign(4);
5989
5990 MachineFunction &MF = DAG.getMachineFunction();
5991
5992 // Mark this function as potentially containing a function that contains a
5993 // tail call. As a consequence the frame pointer will be used for dynamicalloc
5994 // and restoring the callers stack pointer in this functions epilog. This is
5995 // done because by tail calling the called function might overwrite the value
5996 // in this function's (MF) stack pointer stack slot 0(SP).
5997 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
5998 CallConv == CallingConv::Fast)
5999 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
6000
6001 // Count how many bytes are to be pushed on the stack, including the linkage
6002 // area, parameter list area and the part of the local variable space which
6003 // contains copies of aggregates which are passed by value.
6004
6005 // Assign locations to all of the outgoing arguments.
6006 SmallVector<CCValAssign, 16> ArgLocs;
6007 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
6008
6009 // Reserve space for the linkage area on the stack.
6010 CCInfo.AllocateStack(Size: Subtarget.getFrameLowering()->getLinkageSize(),
6011 Alignment: PtrAlign);
6012
6013 if (IsVarArg) {
6014 // Handle fixed and variable vector arguments differently.
6015 // Fixed vector arguments go into registers as long as registers are
6016 // available. Variable vector arguments always go into memory.
6017 unsigned NumArgs = Outs.size();
6018
6019 for (unsigned i = 0; i != NumArgs; ++i) {
6020 MVT ArgVT = Outs[i].VT;
6021 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
6022 bool Result;
6023
6024 if (!ArgFlags.isVarArg()) {
6025 Result = CC_PPC32_SVR4(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full, ArgFlags,
6026 OrigTy: Outs[i].OrigTy, State&: CCInfo);
6027 } else {
6028 Result = CC_PPC32_SVR4_VarArg(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full,
6029 ArgFlags, OrigTy: Outs[i].OrigTy, State&: CCInfo);
6030 }
6031
6032 if (Result) {
6033#ifndef NDEBUG
6034 errs() << "Call operand #" << i << " has unhandled type "
6035 << ArgVT << "\n";
6036#endif
6037 llvm_unreachable(nullptr);
6038 }
6039 }
6040 } else {
6041 // All arguments are treated the same.
6042 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4);
6043 }
6044
6045 // Assign locations to all of the outgoing aggregate by value arguments.
6046 SmallVector<CCValAssign, 16> ByValArgLocs;
6047 CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext());
6048
6049 // Reserve stack space for the allocations in CCInfo.
6050 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
6051
6052 CCByValInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4_ByVal);
6053
6054 // Size of the linkage area, parameter list area and the part of the local
6055 // space variable where copies of aggregates which are passed by value are
6056 // stored.
6057 unsigned NumBytes = CCByValInfo.getStackSize();
6058
6059 // Calculate by how many bytes the stack has to be adjusted in case of tail
6060 // call optimization.
6061 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: IsTailCall, ParamSize: NumBytes);
6062
6063 // Adjust the stack pointer for the new arguments...
6064 // These operations are automatically eliminated by the prolog/epilog pass
6065 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6066 SDValue CallSeqStart = Chain;
6067
6068 // Load the return address and frame pointer so it can be moved somewhere else
6069 // later.
6070 SDValue LROp, FPOp;
6071 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6072
6073 // Set up a copy of the stack pointer for use loading and storing any
6074 // arguments that may not fit in the registers available for argument
6075 // passing.
6076 SDValue StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
6077
6078 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6079 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6080 SmallVector<SDValue, 8> MemOpChains;
6081
6082 bool seenFloatArg = false;
6083 // Walk the register/memloc assignments, inserting copies/loads.
6084 // i - Tracks the index into the list of registers allocated for the call
6085 // RealArgIdx - Tracks the index into the list of actual function arguments
6086 // j - Tracks the index into the list of byval arguments
6087 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size();
6088 i != e;
6089 ++i, ++RealArgIdx) {
6090 CCValAssign &VA = ArgLocs[i];
6091 SDValue Arg = OutVals[RealArgIdx];
6092 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags;
6093
6094 if (Flags.isByVal()) {
6095 // Argument is an aggregate which is passed by value, thus we need to
6096 // create a copy of it in the local variable space of the current stack
6097 // frame (which is the stack frame of the caller) and pass the address of
6098 // this copy to the callee.
6099 assert((j < ByValArgLocs.size()) && "Index out of bounds!");
6100 CCValAssign &ByValVA = ByValArgLocs[j++];
6101 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
6102
6103 // Memory reserved in the local variable space of the callers stack frame.
6104 unsigned LocMemOffset = ByValVA.getLocMemOffset();
6105
6106 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6107 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6108 N1: StackPtr, N2: PtrOff);
6109
6110 // Create a copy of the argument in the local area of the current
6111 // stack frame.
6112 SDValue MemcpyCall =
6113 CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6114 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6115 Flags, DAG, dl);
6116
6117 // This must go outside the CALLSEQ_START..END.
6118 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: NumBytes, OutSize: 0,
6119 DL: SDLoc(MemcpyCall));
6120 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6121 To: NewCallSeqStart.getNode());
6122 Chain = CallSeqStart = NewCallSeqStart;
6123
6124 // Pass the address of the aggregate copy on the stack either in a
6125 // physical register or in the parameter list area of the current stack
6126 // frame to the callee.
6127 Arg = PtrOff;
6128 }
6129
6130 // When useCRBits() is true, there can be i1 arguments.
6131 // It is because getRegisterType(MVT::i1) => MVT::i1,
6132 // and for other integer types getRegisterType() => MVT::i32.
6133 // Extend i1 and ensure callee will get i32.
6134 if (Arg.getValueType() == MVT::i1)
6135 Arg = DAG.getNode(Opcode: Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
6136 DL: dl, VT: MVT::i32, Operand: Arg);
6137
6138 if (VA.isRegLoc()) {
6139 seenFloatArg |= VA.getLocVT().isFloatingPoint();
6140 // Put argument in a physical register.
6141 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) {
6142 bool IsLE = Subtarget.isLittleEndian();
6143 SDValue SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6144 N2: DAG.getIntPtrConstant(Val: IsLE ? 0 : 1, DL: dl));
6145 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y: SVal.getValue(R: 0)));
6146 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6147 N2: DAG.getIntPtrConstant(Val: IsLE ? 1 : 0, DL: dl));
6148 RegsToPass.push_back(Elt: std::make_pair(x: ArgLocs[++i].getLocReg(),
6149 y: SVal.getValue(R: 0)));
6150 } else
6151 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
6152 } else {
6153 // Put argument in the parameter list area of the current stack frame.
6154 assert(VA.isMemLoc());
6155 unsigned LocMemOffset = VA.getLocMemOffset();
6156
6157 if (!IsTailCall) {
6158 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6159 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6160 N1: StackPtr, N2: PtrOff);
6161
6162 MemOpChains.push_back(
6163 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
6164 } else {
6165 // Calculate and remember argument location.
6166 CalculateTailCallArgDest(DAG, MF, IsPPC64: false, Arg, SPDiff, ArgOffset: LocMemOffset,
6167 TailCallArguments);
6168 }
6169 }
6170 }
6171
6172 if (!MemOpChains.empty())
6173 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6174
6175 // Build a sequence of copy-to-reg nodes chained together with token chain
6176 // and flag operands which copy the outgoing args into the appropriate regs.
6177 SDValue InGlue;
6178 for (const auto &[Reg, N] : RegsToPass) {
6179 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6180 InGlue = Chain.getValue(R: 1);
6181 }
6182
6183 // Set CR bit 6 to true if this is a vararg call with floating args passed in
6184 // registers.
6185 if (IsVarArg) {
6186 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
6187 SDValue Ops[] = { Chain, InGlue };
6188
6189 Chain = DAG.getNode(Opcode: seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, DL: dl,
6190 VTList: VTs, Ops: ArrayRef(Ops, InGlue.getNode() ? 2 : 1));
6191
6192 InGlue = Chain.getValue(R: 1);
6193 }
6194
6195 if (IsTailCall)
6196 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6197 TailCallArguments);
6198
6199 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6200 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6201}
6202
6203// Copy an argument into memory, being careful to do this outside the
6204// call sequence for the call to which the argument belongs.
6205SDValue PPCTargetLowering::createMemcpyOutsideCallSeq(
6206 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
6207 SelectionDAG &DAG, const SDLoc &dl) const {
6208 SDValue MemcpyCall = CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6209 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6210 Flags, DAG, dl);
6211 // The MEMCPY must go outside the CALLSEQ_START..END.
6212 int64_t FrameSize = CallSeqStart.getConstantOperandVal(i: 1);
6213 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: FrameSize, OutSize: 0,
6214 DL: SDLoc(MemcpyCall));
6215 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6216 To: NewCallSeqStart.getNode());
6217 return NewCallSeqStart;
6218}
6219
6220SDValue PPCTargetLowering::LowerCall_64SVR4(
6221 SDValue Chain, SDValue Callee, CallFlags CFlags,
6222 const SmallVectorImpl<ISD::OutputArg> &Outs,
6223 const SmallVectorImpl<SDValue> &OutVals,
6224 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
6225 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
6226 const CallBase *CB) const {
6227 bool isELFv2ABI = Subtarget.isELFv2ABI();
6228 bool isLittleEndian = Subtarget.isLittleEndian();
6229 unsigned NumOps = Outs.size();
6230 bool IsSibCall = false;
6231 bool IsFastCall = CFlags.CallConv == CallingConv::Fast;
6232
6233 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
6234 unsigned PtrByteSize = 8;
6235
6236 MachineFunction &MF = DAG.getMachineFunction();
6237
6238 if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt)
6239 IsSibCall = true;
6240
6241 // Mark this function as potentially containing a function that contains a
6242 // tail call. As a consequence the frame pointer will be used for dynamicalloc
6243 // and restoring the callers stack pointer in this functions epilog. This is
6244 // done because by tail calling the called function might overwrite the value
6245 // in this function's (MF) stack pointer stack slot 0(SP).
6246 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6247 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
6248
6249 assert(!(IsFastCall && CFlags.IsVarArg) &&
6250 "fastcc not supported on varargs functions");
6251
6252 // Count how many bytes are to be pushed on the stack, including the linkage
6253 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes
6254 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage
6255 // area is 32 bytes reserved space for [SP][CR][LR][TOC].
6256 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
6257 unsigned NumBytes = LinkageSize;
6258 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
6259
6260 static const MCPhysReg GPR[] = {
6261 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6262 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
6263 };
6264 static const MCPhysReg VR[] = {
6265 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
6266 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
6267 };
6268
6269 const unsigned NumGPRs = std::size(GPR);
6270 const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
6271 const unsigned NumVRs = std::size(VR);
6272
6273 // On ELFv2, we can avoid allocating the parameter area if all the arguments
6274 // can be passed to the callee in registers.
6275 // For the fast calling convention, there is another check below.
6276 // Note: We should keep consistent with LowerFormalArguments_64SVR4()
6277 bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall;
6278 if (!HasParameterArea) {
6279 unsigned ParamAreaSize = NumGPRs * PtrByteSize;
6280 unsigned AvailableFPRs = NumFPRs;
6281 unsigned AvailableVRs = NumVRs;
6282 unsigned NumBytesTmp = NumBytes;
6283 for (unsigned i = 0; i != NumOps; ++i) {
6284 if (Outs[i].Flags.isNest()) continue;
6285 if (CalculateStackSlotUsed(ArgVT: Outs[i].VT, OrigVT: Outs[i].ArgVT, Flags: Outs[i].Flags,
6286 PtrByteSize, LinkageSize, ParamAreaSize,
6287 ArgOffset&: NumBytesTmp, AvailableFPRs, AvailableVRs))
6288 HasParameterArea = true;
6289 }
6290 }
6291
6292 // When using the fast calling convention, we don't provide backing for
6293 // arguments that will be in registers.
6294 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0;
6295
6296 // Avoid allocating parameter area for fastcc functions if all the arguments
6297 // can be passed in the registers.
6298 if (IsFastCall)
6299 HasParameterArea = false;
6300
6301 // Add up all the space actually used.
6302 for (unsigned i = 0; i != NumOps; ++i) {
6303 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6304 EVT ArgVT = Outs[i].VT;
6305 EVT OrigVT = Outs[i].ArgVT;
6306
6307 if (Flags.isNest())
6308 continue;
6309
6310 if (IsFastCall) {
6311 if (Flags.isByVal()) {
6312 NumGPRsUsed += (Flags.getByValSize()+7)/8;
6313 if (NumGPRsUsed > NumGPRs)
6314 HasParameterArea = true;
6315 } else {
6316 switch (ArgVT.getSimpleVT().SimpleTy) {
6317 default: llvm_unreachable("Unexpected ValueType for argument!");
6318 case MVT::i1:
6319 case MVT::i32:
6320 case MVT::i64:
6321 if (++NumGPRsUsed <= NumGPRs)
6322 continue;
6323 break;
6324 case MVT::v4i32:
6325 case MVT::v8i16:
6326 case MVT::v16i8:
6327 case MVT::v2f64:
6328 case MVT::v2i64:
6329 case MVT::v1i128:
6330 case MVT::f128:
6331 if (++NumVRsUsed <= NumVRs)
6332 continue;
6333 break;
6334 case MVT::v4f32:
6335 if (++NumVRsUsed <= NumVRs)
6336 continue;
6337 break;
6338 case MVT::f32:
6339 case MVT::f64:
6340 if (++NumFPRsUsed <= NumFPRs)
6341 continue;
6342 break;
6343 }
6344 HasParameterArea = true;
6345 }
6346 }
6347
6348 /* Respect alignment of argument on the stack. */
6349 auto Alignement =
6350 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6351 NumBytes = alignTo(Size: NumBytes, A: Alignement);
6352
6353 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
6354 if (Flags.isInConsecutiveRegsLast())
6355 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6356 }
6357
6358 unsigned NumBytesActuallyUsed = NumBytes;
6359
6360 // In the old ELFv1 ABI,
6361 // the prolog code of the callee may store up to 8 GPR argument registers to
6362 // the stack, allowing va_start to index over them in memory if its varargs.
6363 // Because we cannot tell if this is needed on the caller side, we have to
6364 // conservatively assume that it is needed. As such, make sure we have at
6365 // least enough stack space for the caller to store the 8 GPRs.
6366 // In the ELFv2 ABI, we allocate the parameter area iff a callee
6367 // really requires memory operands, e.g. a vararg function.
6368 if (HasParameterArea)
6369 NumBytes = std::max(a: NumBytes, b: LinkageSize + 8 * PtrByteSize);
6370 else
6371 NumBytes = LinkageSize;
6372
6373 // Tail call needs the stack to be aligned.
6374 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6375 NumBytes = EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes);
6376
6377 int SPDiff = 0;
6378
6379 // Calculate by how many bytes the stack has to be adjusted in case of tail
6380 // call optimization.
6381 if (!IsSibCall)
6382 SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: CFlags.IsTailCall, ParamSize: NumBytes);
6383
6384 // To protect arguments on the stack from being clobbered in a tail call,
6385 // force all the loads to happen before doing any other lowering.
6386 if (CFlags.IsTailCall)
6387 Chain = DAG.getStackArgumentTokenFactor(Chain);
6388
6389 // Adjust the stack pointer for the new arguments...
6390 // These operations are automatically eliminated by the prolog/epilog pass
6391 if (!IsSibCall)
6392 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6393 SDValue CallSeqStart = Chain;
6394
6395 // Load the return address and frame pointer so it can be move somewhere else
6396 // later.
6397 SDValue LROp, FPOp;
6398 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6399
6400 // Set up a copy of the stack pointer for use loading and storing any
6401 // arguments that may not fit in the registers available for argument
6402 // passing.
6403 SDValue StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
6404
6405 // Figure out which arguments are going to go in registers, and which in
6406 // memory. Also, if this is a vararg function, floating point operations
6407 // must be stored to our stack, and loaded into integer regs as well, if
6408 // any integer regs are available for argument passing.
6409 unsigned ArgOffset = LinkageSize;
6410
6411 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6412 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6413
6414 SmallVector<SDValue, 8> MemOpChains;
6415 for (unsigned i = 0; i != NumOps; ++i) {
6416 SDValue Arg = OutVals[i];
6417 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6418 EVT ArgVT = Outs[i].VT;
6419 EVT OrigVT = Outs[i].ArgVT;
6420
6421 // PtrOff will be used to store the current argument to the stack if a
6422 // register cannot be found for it.
6423 SDValue PtrOff;
6424
6425 // We re-align the argument offset for each argument, except when using the
6426 // fast calling convention, when we need to make sure we do that only when
6427 // we'll actually use a stack slot.
6428 auto ComputePtrOff = [&]() {
6429 /* Respect alignment of argument on the stack. */
6430 auto Alignment =
6431 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6432 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
6433
6434 PtrOff = DAG.getConstant(Val: ArgOffset, DL: dl, VT: StackPtr.getValueType());
6435
6436 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6437 };
6438
6439 if (!IsFastCall) {
6440 ComputePtrOff();
6441
6442 /* Compute GPR index associated with argument offset. */
6443 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
6444 GPR_idx = std::min(a: GPR_idx, b: NumGPRs);
6445 }
6446
6447 // Promote integers to 64-bit values.
6448 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) {
6449 // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
6450 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
6451 Arg = DAG.getNode(Opcode: ExtOp, DL: dl, VT: MVT::i64, Operand: Arg);
6452 }
6453
6454 // FIXME memcpy is used way more than necessary. Correctness first.
6455 // Note: "by value" is code for passing a structure by value, not
6456 // basic types.
6457 if (Flags.isByVal()) {
6458 // Note: Size includes alignment padding, so
6459 // struct x { short a; char b; }
6460 // will have Size = 4. With #pragma pack(1), it will have Size = 3.
6461 // These are the proper values we need for right-justifying the
6462 // aggregate in a parameter register.
6463 unsigned Size = Flags.getByValSize();
6464
6465 // An empty aggregate parameter takes up no storage and no
6466 // registers.
6467 if (Size == 0)
6468 continue;
6469
6470 if (IsFastCall)
6471 ComputePtrOff();
6472
6473 // All aggregates smaller than 8 bytes must be passed right-justified.
6474 if (Size==1 || Size==2 || Size==4) {
6475 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
6476 if (GPR_idx != NumGPRs) {
6477 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: Arg,
6478 PtrInfo: MachinePointerInfo(), MemVT: VT);
6479 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6480 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6481
6482 ArgOffset += PtrByteSize;
6483 continue;
6484 }
6485 }
6486
6487 if (GPR_idx == NumGPRs && Size < 8) {
6488 SDValue AddPtr = PtrOff;
6489 if (!isLittleEndian) {
6490 SDValue Const = DAG.getConstant(Val: PtrByteSize - Size, DL: dl,
6491 VT: PtrOff.getValueType());
6492 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6493 }
6494 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6495 CallSeqStart,
6496 Flags, DAG, dl);
6497 ArgOffset += PtrByteSize;
6498 continue;
6499 }
6500 // Copy the object to parameter save area if it can not be entirely passed
6501 // by registers.
6502 // FIXME: we only need to copy the parts which need to be passed in
6503 // parameter save area. For the parts passed by registers, we don't need
6504 // to copy them to the stack although we need to allocate space for them
6505 // in parameter save area.
6506 if ((NumGPRs - GPR_idx) * PtrByteSize < Size)
6507 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
6508 CallSeqStart,
6509 Flags, DAG, dl);
6510
6511 // When a register is available, pass a small aggregate right-justified.
6512 if (Size < 8 && GPR_idx != NumGPRs) {
6513 // The easiest way to get this right-justified in a register
6514 // is to copy the structure into the rightmost portion of a
6515 // local variable slot, then load the whole slot into the
6516 // register.
6517 // FIXME: The memcpy seems to produce pretty awful code for
6518 // small aggregates, particularly for packed ones.
6519 // FIXME: It would be preferable to use the slot in the
6520 // parameter save area instead of a new local variable.
6521 SDValue AddPtr = PtrOff;
6522 if (!isLittleEndian) {
6523 SDValue Const = DAG.getConstant(Val: 8 - Size, DL: dl, VT: PtrOff.getValueType());
6524 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6525 }
6526 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6527 CallSeqStart,
6528 Flags, DAG, dl);
6529
6530 // Load the slot into the register.
6531 SDValue Load =
6532 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6533 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6534 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6535
6536 // Done with this argument.
6537 ArgOffset += PtrByteSize;
6538 continue;
6539 }
6540
6541 // For aggregates larger than PtrByteSize, copy the pieces of the
6542 // object that fit into registers from the parameter save area.
6543 for (unsigned j=0; j<Size; j+=PtrByteSize) {
6544 SDValue Const = DAG.getConstant(Val: j, DL: dl, VT: PtrOff.getValueType());
6545 SDValue AddArg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Arg, N2: Const);
6546 if (GPR_idx != NumGPRs) {
6547 unsigned LoadSizeInBits = std::min(a: PtrByteSize, b: (Size - j)) * 8;
6548 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadSizeInBits);
6549 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: AddArg,
6550 PtrInfo: MachinePointerInfo(), MemVT: ObjType);
6551
6552 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6553 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6554 ArgOffset += PtrByteSize;
6555 } else {
6556 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
6557 break;
6558 }
6559 }
6560 continue;
6561 }
6562
6563 switch (Arg.getSimpleValueType().SimpleTy) {
6564 default: llvm_unreachable("Unexpected ValueType for argument!");
6565 case MVT::i1:
6566 case MVT::i32:
6567 case MVT::i64:
6568 if (Flags.isNest()) {
6569 // The 'nest' parameter, if any, is passed in R11.
6570 RegsToPass.push_back(Elt: std::make_pair(x: PPC::X11, y&: Arg));
6571 break;
6572 }
6573
6574 // These can be scalar arguments or elements of an integer array type
6575 // passed directly. Clang may use those instead of "byval" aggregate
6576 // types to avoid forcing arguments to memory unnecessarily.
6577 if (GPR_idx != NumGPRs) {
6578 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Arg));
6579 } else {
6580 if (IsFastCall)
6581 ComputePtrOff();
6582
6583 assert(HasParameterArea &&
6584 "Parameter area must exist to pass an argument in memory.");
6585 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6586 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6587 TailCallArguments, dl);
6588 if (IsFastCall)
6589 ArgOffset += PtrByteSize;
6590 }
6591 if (!IsFastCall)
6592 ArgOffset += PtrByteSize;
6593 break;
6594 case MVT::f32:
6595 case MVT::f64: {
6596 // These can be scalar arguments or elements of a float array type
6597 // passed directly. The latter are used to implement ELFv2 homogenous
6598 // float aggregates.
6599
6600 // Named arguments go into FPRs first, and once they overflow, the
6601 // remaining arguments go into GPRs and then the parameter save area.
6602 // Unnamed arguments for vararg functions always go to GPRs and
6603 // then the parameter save area. For now, put all arguments to vararg
6604 // routines always in both locations (FPR *and* GPR or stack slot).
6605 bool NeedGPROrStack = CFlags.IsVarArg || FPR_idx == NumFPRs;
6606 bool NeededLoad = false;
6607
6608 // First load the argument into the next available FPR.
6609 if (FPR_idx != NumFPRs)
6610 RegsToPass.push_back(Elt: std::make_pair(x: FPR[FPR_idx++], y&: Arg));
6611
6612 // Next, load the argument into GPR or stack slot if needed.
6613 if (!NeedGPROrStack)
6614 ;
6615 else if (GPR_idx != NumGPRs && !IsFastCall) {
6616 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
6617 // once we support fp <-> gpr moves.
6618
6619 // In the non-vararg case, this can only ever happen in the
6620 // presence of f32 array types, since otherwise we never run
6621 // out of FPRs before running out of GPRs.
6622 SDValue ArgVal;
6623
6624 // Double values are always passed in a single GPR.
6625 if (Arg.getValueType() != MVT::f32) {
6626 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: Arg);
6627
6628 // Non-array float values are extended and passed in a GPR.
6629 } else if (!Flags.isInConsecutiveRegs()) {
6630 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6631 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6632
6633 // If we have an array of floats, we collect every odd element
6634 // together with its predecessor into one GPR.
6635 } else if (ArgOffset % PtrByteSize != 0) {
6636 SDValue Lo, Hi;
6637 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: OutVals[i - 1]);
6638 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6639 if (!isLittleEndian)
6640 std::swap(a&: Lo, b&: Hi);
6641 ArgVal = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6642
6643 // The final element, if even, goes into the first half of a GPR.
6644 } else if (Flags.isInConsecutiveRegsLast()) {
6645 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6646 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6647 if (!isLittleEndian)
6648 ArgVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: ArgVal,
6649 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
6650
6651 // Non-final even elements are skipped; they will be handled
6652 // together the with subsequent argument on the next go-around.
6653 } else
6654 ArgVal = SDValue();
6655
6656 if (ArgVal.getNode())
6657 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: ArgVal));
6658 } else {
6659 if (IsFastCall)
6660 ComputePtrOff();
6661
6662 // Single-precision floating-point values are mapped to the
6663 // second (rightmost) word of the stack doubleword.
6664 if (Arg.getValueType() == MVT::f32 &&
6665 !isLittleEndian && !Flags.isInConsecutiveRegs()) {
6666 SDValue ConstFour = DAG.getConstant(Val: 4, DL: dl, VT: PtrOff.getValueType());
6667 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: ConstFour);
6668 }
6669
6670 assert(HasParameterArea &&
6671 "Parameter area must exist to pass an argument in memory.");
6672 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6673 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6674 TailCallArguments, dl);
6675
6676 NeededLoad = true;
6677 }
6678 // When passing an array of floats, the array occupies consecutive
6679 // space in the argument area; only round up to the next doubleword
6680 // at the end of the array. Otherwise, each float takes 8 bytes.
6681 if (!IsFastCall || NeededLoad) {
6682 ArgOffset += (Arg.getValueType() == MVT::f32 &&
6683 Flags.isInConsecutiveRegs()) ? 4 : 8;
6684 if (Flags.isInConsecutiveRegsLast())
6685 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6686 }
6687 break;
6688 }
6689 case MVT::v4f32:
6690 case MVT::v4i32:
6691 case MVT::v8i16:
6692 case MVT::v16i8:
6693 case MVT::v2f64:
6694 case MVT::v2i64:
6695 case MVT::v1i128:
6696 case MVT::f128:
6697 // These can be scalar arguments or elements of a vector array type
6698 // passed directly. The latter are used to implement ELFv2 homogenous
6699 // vector aggregates.
6700
6701 // For a varargs call, named arguments go into VRs or on the stack as
6702 // usual; unnamed arguments always go to the stack or the corresponding
6703 // GPRs when within range. For now, we always put the value in both
6704 // locations (or even all three).
6705 if (CFlags.IsVarArg) {
6706 assert(HasParameterArea &&
6707 "Parameter area must exist if we have a varargs call.");
6708 // We could elide this store in the case where the object fits
6709 // entirely in R registers. Maybe later.
6710 SDValue Store =
6711 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6712 MemOpChains.push_back(Elt: Store);
6713 if (VR_idx != NumVRs) {
6714 SDValue Load =
6715 DAG.getLoad(VT: MVT::v4f32, dl, Chain: Store, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6716 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6717 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Load));
6718 }
6719 ArgOffset += 16;
6720 for (unsigned i=0; i<16; i+=PtrByteSize) {
6721 if (GPR_idx == NumGPRs)
6722 break;
6723 SDValue Ix = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
6724 N2: DAG.getConstant(Val: i, DL: dl, VT: PtrVT));
6725 SDValue Load =
6726 DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Ix, PtrInfo: MachinePointerInfo());
6727 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6728 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6729 }
6730 break;
6731 }
6732
6733 // Non-varargs Altivec params go into VRs or on the stack.
6734 if (VR_idx != NumVRs) {
6735 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Arg));
6736 } else {
6737 if (IsFastCall)
6738 ComputePtrOff();
6739
6740 assert(HasParameterArea &&
6741 "Parameter area must exist to pass an argument in memory.");
6742 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6743 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: true, MemOpChains,
6744 TailCallArguments, dl);
6745 if (IsFastCall)
6746 ArgOffset += 16;
6747 }
6748
6749 if (!IsFastCall)
6750 ArgOffset += 16;
6751 break;
6752 }
6753 }
6754
6755 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) &&
6756 "mismatch in size of parameter area");
6757 (void)NumBytesActuallyUsed;
6758
6759 if (!MemOpChains.empty())
6760 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6761
6762 // Check if this is an indirect call (MTCTR/BCTRL).
6763 // See prepareDescriptorIndirectCall and buildCallOperands for more
6764 // information about calls through function pointers in the 64-bit SVR4 ABI.
6765 if (CFlags.IsIndirect) {
6766 // For 64-bit ELFv2 ABI with PCRel, do not save the TOC of the
6767 // caller in the TOC save area.
6768 if (isTOCSaveRestoreRequired(Subtarget)) {
6769 assert(!CFlags.IsTailCall && "Indirect tails calls not supported");
6770 // Load r2 into a virtual register and store it to the TOC save area.
6771 setUsesTOCBasePtr(DAG);
6772 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: PPC::X2, VT: MVT::i64);
6773 // TOC save area offset.
6774 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
6775 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
6776 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6777 Chain = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
6778 PtrInfo: MachinePointerInfo::getStack(
6779 MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
6780 }
6781 // In the ELFv2 ABI, R12 must contain the address of an indirect callee.
6782 // This does not mean the MTCTR instruction must use R12; it's easier
6783 // to model this as an extra parameter, so do that.
6784 if (isELFv2ABI && !CFlags.IsPatchPoint)
6785 RegsToPass.push_back(Elt: std::make_pair(x: (unsigned)PPC::X12, y&: Callee));
6786 }
6787
6788 // Build a sequence of copy-to-reg nodes chained together with token chain
6789 // and flag operands which copy the outgoing args into the appropriate regs.
6790 SDValue InGlue;
6791 for (const auto &[Reg, N] : RegsToPass) {
6792 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6793 InGlue = Chain.getValue(R: 1);
6794 }
6795
6796 if (CFlags.IsTailCall && !IsSibCall)
6797 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6798 TailCallArguments);
6799
6800 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6801 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6802}
6803
6804// Returns true when the shadow of a general purpose argument register
6805// in the parameter save area is aligned to at least 'RequiredAlign'.
6806static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) {
6807 assert(RequiredAlign.value() <= 16 &&
6808 "Required alignment greater than stack alignment.");
6809 switch (Reg) {
6810 default:
6811 report_fatal_error(reason: "called on invalid register.");
6812 case PPC::R5:
6813 case PPC::R9:
6814 case PPC::X3:
6815 case PPC::X5:
6816 case PPC::X7:
6817 case PPC::X9:
6818 // These registers are 16 byte aligned which is the most strict aligment
6819 // we can support.
6820 return true;
6821 case PPC::R3:
6822 case PPC::R7:
6823 case PPC::X4:
6824 case PPC::X6:
6825 case PPC::X8:
6826 case PPC::X10:
6827 // The shadow of these registers in the PSA is 8 byte aligned.
6828 return RequiredAlign <= 8;
6829 case PPC::R4:
6830 case PPC::R6:
6831 case PPC::R8:
6832 case PPC::R10:
6833 return RequiredAlign <= 4;
6834 }
6835}
6836
6837static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
6838 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
6839 Type *OrigTy, CCState &State) {
6840 const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(
6841 State.getMachineFunction().getSubtarget());
6842 const bool IsPPC64 = Subtarget.isPPC64();
6843 const unsigned PtrSize = IsPPC64 ? 8 : 4;
6844 const Align PtrAlign(PtrSize);
6845 const Align StackAlign(16);
6846 const MVT RegVT = Subtarget.getScalarIntVT();
6847
6848 if (ValVT == MVT::f128)
6849 report_fatal_error(reason: "f128 is unimplemented on AIX.");
6850
6851 static const MCPhysReg GPR_32[] = {// 32-bit registers.
6852 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
6853 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
6854 static const MCPhysReg GPR_64[] = {// 64-bit registers.
6855 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6856 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
6857
6858 static const MCPhysReg VR[] = {// Vector registers.
6859 PPC::V2, PPC::V3, PPC::V4, PPC::V5,
6860 PPC::V6, PPC::V7, PPC::V8, PPC::V9,
6861 PPC::V10, PPC::V11, PPC::V12, PPC::V13};
6862
6863 const ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32;
6864
6865 if (ArgFlags.isNest()) {
6866 MCRegister EnvReg = State.AllocateReg(Reg: IsPPC64 ? PPC::X11 : PPC::R11);
6867 if (!EnvReg)
6868 report_fatal_error(reason: "More then one nest argument.");
6869 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: EnvReg, LocVT: RegVT, HTP: LocInfo));
6870 return false;
6871 }
6872
6873 if (ArgFlags.isByVal()) {
6874 const Align ByValAlign(ArgFlags.getNonZeroByValAlign());
6875 if (ByValAlign > StackAlign)
6876 report_fatal_error(reason: "Pass-by-value arguments with alignment greater than "
6877 "16 are not supported.");
6878
6879 const unsigned ByValSize = ArgFlags.getByValSize();
6880 const Align ObjAlign = ByValAlign > PtrAlign ? ByValAlign : PtrAlign;
6881
6882 // An empty aggregate parameter takes up no storage and no registers,
6883 // but needs a MemLoc for a stack slot for the formal arguments side.
6884 if (ByValSize == 0) {
6885 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6886 Offset: State.getStackSize(), LocVT: RegVT, HTP: LocInfo));
6887 return false;
6888 }
6889
6890 // Shadow allocate any registers that are not properly aligned.
6891 unsigned NextReg = State.getFirstUnallocated(Regs: GPRs);
6892 while (NextReg != GPRs.size() &&
6893 !isGPRShadowAligned(Reg: GPRs[NextReg], RequiredAlign: ObjAlign)) {
6894 // Shadow allocate next registers since its aligment is not strict enough.
6895 MCRegister Reg = State.AllocateReg(Regs: GPRs);
6896 // Allocate the stack space shadowed by said register.
6897 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6898 assert(Reg && "Alocating register unexpectedly failed.");
6899 (void)Reg;
6900 NextReg = State.getFirstUnallocated(Regs: GPRs);
6901 }
6902
6903 const unsigned StackSize = alignTo(Size: ByValSize, A: ObjAlign);
6904 unsigned Offset = State.AllocateStack(Size: StackSize, Alignment: ObjAlign);
6905 for (const unsigned E = Offset + StackSize; Offset < E; Offset += PtrSize) {
6906 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6907 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6908 else {
6909 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6910 Offset, LocVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6911 HTP: LocInfo));
6912 break;
6913 }
6914 }
6915 return false;
6916 }
6917
6918 // Arguments always reserve parameter save area.
6919 switch (ValVT.SimpleTy) {
6920 default:
6921 report_fatal_error(reason: "Unhandled value type for argument.");
6922 case MVT::i64:
6923 // i64 arguments should have been split to i32 for PPC32.
6924 assert(IsPPC64 && "PPC32 should have split i64 values.");
6925 [[fallthrough]];
6926 case MVT::i1:
6927 case MVT::i32: {
6928 const unsigned Offset = State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6929 // AIX integer arguments are always passed in register width.
6930 if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits())
6931 LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt
6932 : CCValAssign::LocInfo::ZExt;
6933 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6934 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6935 else
6936 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT: RegVT, HTP: LocInfo));
6937
6938 return false;
6939 }
6940 case MVT::f32:
6941 case MVT::f64: {
6942 // Parameter save area (PSA) is reserved even if the float passes in fpr.
6943 const unsigned StoreSize = LocVT.getStoreSize();
6944 // Floats are always 4-byte aligned in the PSA on AIX.
6945 // This includes f64 in 64-bit mode for ABI compatibility.
6946 const unsigned Offset =
6947 State.AllocateStack(Size: IsPPC64 ? 8 : StoreSize, Alignment: Align(4));
6948 MCRegister FReg = State.AllocateReg(Regs: FPR);
6949 if (FReg)
6950 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: FReg, LocVT, HTP: LocInfo));
6951
6952 // Reserve and initialize GPRs or initialize the PSA as required.
6953 for (unsigned I = 0; I < StoreSize; I += PtrSize) {
6954 if (MCRegister Reg = State.AllocateReg(Regs: GPRs)) {
6955 assert(FReg && "An FPR should be available when a GPR is reserved.");
6956 if (State.isVarArg()) {
6957 // Successfully reserved GPRs are only initialized for vararg calls.
6958 // Custom handling is required for:
6959 // f64 in PPC32 needs to be split into 2 GPRs.
6960 // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR.
6961 State.addLoc(
6962 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6963 }
6964 } else {
6965 // If there are insufficient GPRs, the PSA needs to be initialized.
6966 // Initialization occurs even if an FPR was initialized for
6967 // compatibility with the AIX XL compiler. The full memory for the
6968 // argument will be initialized even if a prior word is saved in GPR.
6969 // A custom memLoc is used when the argument also passes in FPR so
6970 // that the callee handling can skip over it easily.
6971 State.addLoc(
6972 V: FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT,
6973 HTP: LocInfo)
6974 : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
6975 break;
6976 }
6977 }
6978
6979 return false;
6980 }
6981 case MVT::v4f32:
6982 case MVT::v4i32:
6983 case MVT::v8i16:
6984 case MVT::v16i8:
6985 case MVT::v2i64:
6986 case MVT::v2f64:
6987 case MVT::v1i128: {
6988 const unsigned VecSize = 16;
6989 const Align VecAlign(VecSize);
6990
6991 if (!State.isVarArg()) {
6992 // If there are vector registers remaining we don't consume any stack
6993 // space.
6994 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
6995 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
6996 return false;
6997 }
6998 // Vectors passed on the stack do not shadow GPRs or FPRs even though they
6999 // might be allocated in the portion of the PSA that is shadowed by the
7000 // GPRs.
7001 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7002 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7003 return false;
7004 }
7005
7006 unsigned NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7007 // Burn any underaligned registers and their shadowed stack space until
7008 // we reach the required alignment.
7009 while (NextRegIndex != GPRs.size() &&
7010 !isGPRShadowAligned(Reg: GPRs[NextRegIndex], RequiredAlign: VecAlign)) {
7011 // Shadow allocate register and its stack shadow.
7012 MCRegister Reg = State.AllocateReg(Regs: GPRs);
7013 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
7014 assert(Reg && "Allocating register unexpectedly failed.");
7015 (void)Reg;
7016 NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7017 }
7018
7019 // Vectors that are passed as fixed arguments are handled differently.
7020 // They are passed in VRs if any are available (unlike arguments passed
7021 // through ellipses) and shadow GPRs (unlike arguments to non-vaarg
7022 // functions)
7023 if (!ArgFlags.isVarArg()) {
7024 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
7025 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
7026 // Shadow allocate GPRs and stack space even though we pass in a VR.
7027 for (unsigned I = 0; I != VecSize; I += PtrSize)
7028 State.AllocateReg(Regs: GPRs);
7029 State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7030 return false;
7031 }
7032 // No vector registers remain so pass on the stack.
7033 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7034 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7035 return false;
7036 }
7037
7038 // If all GPRS are consumed then we pass the argument fully on the stack.
7039 if (NextRegIndex == GPRs.size()) {
7040 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7041 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7042 return false;
7043 }
7044
7045 // Corner case for 32-bit codegen. We have 2 registers to pass the first
7046 // half of the argument, and then need to pass the remaining half on the
7047 // stack.
7048 if (GPRs[NextRegIndex] == PPC::R9) {
7049 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7050 State.addLoc(
7051 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7052
7053 const MCRegister FirstReg = State.AllocateReg(Reg: PPC::R9);
7054 const MCRegister SecondReg = State.AllocateReg(Reg: PPC::R10);
7055 assert(FirstReg && SecondReg &&
7056 "Allocating R9 or R10 unexpectedly failed.");
7057 State.addLoc(
7058 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: FirstReg, LocVT: RegVT, HTP: LocInfo));
7059 State.addLoc(
7060 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: SecondReg, LocVT: RegVT, HTP: LocInfo));
7061 return false;
7062 }
7063
7064 // We have enough GPRs to fully pass the vector argument, and we have
7065 // already consumed any underaligned registers. Start with the custom
7066 // MemLoc and then the custom RegLocs.
7067 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7068 State.addLoc(
7069 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7070 for (unsigned I = 0; I != VecSize; I += PtrSize) {
7071 const MCRegister Reg = State.AllocateReg(Regs: GPRs);
7072 assert(Reg && "Failed to allocated register for vararg vector argument");
7073 State.addLoc(
7074 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
7075 }
7076 return false;
7077 }
7078 }
7079 return true;
7080}
7081
7082// So far, this function is only used by LowerFormalArguments_AIX()
7083static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT,
7084 bool IsPPC64,
7085 bool HasP8Vector,
7086 bool HasVSX) {
7087 assert((IsPPC64 || SVT != MVT::i64) &&
7088 "i64 should have been split for 32-bit codegen.");
7089
7090 switch (SVT) {
7091 default:
7092 report_fatal_error(reason: "Unexpected value type for formal argument");
7093 case MVT::i1:
7094 case MVT::i32:
7095 case MVT::i64:
7096 return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7097 case MVT::f32:
7098 return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass;
7099 case MVT::f64:
7100 return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass;
7101 case MVT::v4f32:
7102 case MVT::v4i32:
7103 case MVT::v8i16:
7104 case MVT::v16i8:
7105 case MVT::v2i64:
7106 case MVT::v2f64:
7107 case MVT::v1i128:
7108 return &PPC::VRRCRegClass;
7109 }
7110}
7111
7112static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT,
7113 SelectionDAG &DAG, SDValue ArgValue,
7114 MVT LocVT, const SDLoc &dl) {
7115 assert(ValVT.isScalarInteger() && LocVT.isScalarInteger());
7116 assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits());
7117
7118 if (Flags.isSExt())
7119 ArgValue = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: LocVT, N1: ArgValue,
7120 N2: DAG.getValueType(ValVT));
7121 else if (Flags.isZExt())
7122 ArgValue = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: LocVT, N1: ArgValue,
7123 N2: DAG.getValueType(ValVT));
7124
7125 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ValVT, Operand: ArgValue);
7126}
7127
7128static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
7129 const unsigned LASize = FL->getLinkageSize();
7130
7131 if (PPC::GPRCRegClass.contains(Reg)) {
7132 assert(Reg >= PPC::R3 && Reg <= PPC::R10 &&
7133 "Reg must be a valid argument register!");
7134 return LASize + 4 * (Reg - PPC::R3);
7135 }
7136
7137 if (PPC::G8RCRegClass.contains(Reg)) {
7138 assert(Reg >= PPC::X3 && Reg <= PPC::X10 &&
7139 "Reg must be a valid argument register!");
7140 return LASize + 8 * (Reg - PPC::X3);
7141 }
7142
7143 llvm_unreachable("Only general purpose registers expected.");
7144}
7145
7146// AIX ABI Stack Frame Layout:
7147//
7148// Low Memory +--------------------------------------------+
7149// SP +---> | Back chain | ---+
7150// | +--------------------------------------------+ |
7151// | | Saved Condition Register | |
7152// | +--------------------------------------------+ |
7153// | | Saved Linkage Register | |
7154// | +--------------------------------------------+ | Linkage Area
7155// | | Reserved for compilers | |
7156// | +--------------------------------------------+ |
7157// | | Reserved for binders | |
7158// | +--------------------------------------------+ |
7159// | | Saved TOC pointer | ---+
7160// | +--------------------------------------------+
7161// | | Parameter save area |
7162// | +--------------------------------------------+
7163// | | Alloca space |
7164// | +--------------------------------------------+
7165// | | Local variable space |
7166// | +--------------------------------------------+
7167// | | Float/int conversion temporary |
7168// | +--------------------------------------------+
7169// | | Save area for AltiVec registers |
7170// | +--------------------------------------------+
7171// | | AltiVec alignment padding |
7172// | +--------------------------------------------+
7173// | | Save area for VRSAVE register |
7174// | +--------------------------------------------+
7175// | | Save area for General Purpose registers |
7176// | +--------------------------------------------+
7177// | | Save area for Floating Point registers |
7178// | +--------------------------------------------+
7179// +---- | Back chain |
7180// High Memory +--------------------------------------------+
7181//
7182// Specifications:
7183// AIX 7.2 Assembler Language Reference
7184// Subroutine linkage convention
7185
7186SDValue PPCTargetLowering::LowerFormalArguments_AIX(
7187 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
7188 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7189 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
7190
7191 assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold ||
7192 CallConv == CallingConv::Fast) &&
7193 "Unexpected calling convention!");
7194
7195 if (getTargetMachine().Options.GuaranteedTailCallOpt)
7196 report_fatal_error(reason: "Tail call support is unimplemented on AIX.");
7197
7198 if (useSoftFloat())
7199 report_fatal_error(reason: "Soft float support is unimplemented on AIX.");
7200
7201 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7202
7203 const bool IsPPC64 = Subtarget.isPPC64();
7204 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7205
7206 // Assign locations to all of the incoming arguments.
7207 SmallVector<CCValAssign, 16> ArgLocs;
7208 MachineFunction &MF = DAG.getMachineFunction();
7209 MachineFrameInfo &MFI = MF.getFrameInfo();
7210 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
7211 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
7212
7213 const EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
7214 // Reserve space for the linkage area on the stack.
7215 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7216 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7217 uint64_t SaveStackPos = CCInfo.getStackSize();
7218 bool SaveParams = MF.getFunction().hasFnAttribute(Kind: "save-reg-params");
7219 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_AIX);
7220
7221 SmallVector<SDValue, 8> MemOps;
7222
7223 for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) {
7224 CCValAssign &VA = ArgLocs[I++];
7225 MVT LocVT = VA.getLocVT();
7226 MVT ValVT = VA.getValVT();
7227 ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags;
7228
7229 EVT ArgVT = Ins[VA.getValNo()].ArgVT;
7230 bool ArgSignExt = Ins[VA.getValNo()].Flags.isSExt();
7231 // For compatibility with the AIX XL compiler, the float args in the
7232 // parameter save area are initialized even if the argument is available
7233 // in register. The caller is required to initialize both the register
7234 // and memory, however, the callee can choose to expect it in either.
7235 // The memloc is dismissed here because the argument is retrieved from
7236 // the register.
7237 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint())
7238 continue;
7239
7240 if (SaveParams && VA.isRegLoc() && !Flags.isByVal() && !VA.needsCustom()) {
7241 const TargetRegisterClass *RegClass = getRegClassForSVT(
7242 SVT: LocVT.SimpleTy, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(), HasVSX: Subtarget.hasVSX());
7243 // On PPC64, debugger assumes extended 8-byte values are stored from GPR.
7244 MVT SaveVT = RegClass == &PPC::G8RCRegClass ? MVT::i64 : LocVT;
7245 const Register VReg = MF.addLiveIn(PReg: VA.getLocReg(), RC: RegClass);
7246 SDValue Parm = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: SaveVT);
7247 int FI = MFI.CreateFixedObject(Size: SaveVT.getStoreSize(), SPOffset: SaveStackPos, IsImmutable: true);
7248 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7249 SDValue StoreReg = DAG.getStore(Chain, dl, Val: Parm, Ptr: FIN,
7250 PtrInfo: MachinePointerInfo(), Alignment: Align(PtrByteSize));
7251 SaveStackPos = alignTo(Value: SaveStackPos + SaveVT.getStoreSize(), Align: PtrByteSize);
7252 MemOps.push_back(Elt: StoreReg);
7253 }
7254
7255 if (SaveParams && (VA.isMemLoc() || Flags.isByVal()) && !VA.needsCustom()) {
7256 unsigned StoreSize =
7257 Flags.isByVal() ? Flags.getByValSize() : LocVT.getStoreSize();
7258 SaveStackPos = alignTo(Value: SaveStackPos + StoreSize, Align: PtrByteSize);
7259 }
7260
7261 auto HandleMemLoc = [&]() {
7262 const unsigned LocSize = LocVT.getStoreSize();
7263 const unsigned ValSize = ValVT.getStoreSize();
7264 assert((ValSize <= LocSize) &&
7265 "Object size is larger than size of MemLoc");
7266 int CurArgOffset = VA.getLocMemOffset();
7267 // Objects are right-justified because AIX is big-endian.
7268 if (LocSize > ValSize)
7269 CurArgOffset += LocSize - ValSize;
7270 // Potential tail calls could cause overwriting of argument stack slots.
7271 const bool IsImmutable =
7272 !(getTargetMachine().Options.GuaranteedTailCallOpt &&
7273 (CallConv == CallingConv::Fast));
7274 int FI = MFI.CreateFixedObject(Size: ValSize, SPOffset: CurArgOffset, IsImmutable);
7275 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7276 SDValue ArgValue =
7277 DAG.getLoad(VT: ValVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
7278
7279 // While the ABI specifies the argument type is (sign or zero) extended
7280 // out to register width, not all code is compliant. We truncate and
7281 // re-extend to be more forgiving of these callers when the argument type
7282 // is smaller than register width.
7283 if (!ArgVT.isVector() && !ValVT.isVector() && ArgVT.isInteger() &&
7284 ValVT.isInteger() &&
7285 ArgVT.getScalarSizeInBits() < ValVT.getScalarSizeInBits()) {
7286 // It is possible to have either real integer values
7287 // or integers that were not originally integers.
7288 // In the latter case, these could have came from structs,
7289 // and these integers would not have an extend on the parameter.
7290 // Since these types of integers do not have an extend specified
7291 // in the first place, the type of extend that we do should not matter.
7292 EVT TruncatedArgVT = ArgVT.isSimple() && ArgVT.getSimpleVT() == MVT::i1
7293 ? MVT::i8
7294 : ArgVT;
7295 SDValue ArgValueTrunc =
7296 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncatedArgVT, Operand: ArgValue);
7297 SDValue ArgValueExt =
7298 ArgSignExt ? DAG.getSExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT)
7299 : DAG.getZExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT);
7300 InVals.push_back(Elt: ArgValueExt);
7301 } else {
7302 InVals.push_back(Elt: ArgValue);
7303 }
7304 };
7305
7306 // Vector arguments to VaArg functions are passed both on the stack, and
7307 // in any available GPRs. Load the value from the stack and add the GPRs
7308 // as live ins.
7309 if (VA.isMemLoc() && VA.needsCustom()) {
7310 assert(ValVT.isVector() && "Unexpected Custom MemLoc type.");
7311 assert(isVarArg && "Only use custom memloc for vararg.");
7312 // ValNo of the custom MemLoc, so we can compare it to the ValNo of the
7313 // matching custom RegLocs.
7314 const unsigned OriginalValNo = VA.getValNo();
7315 (void)OriginalValNo;
7316
7317 auto HandleCustomVecRegLoc = [&]() {
7318 assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7319 "Missing custom RegLoc.");
7320 VA = ArgLocs[I++];
7321 assert(VA.getValVT().isVector() &&
7322 "Unexpected Val type for custom RegLoc.");
7323 assert(VA.getValNo() == OriginalValNo &&
7324 "ValNo mismatch between custom MemLoc and RegLoc.");
7325 MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy;
7326 MF.addLiveIn(PReg: VA.getLocReg(),
7327 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7328 HasVSX: Subtarget.hasVSX()));
7329 };
7330
7331 HandleMemLoc();
7332 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7333 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7334 // R10.
7335 HandleCustomVecRegLoc();
7336 HandleCustomVecRegLoc();
7337
7338 // If we are targeting 32-bit, there might be 2 extra custom RegLocs if
7339 // we passed the vector in R5, R6, R7 and R8.
7340 if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) {
7341 assert(!IsPPC64 &&
7342 "Only 2 custom RegLocs expected for 64-bit codegen.");
7343 HandleCustomVecRegLoc();
7344 HandleCustomVecRegLoc();
7345 }
7346
7347 continue;
7348 }
7349
7350 if (VA.isRegLoc()) {
7351 if (VA.getValVT().isScalarInteger())
7352 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7353 else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) {
7354 switch (VA.getValVT().SimpleTy) {
7355 default:
7356 report_fatal_error(reason: "Unhandled value type for argument.");
7357 case MVT::f32:
7358 FuncInfo->appendParameterType(Type: PPCFunctionInfo::ShortFloatingPoint);
7359 break;
7360 case MVT::f64:
7361 FuncInfo->appendParameterType(Type: PPCFunctionInfo::LongFloatingPoint);
7362 break;
7363 }
7364 } else if (VA.getValVT().isVector()) {
7365 switch (VA.getValVT().SimpleTy) {
7366 default:
7367 report_fatal_error(reason: "Unhandled value type for argument.");
7368 case MVT::v16i8:
7369 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorChar);
7370 break;
7371 case MVT::v8i16:
7372 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorShort);
7373 break;
7374 case MVT::v4i32:
7375 case MVT::v2i64:
7376 case MVT::v1i128:
7377 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorInt);
7378 break;
7379 case MVT::v4f32:
7380 case MVT::v2f64:
7381 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorFloat);
7382 break;
7383 }
7384 }
7385 }
7386
7387 if (Flags.isByVal() && VA.isMemLoc()) {
7388 const unsigned Size =
7389 alignTo(Value: Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize,
7390 Align: PtrByteSize);
7391 const int FI = MF.getFrameInfo().CreateFixedObject(
7392 Size, SPOffset: VA.getLocMemOffset(), /* IsImmutable */ false,
7393 /* IsAliased */ isAliased: true);
7394 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7395 InVals.push_back(Elt: FIN);
7396
7397 continue;
7398 }
7399
7400 if (Flags.isByVal()) {
7401 assert(VA.isRegLoc() && "MemLocs should already be handled.");
7402
7403 const MCPhysReg ArgReg = VA.getLocReg();
7404 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
7405
7406 const unsigned StackSize = alignTo(Value: Flags.getByValSize(), Align: PtrByteSize);
7407 const int FI = MF.getFrameInfo().CreateFixedObject(
7408 Size: StackSize, SPOffset: mapArgRegToOffsetAIX(Reg: ArgReg, FL), /* IsImmutable */ false,
7409 /* IsAliased */ isAliased: true);
7410 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7411 InVals.push_back(Elt: FIN);
7412
7413 // Add live ins for all the RegLocs for the same ByVal.
7414 const TargetRegisterClass *RegClass =
7415 IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7416
7417 auto HandleRegLoc = [&, RegClass, LocVT](const MCPhysReg PhysReg,
7418 unsigned Offset) {
7419 const Register VReg = MF.addLiveIn(PReg: PhysReg, RC: RegClass);
7420 // Since the callers side has left justified the aggregate in the
7421 // register, we can simply store the entire register into the stack
7422 // slot.
7423 SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7424 // The store to the fixedstack object is needed becuase accessing a
7425 // field of the ByVal will use a gep and load. Ideally we will optimize
7426 // to extracting the value from the register directly, and elide the
7427 // stores when the arguments address is not taken, but that will need to
7428 // be future work.
7429 SDValue Store = DAG.getStore(
7430 Chain: CopyFrom.getValue(R: 1), dl, Val: CopyFrom,
7431 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: FIN, Offset: TypeSize::getFixed(ExactSize: Offset)),
7432 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset));
7433
7434 MemOps.push_back(Elt: Store);
7435 };
7436
7437 unsigned Offset = 0;
7438 HandleRegLoc(VA.getLocReg(), Offset);
7439 Offset += PtrByteSize;
7440 for (; Offset != StackSize && ArgLocs[I].isRegLoc();
7441 Offset += PtrByteSize) {
7442 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7443 "RegLocs should be for ByVal argument.");
7444
7445 const CCValAssign RL = ArgLocs[I++];
7446 HandleRegLoc(RL.getLocReg(), Offset);
7447 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7448 }
7449
7450 if (Offset != StackSize) {
7451 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7452 "Expected MemLoc for remaining bytes.");
7453 assert(ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes.");
7454 // Consume the MemLoc.The InVal has already been emitted, so nothing
7455 // more needs to be done.
7456 ++I;
7457 }
7458
7459 continue;
7460 }
7461
7462 if (VA.isRegLoc() && !VA.needsCustom()) {
7463 MVT::SimpleValueType SVT = ValVT.SimpleTy;
7464 Register VReg =
7465 MF.addLiveIn(PReg: VA.getLocReg(),
7466 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7467 HasVSX: Subtarget.hasVSX()));
7468 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7469 if (ValVT.isScalarInteger() &&
7470 (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) {
7471 ArgValue =
7472 truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl);
7473 }
7474 InVals.push_back(Elt: ArgValue);
7475 continue;
7476 }
7477 if (VA.isMemLoc()) {
7478 HandleMemLoc();
7479 continue;
7480 }
7481 }
7482
7483 // On AIX a minimum of 8 words is saved to the parameter save area.
7484 const unsigned MinParameterSaveArea = 8 * PtrByteSize;
7485 // Area that is at least reserved in the caller of this function.
7486 unsigned CallerReservedArea = std::max<unsigned>(
7487 a: CCInfo.getStackSize(), b: LinkageSize + MinParameterSaveArea);
7488
7489 // Set the size that is at least reserved in caller of this function. Tail
7490 // call optimized function's reserved stack space needs to be aligned so
7491 // that taking the difference between two stack areas will result in an
7492 // aligned stack.
7493 CallerReservedArea =
7494 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: CallerReservedArea);
7495 FuncInfo->setMinReservedArea(CallerReservedArea);
7496
7497 if (isVarArg) {
7498 int VAListIndex = 0;
7499 // If any of the optional arguments are passed in register then the fixed
7500 // stack object we spill into is not immutable. Create a fixed stack object
7501 // that overlaps the remainder of the parameter save area.
7502 if (CCInfo.getStackSize() < (LinkageSize + MinParameterSaveArea)) {
7503 unsigned FixedStackSize =
7504 LinkageSize + MinParameterSaveArea - CCInfo.getStackSize();
7505 VAListIndex =
7506 MFI.CreateFixedObject(Size: FixedStackSize, SPOffset: CCInfo.getStackSize(),
7507 /* IsImmutable */ false, /* IsAliased */ isAliased: true);
7508 } else {
7509 // All the arguments passed through ellipses are on the stack. Create a
7510 // dummy fixed stack object the same size as a pointer since we don't
7511 // know the actual size.
7512 VAListIndex =
7513 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: CCInfo.getStackSize(),
7514 /* IsImmutable */ true, /* IsAliased */ isAliased: true);
7515 }
7516
7517 FuncInfo->setVarArgsFrameIndex(VAListIndex);
7518 SDValue FIN = DAG.getFrameIndex(FI: VAListIndex, VT: PtrVT);
7519
7520 static const MCPhysReg GPR_32[] = {PPC::R3, PPC::R4, PPC::R5, PPC::R6,
7521 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
7522
7523 static const MCPhysReg GPR_64[] = {PPC::X3, PPC::X4, PPC::X5, PPC::X6,
7524 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
7525 const unsigned NumGPArgRegs = std::size(IsPPC64 ? GPR_64 : GPR_32);
7526
7527 // The fixed integer arguments of a variadic function are stored to the
7528 // VarArgsFrameIndex on the stack so that they may be loaded by
7529 // dereferencing the result of va_next.
7530 for (unsigned
7531 GPRIndex = (CCInfo.getStackSize() - LinkageSize) / PtrByteSize,
7532 Offset = 0;
7533 GPRIndex < NumGPArgRegs; ++GPRIndex, Offset += PtrByteSize) {
7534
7535 const Register VReg =
7536 IsPPC64 ? MF.addLiveIn(PReg: GPR_64[GPRIndex], RC: &PPC::G8RCRegClass)
7537 : MF.addLiveIn(PReg: GPR_32[GPRIndex], RC: &PPC::GPRCRegClass);
7538
7539 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
7540 MachinePointerInfo MPI =
7541 MachinePointerInfo::getFixedStack(MF, FI: VAListIndex, Offset);
7542 SDValue Store = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MPI);
7543 MemOps.push_back(Elt: Store);
7544 // Increment the address for the next argument to store.
7545 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
7546 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
7547 }
7548 }
7549
7550 if (!MemOps.empty())
7551 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
7552
7553 return Chain;
7554}
7555
7556SDValue PPCTargetLowering::LowerCall_AIX(
7557 SDValue Chain, SDValue Callee, CallFlags CFlags,
7558 const SmallVectorImpl<ISD::OutputArg> &Outs,
7559 const SmallVectorImpl<SDValue> &OutVals,
7560 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7561 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
7562 const CallBase *CB) const {
7563 // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the
7564 // AIX ABI stack frame layout.
7565
7566 assert((CFlags.CallConv == CallingConv::C ||
7567 CFlags.CallConv == CallingConv::Cold ||
7568 CFlags.CallConv == CallingConv::Fast) &&
7569 "Unexpected calling convention!");
7570
7571 if (CFlags.IsPatchPoint)
7572 report_fatal_error(reason: "This call type is unimplemented on AIX.");
7573
7574 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7575
7576 MachineFunction &MF = DAG.getMachineFunction();
7577 SmallVector<CCValAssign, 16> ArgLocs;
7578 CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
7579 *DAG.getContext());
7580
7581 // Reserve space for the linkage save area (LSA) on the stack.
7582 // In both PPC32 and PPC64 there are 6 reserved slots in the LSA:
7583 // [SP][CR][LR][2 x reserved][TOC].
7584 // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64.
7585 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7586 const bool IsPPC64 = Subtarget.isPPC64();
7587 const EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7588 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7589 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7590 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_AIX);
7591
7592 // The prolog code of the callee may store up to 8 GPR argument registers to
7593 // the stack, allowing va_start to index over them in memory if the callee
7594 // is variadic.
7595 // Because we cannot tell if this is needed on the caller side, we have to
7596 // conservatively assume that it is needed. As such, make sure we have at
7597 // least enough stack space for the caller to store the 8 GPRs.
7598 const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize;
7599 const unsigned NumBytes = std::max<unsigned>(
7600 a: LinkageSize + MinParameterSaveAreaSize, b: CCInfo.getStackSize());
7601
7602 // Adjust the stack pointer for the new arguments...
7603 // These operations are automatically eliminated by the prolog/epilog pass.
7604 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
7605 SDValue CallSeqStart = Chain;
7606
7607 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
7608 SmallVector<SDValue, 8> MemOpChains;
7609
7610 // Set up a copy of the stack pointer for loading and storing any
7611 // arguments that may not fit in the registers available for argument
7612 // passing.
7613 const SDValue StackPtr = IsPPC64 ? DAG.getRegister(Reg: PPC::X1, VT: MVT::i64)
7614 : DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
7615
7616 for (unsigned I = 0, E = ArgLocs.size(); I != E;) {
7617 const unsigned ValNo = ArgLocs[I].getValNo();
7618 SDValue Arg = OutVals[ValNo];
7619 ISD::ArgFlagsTy Flags = Outs[ValNo].Flags;
7620
7621 if (Flags.isByVal()) {
7622 const unsigned ByValSize = Flags.getByValSize();
7623
7624 // Nothing to do for zero-sized ByVals on the caller side.
7625 if (!ByValSize) {
7626 ++I;
7627 continue;
7628 }
7629
7630 auto GetLoad = [&](EVT VT, unsigned LoadOffset) {
7631 return DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: PtrVT, Chain,
7632 Ptr: (LoadOffset != 0)
7633 ? DAG.getObjectPtrOffset(
7634 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7635 : Arg,
7636 PtrInfo: MachinePointerInfo(), MemVT: VT);
7637 };
7638
7639 unsigned LoadOffset = 0;
7640
7641 // Initialize registers, which are fully occupied by the by-val argument.
7642 while (LoadOffset + PtrByteSize <= ByValSize && ArgLocs[I].isRegLoc()) {
7643 SDValue Load = GetLoad(PtrVT, LoadOffset);
7644 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7645 LoadOffset += PtrByteSize;
7646 const CCValAssign &ByValVA = ArgLocs[I++];
7647 assert(ByValVA.getValNo() == ValNo &&
7648 "Unexpected location for pass-by-value argument.");
7649 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: Load));
7650 }
7651
7652 if (LoadOffset == ByValSize)
7653 continue;
7654
7655 // There must be one more loc to handle the remainder.
7656 assert(ArgLocs[I].getValNo() == ValNo &&
7657 "Expected additional location for by-value argument.");
7658
7659 if (ArgLocs[I].isMemLoc()) {
7660 assert(LoadOffset < ByValSize && "Unexpected memloc for by-val arg.");
7661 const CCValAssign &ByValVA = ArgLocs[I++];
7662 ISD::ArgFlagsTy MemcpyFlags = Flags;
7663 // Only memcpy the bytes that don't pass in register.
7664 MemcpyFlags.setByValSize(ByValSize - LoadOffset);
7665 Chain = CallSeqStart = createMemcpyOutsideCallSeq(
7666 Arg: (LoadOffset != 0) ? DAG.getObjectPtrOffset(
7667 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7668 : Arg,
7669 PtrOff: DAG.getObjectPtrOffset(
7670 SL: dl, Ptr: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByValVA.getLocMemOffset())),
7671 CallSeqStart, Flags: MemcpyFlags, DAG, dl);
7672 continue;
7673 }
7674
7675 // Initialize the final register residue.
7676 // Any residue that occupies the final by-val arg register must be
7677 // left-justified on AIX. Loads must be a power-of-2 size and cannot be
7678 // larger than the ByValSize. For example: a 7 byte by-val arg requires 4,
7679 // 2 and 1 byte loads.
7680 const unsigned ResidueBytes = ByValSize % PtrByteSize;
7681 assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize &&
7682 "Unexpected register residue for by-value argument.");
7683 SDValue ResidueVal;
7684 for (unsigned Bytes = 0; Bytes != ResidueBytes;) {
7685 const unsigned N = llvm::bit_floor(Value: ResidueBytes - Bytes);
7686 const MVT VT =
7687 N == 1 ? MVT::i8
7688 : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64));
7689 SDValue Load = GetLoad(VT, LoadOffset);
7690 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7691 LoadOffset += N;
7692 Bytes += N;
7693
7694 // By-val arguments are passed left-justfied in register.
7695 // Every load here needs to be shifted, otherwise a full register load
7696 // should have been used.
7697 assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) &&
7698 "Unexpected load emitted during handling of pass-by-value "
7699 "argument.");
7700 unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8);
7701 EVT ShiftAmountTy =
7702 getShiftAmountTy(LHSTy: Load->getValueType(ResNo: 0), DL: DAG.getDataLayout());
7703 SDValue SHLAmt = DAG.getConstant(Val: NumSHLBits, DL: dl, VT: ShiftAmountTy);
7704 SDValue ShiftedLoad =
7705 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Load.getValueType(), N1: Load, N2: SHLAmt);
7706 ResidueVal = ResidueVal ? DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PtrVT, N1: ResidueVal,
7707 N2: ShiftedLoad)
7708 : ShiftedLoad;
7709 }
7710
7711 const CCValAssign &ByValVA = ArgLocs[I++];
7712 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: ResidueVal));
7713 continue;
7714 }
7715
7716 CCValAssign &VA = ArgLocs[I++];
7717 const MVT LocVT = VA.getLocVT();
7718 const MVT ValVT = VA.getValVT();
7719
7720 switch (VA.getLocInfo()) {
7721 default:
7722 report_fatal_error(reason: "Unexpected argument extension type.");
7723 case CCValAssign::Full:
7724 break;
7725 case CCValAssign::ZExt:
7726 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7727 break;
7728 case CCValAssign::SExt:
7729 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7730 break;
7731 }
7732
7733 if (VA.isRegLoc() && !VA.needsCustom()) {
7734 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
7735 continue;
7736 }
7737
7738 // Vector arguments passed to VarArg functions need custom handling when
7739 // they are passed (at least partially) in GPRs.
7740 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) {
7741 assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args.");
7742 // Store value to its stack slot.
7743 SDValue PtrOff =
7744 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7745 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7746 SDValue Store =
7747 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
7748 MemOpChains.push_back(Elt: Store);
7749 const unsigned OriginalValNo = VA.getValNo();
7750 // Then load the GPRs from the stack
7751 unsigned LoadOffset = 0;
7752 auto HandleCustomVecRegLoc = [&]() {
7753 assert(I != E && "Unexpected end of CCvalAssigns.");
7754 assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7755 "Expected custom RegLoc.");
7756 CCValAssign RegVA = ArgLocs[I++];
7757 assert(RegVA.getValNo() == OriginalValNo &&
7758 "Custom MemLoc ValNo and custom RegLoc ValNo must match.");
7759 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
7760 N2: DAG.getConstant(Val: LoadOffset, DL: dl, VT: PtrVT));
7761 SDValue Load = DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Add, PtrInfo: MachinePointerInfo());
7762 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7763 RegsToPass.push_back(Elt: std::make_pair(x: RegVA.getLocReg(), y&: Load));
7764 LoadOffset += PtrByteSize;
7765 };
7766
7767 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7768 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7769 // R10.
7770 HandleCustomVecRegLoc();
7771 HandleCustomVecRegLoc();
7772
7773 if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7774 ArgLocs[I].getValNo() == OriginalValNo) {
7775 assert(!IsPPC64 &&
7776 "Only 2 custom RegLocs expected for 64-bit codegen.");
7777 HandleCustomVecRegLoc();
7778 HandleCustomVecRegLoc();
7779 }
7780
7781 continue;
7782 }
7783
7784 if (VA.isMemLoc()) {
7785 SDValue PtrOff =
7786 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7787 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7788 MemOpChains.push_back(
7789 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff,
7790 PtrInfo: MachinePointerInfo::getStack(MF, Offset: VA.getLocMemOffset()),
7791 Alignment: Subtarget.getFrameLowering()->getStackAlign()));
7792
7793 continue;
7794 }
7795
7796 if (!ValVT.isFloatingPoint())
7797 report_fatal_error(
7798 reason: "Unexpected register handling for calling convention.");
7799
7800 // Custom handling is used for GPR initializations for vararg float
7801 // arguments.
7802 assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg &&
7803 LocVT.isInteger() &&
7804 "Custom register handling only expected for VarArg.");
7805
7806 SDValue ArgAsInt =
7807 DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), V: Arg);
7808
7809 if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize())
7810 // f32 in 32-bit GPR
7811 // f64 in 64-bit GPR
7812 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgAsInt));
7813 else if (Arg.getValueType().getFixedSizeInBits() <
7814 LocVT.getFixedSizeInBits())
7815 // f32 in 64-bit GPR.
7816 RegsToPass.push_back(Elt: std::make_pair(
7817 x: VA.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: LocVT)));
7818 else {
7819 // f64 in two 32-bit GPRs
7820 // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs.
7821 assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 &&
7822 "Unexpected custom register for argument!");
7823 CCValAssign &GPR1 = VA;
7824 SDValue MSWAsI64 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgAsInt,
7825 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i8));
7826 RegsToPass.push_back(Elt: std::make_pair(
7827 x: GPR1.getLocReg(), y: DAG.getZExtOrTrunc(Op: MSWAsI64, DL: dl, VT: MVT::i32)));
7828
7829 if (I != E) {
7830 // If only 1 GPR was available, there will only be one custom GPR and
7831 // the argument will also pass in memory.
7832 CCValAssign &PeekArg = ArgLocs[I];
7833 if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) {
7834 assert(PeekArg.needsCustom() && "A second custom GPR is expected.");
7835 CCValAssign &GPR2 = ArgLocs[I++];
7836 RegsToPass.push_back(Elt: std::make_pair(
7837 x: GPR2.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: MVT::i32)));
7838 }
7839 }
7840 }
7841 }
7842
7843 if (!MemOpChains.empty())
7844 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
7845
7846 // For indirect calls, we need to save the TOC base to the stack for
7847 // restoration after the call.
7848 if (CFlags.IsIndirect && !Subtarget.usePointerGlueHelper()) {
7849 assert(!CFlags.IsTailCall && "Indirect tail-calls not supported.");
7850 const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister();
7851 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
7852 const MVT PtrVT = Subtarget.getScalarIntVT();
7853 const unsigned TOCSaveOffset =
7854 Subtarget.getFrameLowering()->getTOCSaveOffset();
7855
7856 setUsesTOCBasePtr(DAG);
7857 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: TOCBaseReg, VT: PtrVT);
7858 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
7859 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: PtrVT);
7860 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7861 Chain = DAG.getStore(
7862 Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
7863 PtrInfo: MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
7864 }
7865
7866 // Build a sequence of copy-to-reg nodes chained together with token chain
7867 // and flag operands which copy the outgoing args into the appropriate regs.
7868 SDValue InGlue;
7869 for (auto Reg : RegsToPass) {
7870 Chain = DAG.getCopyToReg(Chain, dl, Reg: Reg.first, N: Reg.second, Glue: InGlue);
7871 InGlue = Chain.getValue(R: 1);
7872 }
7873
7874 const int SPDiff = 0;
7875 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
7876 Callee, SPDiff, NumBytes, Ins, InVals, CB);
7877}
7878
7879bool
7880PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
7881 MachineFunction &MF, bool isVarArg,
7882 const SmallVectorImpl<ISD::OutputArg> &Outs,
7883 LLVMContext &Context,
7884 const Type *RetTy) const {
7885 SmallVector<CCValAssign, 16> RVLocs;
7886 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
7887 return CCInfo.CheckReturn(
7888 Outs, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7889 ? RetCC_PPC_Cold
7890 : RetCC_PPC);
7891}
7892
7893SDValue
7894PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
7895 bool isVarArg,
7896 const SmallVectorImpl<ISD::OutputArg> &Outs,
7897 const SmallVectorImpl<SDValue> &OutVals,
7898 const SDLoc &dl, SelectionDAG &DAG) const {
7899 SmallVector<CCValAssign, 16> RVLocs;
7900 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
7901 *DAG.getContext());
7902 CCInfo.AnalyzeReturn(Outs,
7903 Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7904 ? RetCC_PPC_Cold
7905 : RetCC_PPC);
7906
7907 SDValue Glue;
7908 SmallVector<SDValue, 4> RetOps(1, Chain);
7909
7910 // Copy the result values into the output registers.
7911 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) {
7912 CCValAssign &VA = RVLocs[i];
7913 assert(VA.isRegLoc() && "Can only return in registers!");
7914
7915 SDValue Arg = OutVals[RealResIdx];
7916
7917 switch (VA.getLocInfo()) {
7918 default: llvm_unreachable("Unknown loc info!");
7919 case CCValAssign::Full: break;
7920 case CCValAssign::AExt:
7921 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7922 break;
7923 case CCValAssign::ZExt:
7924 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7925 break;
7926 case CCValAssign::SExt:
7927 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7928 break;
7929 }
7930 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
7931 bool isLittleEndian = Subtarget.isLittleEndian();
7932 // Legalize ret f64 -> ret 2 x i32.
7933 SDValue SVal =
7934 DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7935 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 0 : 1, DL: dl));
7936 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7937 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7938 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7939 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 1 : 0, DL: dl));
7940 Glue = Chain.getValue(R: 1);
7941 VA = RVLocs[++i]; // skip ahead to next loc
7942 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7943 } else
7944 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Arg, Glue);
7945 Glue = Chain.getValue(R: 1);
7946 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7947 }
7948
7949 RetOps[0] = Chain; // Update chain.
7950
7951 // Add the glue if we have it.
7952 if (Glue.getNode())
7953 RetOps.push_back(Elt: Glue);
7954
7955 return DAG.getNode(Opcode: PPCISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
7956}
7957
7958SDValue
7959PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,
7960 SelectionDAG &DAG) const {
7961 SDLoc dl(Op);
7962
7963 // Get the correct type for integers.
7964 EVT IntVT = Op.getValueType();
7965
7966 // Get the inputs.
7967 SDValue Chain = Op.getOperand(i: 0);
7968 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
7969 // Build a DYNAREAOFFSET node.
7970 SDValue Ops[2] = {Chain, FPSIdx};
7971 SDVTList VTs = DAG.getVTList(VT: IntVT);
7972 return DAG.getNode(Opcode: PPCISD::DYNAREAOFFSET, DL: dl, VTList: VTs, Ops);
7973}
7974
7975SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
7976 SelectionDAG &DAG) const {
7977 // When we pop the dynamic allocation we need to restore the SP link.
7978 SDLoc dl(Op);
7979
7980 // Get the correct type for pointers.
7981 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7982
7983 // Construct the stack pointer operand.
7984 bool isPPC64 = Subtarget.isPPC64();
7985 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1;
7986 SDValue StackPtr = DAG.getRegister(Reg: SP, VT: PtrVT);
7987
7988 // Get the operands for the STACKRESTORE.
7989 SDValue Chain = Op.getOperand(i: 0);
7990 SDValue SaveSP = Op.getOperand(i: 1);
7991
7992 // Load the old link SP.
7993 SDValue LoadLinkSP =
7994 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
7995
7996 // Restore the stack pointer.
7997 Chain = DAG.getCopyToReg(Chain: LoadLinkSP.getValue(R: 1), dl, Reg: SP, N: SaveSP);
7998
7999 // Store the old link SP.
8000 return DAG.getStore(Chain, dl, Val: LoadLinkSP, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
8001}
8002
8003SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const {
8004 MachineFunction &MF = DAG.getMachineFunction();
8005 bool isPPC64 = Subtarget.isPPC64();
8006 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8007
8008 // Get current frame pointer save index. The users of this index will be
8009 // primarily DYNALLOC instructions.
8010 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8011 int RASI = FI->getReturnAddrSaveIndex();
8012
8013 // If the frame pointer save index hasn't been defined yet.
8014 if (!RASI) {
8015 // Find out what the fix offset of the frame pointer save area.
8016 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset();
8017 // Allocate the frame index for frame pointer save area.
8018 RASI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: LROffset, IsImmutable: false);
8019 // Save the result.
8020 FI->setReturnAddrSaveIndex(RASI);
8021 }
8022 return DAG.getFrameIndex(FI: RASI, VT: PtrVT);
8023}
8024
8025SDValue
8026PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
8027 MachineFunction &MF = DAG.getMachineFunction();
8028 bool isPPC64 = Subtarget.isPPC64();
8029 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8030
8031 // Get current frame pointer save index. The users of this index will be
8032 // primarily DYNALLOC instructions.
8033 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8034 int FPSI = FI->getFramePointerSaveIndex();
8035
8036 // If the frame pointer save index hasn't been defined yet.
8037 if (!FPSI) {
8038 // Find out what the fix offset of the frame pointer save area.
8039 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset();
8040 // Allocate the frame index for frame pointer save area.
8041 FPSI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: FPOffset, IsImmutable: true);
8042 // Save the result.
8043 FI->setFramePointerSaveIndex(FPSI);
8044 }
8045 return DAG.getFrameIndex(FI: FPSI, VT: PtrVT);
8046}
8047
8048SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
8049 SelectionDAG &DAG) const {
8050 MachineFunction &MF = DAG.getMachineFunction();
8051 // Get the inputs.
8052 SDValue Chain = Op.getOperand(i: 0);
8053 SDValue Size = Op.getOperand(i: 1);
8054 SDLoc dl(Op);
8055
8056 // Get the correct type for pointers.
8057 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8058 // Negate the size.
8059 SDValue NegSize = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: PtrVT,
8060 N1: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), N2: Size);
8061 // Construct a node for the frame pointer save index.
8062 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
8063 SDValue Ops[3] = { Chain, NegSize, FPSIdx };
8064 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::Other);
8065 if (hasInlineStackProbe(MF))
8066 return DAG.getNode(Opcode: PPCISD::PROBED_ALLOCA, DL: dl, VTList: VTs, Ops);
8067 return DAG.getNode(Opcode: PPCISD::DYNALLOC, DL: dl, VTList: VTs, Ops);
8068}
8069
8070SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
8071 SelectionDAG &DAG) const {
8072 MachineFunction &MF = DAG.getMachineFunction();
8073
8074 bool isPPC64 = Subtarget.isPPC64();
8075 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8076
8077 int FI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64 ? 8 : 4, SPOffset: 0, IsImmutable: false);
8078 return DAG.getFrameIndex(FI, VT: PtrVT);
8079}
8080
8081SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
8082 SelectionDAG &DAG) const {
8083 SDLoc DL(Op);
8084 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_SETJMP, DL,
8085 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8086 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8087}
8088
8089SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
8090 SelectionDAG &DAG) const {
8091 SDLoc DL(Op);
8092 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_LONGJMP, DL, VT: MVT::Other,
8093 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8094}
8095
8096SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
8097 if (Op.getValueType().isVector())
8098 return LowerVectorLoad(Op, DAG);
8099
8100 assert(Op.getValueType() == MVT::i1 &&
8101 "Custom lowering only for i1 loads");
8102
8103 // First, load 8 bits into 32 bits, then truncate to 1 bit.
8104
8105 SDLoc dl(Op);
8106 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
8107
8108 SDValue Chain = LD->getChain();
8109 SDValue BasePtr = LD->getBasePtr();
8110 MachineMemOperand *MMO = LD->getMemOperand();
8111
8112 SDValue NewLD =
8113 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: getPointerTy(DL: DAG.getDataLayout()), Chain,
8114 Ptr: BasePtr, MemVT: MVT::i8, MMO);
8115 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewLD);
8116
8117 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
8118 return DAG.getMergeValues(Ops, dl);
8119}
8120
8121SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
8122 if (Op.getOperand(i: 1).getValueType().isVector())
8123 return LowerVectorStore(Op, DAG);
8124
8125 assert(Op.getOperand(1).getValueType() == MVT::i1 &&
8126 "Custom lowering only for i1 stores");
8127
8128 // First, zero extend to 32 bits, then use a truncating store to 8 bits.
8129
8130 SDLoc dl(Op);
8131 StoreSDNode *ST = cast<StoreSDNode>(Val&: Op);
8132
8133 SDValue Chain = ST->getChain();
8134 SDValue BasePtr = ST->getBasePtr();
8135 SDValue Value = ST->getValue();
8136 MachineMemOperand *MMO = ST->getMemOperand();
8137
8138 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
8139 Operand: Value);
8140 return DAG.getTruncStore(Chain, dl, Val: Value, Ptr: BasePtr, SVT: MVT::i8, MMO);
8141}
8142
8143// FIXME: Remove this once the ANDI glue bug is fixed:
8144SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
8145 assert(Op.getValueType() == MVT::i1 &&
8146 "Custom lowering only for i1 results");
8147
8148 SDLoc DL(Op);
8149 return DAG.getNode(Opcode: PPCISD::ANDI_rec_1_GT_BIT, DL, VT: MVT::i1, Operand: Op.getOperand(i: 0));
8150}
8151
8152SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
8153 SelectionDAG &DAG) const {
8154
8155 // Implements a vector truncate that fits in a vector register as a shuffle.
8156 // We want to legalize vector truncates down to where the source fits in
8157 // a vector register (and target is therefore smaller than vector register
8158 // size). At that point legalization will try to custom lower the sub-legal
8159 // result and get here - where we can contain the truncate as a single target
8160 // operation.
8161
8162 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows:
8163 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2>
8164 //
8165 // We will implement it for big-endian ordering as this (where x denotes
8166 // undefined):
8167 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to
8168 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u>
8169 //
8170 // The same operation in little-endian ordering will be:
8171 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to
8172 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1>
8173
8174 EVT TrgVT = Op.getValueType();
8175 assert(TrgVT.isVector() && "Vector type expected.");
8176 unsigned TrgNumElts = TrgVT.getVectorNumElements();
8177 EVT EltVT = TrgVT.getVectorElementType();
8178 if (!isOperationCustom(Op: Op.getOpcode(), VT: TrgVT) ||
8179 TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(Value: TrgNumElts) ||
8180 !llvm::has_single_bit<uint32_t>(Value: EltVT.getSizeInBits()))
8181 return SDValue();
8182
8183 SDValue N1 = Op.getOperand(i: 0);
8184 EVT SrcVT = N1.getValueType();
8185 unsigned SrcSize = SrcVT.getSizeInBits();
8186 if (SrcSize > 256 || !isPowerOf2_32(Value: SrcVT.getVectorNumElements()) ||
8187 !llvm::has_single_bit<uint32_t>(
8188 Value: SrcVT.getVectorElementType().getSizeInBits()))
8189 return SDValue();
8190 if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2)
8191 return SDValue();
8192
8193 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8194 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8195
8196 SDLoc DL(Op);
8197 SDValue Op1, Op2;
8198 if (SrcSize == 256) {
8199 EVT VecIdxTy = getVectorIdxTy(DL: DAG.getDataLayout());
8200 EVT SplitVT =
8201 N1.getValueType().getHalfNumVectorElementsVT(Context&: *DAG.getContext());
8202 unsigned SplitNumElts = SplitVT.getVectorNumElements();
8203 Op1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8204 N2: DAG.getConstant(Val: 0, DL, VT: VecIdxTy));
8205 Op2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8206 N2: DAG.getConstant(Val: SplitNumElts, DL, VT: VecIdxTy));
8207 }
8208 else {
8209 Op1 = SrcSize == 128 ? N1 : widenVec(DAG, Vec: N1, dl: DL);
8210 Op2 = DAG.getUNDEF(VT: WideVT);
8211 }
8212
8213 // First list the elements we want to keep.
8214 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits();
8215 SmallVector<int, 16> ShuffV;
8216 if (Subtarget.isLittleEndian())
8217 for (unsigned i = 0; i < TrgNumElts; ++i)
8218 ShuffV.push_back(Elt: i * SizeMult);
8219 else
8220 for (unsigned i = 1; i <= TrgNumElts; ++i)
8221 ShuffV.push_back(Elt: i * SizeMult - 1);
8222
8223 // Populate the remaining elements with undefs.
8224 for (unsigned i = TrgNumElts; i < WideNumElts; ++i)
8225 // ShuffV.push_back(i + WideNumElts);
8226 ShuffV.push_back(Elt: WideNumElts + 1);
8227
8228 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op1);
8229 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op2);
8230 return DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: Op1, N2: Op2, Mask: ShuffV);
8231}
8232
8233/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
8234/// possible.
8235SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
8236 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Op.getOperand(i: 4))->get();
8237 EVT ResVT = Op.getValueType();
8238 EVT CmpVT = Op.getOperand(i: 0).getValueType();
8239 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
8240 SDValue TV = Op.getOperand(i: 2), FV = Op.getOperand(i: 3);
8241 SDLoc dl(Op);
8242
8243 // Without power9-vector, we don't have native instruction for f128 comparison.
8244 // Following transformation to libcall is needed for setcc:
8245 // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE
8246 if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) {
8247 SDValue Z = DAG.getSetCC(
8248 DL: dl, VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: CmpVT),
8249 LHS, RHS, Cond: CC);
8250 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Z.getValueType());
8251 return DAG.getSelectCC(DL: dl, LHS: Z, RHS: Zero, True: TV, False: FV, Cond: ISD::SETNE);
8252 }
8253
8254 // Not FP, or using SPE? Not a fsel.
8255 if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() ||
8256 Subtarget.hasSPE())
8257 return Op;
8258
8259 SDNodeFlags Flags = Op.getNode()->getFlags();
8260
8261 // We have xsmaxc[dq]p/xsminc[dq]p which are OK to emit even in the
8262 // presence of infinities.
8263 if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) {
8264 switch (CC) {
8265 default:
8266 break;
8267 case ISD::SETOGT:
8268 case ISD::SETGT:
8269 return DAG.getNode(Opcode: PPCISD::XSMAXC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8270 case ISD::SETOLT:
8271 case ISD::SETLT:
8272 return DAG.getNode(Opcode: PPCISD::XSMINC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8273 }
8274 }
8275
8276 // We might be able to do better than this under some circumstances, but in
8277 // general, fsel-based lowering of select is a finite-math-only optimization.
8278 // For more information, see section F.3 of the 2.06 ISA specification.
8279 // With ISA 3.0
8280 if (!Flags.hasNoInfs() || !Flags.hasNoNaNs() || ResVT == MVT::f128)
8281 return Op;
8282
8283 // If the RHS of the comparison is a 0.0, we don't need to do the
8284 // subtraction at all.
8285 SDValue Sel1;
8286 if (isFloatingPointZero(Op: RHS))
8287 switch (CC) {
8288 default: break; // SETUO etc aren't handled by fsel.
8289 case ISD::SETNE:
8290 std::swap(a&: TV, b&: FV);
8291 [[fallthrough]];
8292 case ISD::SETEQ:
8293 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8294 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8295 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8296 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
8297 Sel1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Sel1);
8298 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8299 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: Sel1, N3: FV);
8300 case ISD::SETULT:
8301 case ISD::SETLT:
8302 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8303 [[fallthrough]];
8304 case ISD::SETOGE:
8305 case ISD::SETGE:
8306 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8307 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8308 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8309 case ISD::SETUGT:
8310 case ISD::SETGT:
8311 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8312 [[fallthrough]];
8313 case ISD::SETOLE:
8314 case ISD::SETLE:
8315 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8316 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8317 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8318 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: TV, N3: FV);
8319 }
8320
8321 SDValue Cmp;
8322 switch (CC) {
8323 default: break; // SETUO etc aren't handled by fsel.
8324 case ISD::SETNE:
8325 std::swap(a&: TV, b&: FV);
8326 [[fallthrough]];
8327 case ISD::SETEQ:
8328 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8329 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8330 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8331 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8332 if (Sel1.getValueType() == MVT::f32) // Comparison is always 64-bits
8333 Sel1 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Sel1);
8334 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8335 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: Cmp), N2: Sel1, N3: FV);
8336 case ISD::SETULT:
8337 case ISD::SETLT:
8338 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8339 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8340 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8341 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8342 case ISD::SETOGE:
8343 case ISD::SETGE:
8344 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8345 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8346 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8347 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8348 case ISD::SETUGT:
8349 case ISD::SETGT:
8350 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8351 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8352 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8353 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8354 case ISD::SETOLE:
8355 case ISD::SETLE:
8356 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8357 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8358 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8359 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8360 }
8361 return Op;
8362}
8363
8364static unsigned getPPCStrictOpcode(unsigned Opc) {
8365 switch (Opc) {
8366 default:
8367 llvm_unreachable("No strict version of this opcode!");
8368 case PPCISD::FCTIDZ:
8369 return PPCISD::STRICT_FCTIDZ;
8370 case PPCISD::FCTIWZ:
8371 return PPCISD::STRICT_FCTIWZ;
8372 case PPCISD::FCTIDUZ:
8373 return PPCISD::STRICT_FCTIDUZ;
8374 case PPCISD::FCTIWUZ:
8375 return PPCISD::STRICT_FCTIWUZ;
8376 case PPCISD::FCFID:
8377 return PPCISD::STRICT_FCFID;
8378 case PPCISD::FCFIDU:
8379 return PPCISD::STRICT_FCFIDU;
8380 case PPCISD::FCFIDS:
8381 return PPCISD::STRICT_FCFIDS;
8382 case PPCISD::FCFIDUS:
8383 return PPCISD::STRICT_FCFIDUS;
8384 }
8385}
8386
8387static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG,
8388 const PPCSubtarget &Subtarget) {
8389 SDLoc dl(Op);
8390 bool IsStrict = Op->isStrictFPOpcode();
8391 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8392 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8393
8394 // TODO: Any other flags to propagate?
8395 SDNodeFlags Flags;
8396 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8397
8398 // For strict nodes, source is the second operand.
8399 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8400 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8401 MVT DestTy = Op.getSimpleValueType();
8402 assert(Src.getValueType().isFloatingPoint() &&
8403 (DestTy == MVT::i8 || DestTy == MVT::i16 || DestTy == MVT::i32 ||
8404 DestTy == MVT::i64) &&
8405 "Invalid FP_TO_INT types");
8406 if (Src.getValueType() == MVT::f32) {
8407 if (IsStrict) {
8408 Src =
8409 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
8410 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8411 Chain = Src.getValue(R: 1);
8412 } else
8413 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
8414 }
8415 if ((DestTy == MVT::i8 || DestTy == MVT::i16) && Subtarget.hasP9Vector())
8416 DestTy = Subtarget.getScalarIntVT();
8417 unsigned Opc = ISD::DELETED_NODE;
8418 switch (DestTy.SimpleTy) {
8419 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
8420 case MVT::i32:
8421 Opc = IsSigned ? PPCISD::FCTIWZ
8422 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ);
8423 break;
8424 case MVT::i64:
8425 assert((IsSigned || Subtarget.hasFPCVT()) &&
8426 "i64 FP_TO_UINT is supported only with FPCVT");
8427 Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ;
8428 }
8429 EVT ConvTy = Src.getValueType() == MVT::f128 ? MVT::f128 : MVT::f64;
8430 SDValue Conv;
8431 if (IsStrict) {
8432 Opc = getPPCStrictOpcode(Opc);
8433 Conv = DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src},
8434 Flags);
8435 } else {
8436 Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: ConvTy, Operand: Src);
8437 }
8438 return Conv;
8439}
8440
8441void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
8442 SelectionDAG &DAG,
8443 const SDLoc &dl) const {
8444 SDValue Tmp = convertFPToInt(Op, DAG, Subtarget);
8445 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8446 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8447 bool IsStrict = Op->isStrictFPOpcode();
8448
8449 // Convert the FP value to an int value through memory.
8450 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
8451 (IsSigned || Subtarget.hasFPCVT());
8452 SDValue FIPtr = DAG.CreateStackTemporary(VT: i32Stack ? MVT::i32 : MVT::f64);
8453 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
8454 MachinePointerInfo MPI =
8455 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
8456
8457 // Emit a store to the stack slot.
8458 SDValue Chain = IsStrict ? Tmp.getValue(R: 1) : DAG.getEntryNode();
8459 Align Alignment(DAG.getEVTAlign(MemoryVT: Tmp.getValueType()));
8460 if (i32Stack) {
8461 MachineFunction &MF = DAG.getMachineFunction();
8462 Alignment = Align(4);
8463 MachineMemOperand *MMO =
8464 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Alignment);
8465 SDValue Ops[] = { Chain, Tmp, FIPtr };
8466 Chain = DAG.getMemIntrinsicNode(Opcode: PPCISD::STFIWX, dl,
8467 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT: MVT::i32, MMO);
8468 } else
8469 Chain = DAG.getStore(Chain, dl, Val: Tmp, Ptr: FIPtr, PtrInfo: MPI, Alignment);
8470
8471 // Result is a load from the stack slot. If loading 4 bytes, make sure to
8472 // add in a bias on big endian.
8473 if (Op.getValueType() == MVT::i32 && !i32Stack &&
8474 !Subtarget.isLittleEndian()) {
8475 FIPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FIPtr.getValueType(), N1: FIPtr,
8476 N2: DAG.getConstant(Val: 4, DL: dl, VT: FIPtr.getValueType()));
8477 MPI = MPI.getWithOffset(O: 4);
8478 }
8479
8480 RLI.Chain = Chain;
8481 RLI.Ptr = FIPtr;
8482 RLI.MPI = MPI;
8483 RLI.Alignment = Alignment;
8484}
8485
8486/// Custom lowers floating point to integer conversions to use
8487/// the direct move instructions available in ISA 2.07 to avoid the
8488/// need for load/store combinations.
8489SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
8490 SelectionDAG &DAG,
8491 const SDLoc &dl) const {
8492 SDValue Conv = convertFPToInt(Op, DAG, Subtarget);
8493 SDValue Mov = DAG.getNode(Opcode: PPCISD::MFVSR, DL: dl, VT: Op.getValueType(), Operand: Conv);
8494 if (Op->isStrictFPOpcode())
8495 return DAG.getMergeValues(Ops: {Mov, Conv.getValue(R: 1)}, dl);
8496 else
8497 return Mov;
8498}
8499
8500SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
8501 const SDLoc &dl) const {
8502 bool IsStrict = Op->isStrictFPOpcode();
8503 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8504 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8505 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8506 EVT SrcVT = Src.getValueType();
8507 EVT DstVT = Op.getValueType();
8508
8509 // FP to INT conversions are legal for f128.
8510 if (SrcVT == MVT::f128)
8511 return Subtarget.hasP9Vector() ? Op : SDValue();
8512
8513 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
8514 // PPC (the libcall is not available).
8515 if (SrcVT == MVT::ppcf128) {
8516 if (DstVT == MVT::i32) {
8517 // TODO: Conservatively pass only nofpexcept flag here. Need to check and
8518 // set other fast-math flags to FP operations in both strict and
8519 // non-strict cases. (FP_TO_SINT, FSUB)
8520 SDNodeFlags Flags;
8521 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8522
8523 if (IsSigned) {
8524 SDValue Lo, Hi;
8525 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: dl, LoVT: MVT::f64, HiVT: MVT::f64);
8526
8527 // Add the two halves of the long double in round-to-zero mode, and use
8528 // a smaller FP_TO_SINT.
8529 if (IsStrict) {
8530 SDValue Res = DAG.getNode(Opcode: PPCISD::STRICT_FADDRTZ, DL: dl,
8531 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8532 Ops: {Op.getOperand(i: 0), Lo, Hi}, Flags);
8533 return DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8534 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8535 Ops: {Res.getValue(R: 1), Res}, Flags);
8536 } else {
8537 SDValue Res = DAG.getNode(Opcode: PPCISD::FADDRTZ, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
8538 return DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Res);
8539 }
8540 } else {
8541 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
8542 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31));
8543 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
8544 SDValue SignMask = DAG.getConstant(Val: 0x80000000, DL: dl, VT: DstVT);
8545 if (IsStrict) {
8546 // Sel = Src < 0x80000000
8547 // FltOfs = select Sel, 0.0, 0x80000000
8548 // IntOfs = select Sel, 0, 0x80000000
8549 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
8550 SDValue Chain = Op.getOperand(i: 0);
8551 EVT SetCCVT =
8552 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: SrcVT);
8553 EVT DstSetCCVT =
8554 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: DstVT);
8555 SDValue Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
8556 Chain, IsSignaling: true);
8557 Chain = Sel.getValue(R: 1);
8558
8559 SDValue FltOfs = DAG.getSelect(
8560 DL: dl, VT: SrcVT, Cond: Sel, LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
8561 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
8562
8563 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl,
8564 VTList: DAG.getVTList(VT1: SrcVT, VT2: MVT::Other),
8565 Ops: {Chain, Src, FltOfs}, Flags);
8566 Chain = Val.getValue(R: 1);
8567 SDValue SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8568 VTList: DAG.getVTList(VT1: DstVT, VT2: MVT::Other),
8569 Ops: {Chain, Val}, Flags);
8570 Chain = SInt.getValue(R: 1);
8571 SDValue IntOfs = DAG.getSelect(
8572 DL: dl, VT: DstVT, Cond: Sel, LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), RHS: SignMask);
8573 SDValue Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
8574 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
8575 } else {
8576 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
8577 // FIXME: generated code sucks.
8578 SDValue True = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::ppcf128, N1: Src, N2: Cst);
8579 True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: True);
8580 True = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: True, N2: SignMask);
8581 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Src);
8582 return DAG.getSelectCC(DL: dl, LHS: Src, RHS: Cst, True, False, Cond: ISD::SETGE);
8583 }
8584 }
8585 }
8586
8587 return SDValue();
8588 }
8589
8590 if (Subtarget.hasDirectMove() && Subtarget.isPPC64())
8591 return LowerFP_TO_INTDirectMove(Op, DAG, dl);
8592
8593 ReuseLoadInfo RLI;
8594 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8595
8596 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI,
8597 Alignment: RLI.Alignment, MMOFlags: RLI.MMOFlags(),
8598 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8599}
8600
8601// We're trying to insert a regular store, S, and then a load, L. If the
8602// incoming value, O, is a load, we might just be able to have our load use the
8603// address used by O. However, we don't know if anything else will store to
8604// that address before we can load from it. To prevent this situation, we need
8605// to insert our load, L, into the chain as a peer of O. To do this, we give L
8606// the same chain operand as O, we create a token factor from the chain results
8607// of O and L, and we replace all uses of O's chain result with that token
8608// factor (this last part is handled by makeEquivalentMemoryOrdering).
8609bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
8610 ReuseLoadInfo &RLI,
8611 SelectionDAG &DAG,
8612 ISD::LoadExtType ET) const {
8613 // Conservatively skip reusing for constrained FP nodes.
8614 if (Op->isStrictFPOpcode())
8615 return false;
8616
8617 SDLoc dl(Op);
8618 bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT &&
8619 (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32);
8620 if (ET == ISD::NON_EXTLOAD &&
8621 (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) &&
8622 isOperationLegalOrCustom(Op: Op.getOpcode(),
8623 VT: Op.getOperand(i: 0).getValueType())) {
8624
8625 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8626 return true;
8627 }
8628
8629 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op);
8630 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() ||
8631 LD->isNonTemporal())
8632 return false;
8633 if (LD->getMemoryVT() != MemVT)
8634 return false;
8635
8636 // If the result of the load is an illegal type, then we can't build a
8637 // valid chain for reuse since the legalised loads and token factor node that
8638 // ties the legalised loads together uses a different output chain then the
8639 // illegal load.
8640 if (!isTypeLegal(VT: LD->getValueType(ResNo: 0)))
8641 return false;
8642
8643 RLI.Ptr = LD->getBasePtr();
8644 if (LD->isIndexed() && !LD->getOffset().isUndef()) {
8645 assert(LD->getAddressingMode() == ISD::PRE_INC &&
8646 "Non-pre-inc AM on PPC?");
8647 RLI.Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RLI.Ptr.getValueType(), N1: RLI.Ptr,
8648 N2: LD->getOffset());
8649 }
8650
8651 RLI.Chain = LD->getChain();
8652 RLI.MPI = LD->getPointerInfo();
8653 RLI.IsDereferenceable = LD->isDereferenceable();
8654 RLI.IsInvariant = LD->isInvariant();
8655 RLI.Alignment = LD->getAlign();
8656 RLI.AAInfo = LD->getAAInfo();
8657 RLI.Ranges = LD->getRanges();
8658
8659 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1);
8660 return true;
8661}
8662
8663/// Analyze profitability of direct move
8664/// prefer float load to int load plus direct move
8665/// when there is no integer use of int load
8666bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
8667 SDNode *Origin = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0).getNode();
8668 if (Origin->getOpcode() != ISD::LOAD)
8669 return true;
8670
8671 // If there is no LXSIBZX/LXSIHZX, like Power8,
8672 // prefer direct move if the memory size is 1 or 2 bytes.
8673 MachineMemOperand *MMO = cast<LoadSDNode>(Val: Origin)->getMemOperand();
8674 if (!Subtarget.hasP9Vector() &&
8675 (!MMO->getSize().hasValue() || MMO->getSize().getValue() <= 2))
8676 return true;
8677
8678 for (SDUse &Use : Origin->uses()) {
8679
8680 // Only look at the users of the loaded value.
8681 if (Use.getResNo() != 0)
8682 continue;
8683
8684 SDNode *User = Use.getUser();
8685 if (User->getOpcode() != ISD::SINT_TO_FP &&
8686 User->getOpcode() != ISD::UINT_TO_FP &&
8687 User->getOpcode() != ISD::STRICT_SINT_TO_FP &&
8688 User->getOpcode() != ISD::STRICT_UINT_TO_FP)
8689 return true;
8690 }
8691
8692 return false;
8693}
8694
8695static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG,
8696 const PPCSubtarget &Subtarget,
8697 SDValue Chain = SDValue()) {
8698 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8699 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8700 SDLoc dl(Op);
8701
8702 // TODO: Any other flags to propagate?
8703 SDNodeFlags Flags;
8704 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8705
8706 // If we have FCFIDS, then use it when converting to single-precision.
8707 // Otherwise, convert to double-precision and then round.
8708 bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT();
8709 unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS)
8710 : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU);
8711 EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64;
8712 if (Op->isStrictFPOpcode()) {
8713 if (!Chain)
8714 Chain = Op.getOperand(i: 0);
8715 return DAG.getNode(Opcode: getPPCStrictOpcode(Opc: ConvOpc), DL: dl,
8716 VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8717 } else
8718 return DAG.getNode(Opcode: ConvOpc, DL: dl, VT: ConvTy, Operand: Src);
8719}
8720
8721/// Custom lowers integer to floating point conversions to use
8722/// the direct move instructions available in ISA 2.07 to avoid the
8723/// need for load/store combinations.
8724SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
8725 SelectionDAG &DAG,
8726 const SDLoc &dl) const {
8727 assert((Op.getValueType() == MVT::f32 ||
8728 Op.getValueType() == MVT::f64) &&
8729 "Invalid floating point type as target of conversion");
8730 assert(Subtarget.hasFPCVT() &&
8731 "Int to FP conversions with direct moves require FPCVT");
8732 SDValue Src = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
8733 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32;
8734 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP ||
8735 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8736 unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA;
8737 SDValue Mov = DAG.getNode(Opcode: MovOpc, DL: dl, VT: MVT::f64, Operand: Src);
8738 return convertIntToFP(Op, Src: Mov, DAG, Subtarget);
8739}
8740
8741static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
8742
8743 EVT VecVT = Vec.getValueType();
8744 assert(VecVT.isVector() && "Expected a vector type.");
8745 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width.");
8746
8747 EVT EltVT = VecVT.getVectorElementType();
8748 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8749 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8750
8751 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements();
8752 SmallVector<SDValue, 16> Ops(NumConcat);
8753 Ops[0] = Vec;
8754 SDValue UndefVec = DAG.getUNDEF(VT: VecVT);
8755 for (unsigned i = 1; i < NumConcat; ++i)
8756 Ops[i] = UndefVec;
8757
8758 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WideVT, Ops);
8759}
8760
8761SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
8762 const SDLoc &dl) const {
8763 bool IsStrict = Op->isStrictFPOpcode();
8764 unsigned Opc = Op.getOpcode();
8765 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8766 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP ||
8767 Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) &&
8768 "Unexpected conversion type");
8769 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) &&
8770 "Supports conversions to v2f64/v4f32 only.");
8771
8772 // TODO: Any other flags to propagate?
8773 SDNodeFlags Flags;
8774 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8775
8776 bool SignedConv = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
8777 bool FourEltRes = Op.getValueType() == MVT::v4f32;
8778
8779 SDValue Wide = widenVec(DAG, Vec: Src, dl);
8780 EVT WideVT = Wide.getValueType();
8781 unsigned WideNumElts = WideVT.getVectorNumElements();
8782 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64;
8783
8784 SmallVector<int, 16> ShuffV;
8785 for (unsigned i = 0; i < WideNumElts; ++i)
8786 ShuffV.push_back(Elt: i + WideNumElts);
8787
8788 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2;
8789 int SaveElts = FourEltRes ? 4 : 2;
8790 if (Subtarget.isLittleEndian())
8791 for (int i = 0; i < SaveElts; i++)
8792 ShuffV[i * Stride] = i;
8793 else
8794 for (int i = 1; i <= SaveElts; i++)
8795 ShuffV[i * Stride - 1] = i - 1;
8796
8797 SDValue ShuffleSrc2 =
8798 SignedConv ? DAG.getUNDEF(VT: WideVT) : DAG.getConstant(Val: 0, DL: dl, VT: WideVT);
8799 SDValue Arrange = DAG.getVectorShuffle(VT: WideVT, dl, N1: Wide, N2: ShuffleSrc2, Mask: ShuffV);
8800
8801 SDValue Extend;
8802 if (SignedConv) {
8803 Arrange = DAG.getBitcast(VT: IntermediateVT, V: Arrange);
8804 EVT ExtVT = Src.getValueType();
8805 if (Subtarget.hasP9Altivec())
8806 ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideVT.getVectorElementType(),
8807 NumElements: IntermediateVT.getVectorNumElements());
8808
8809 Extend = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: IntermediateVT, N1: Arrange,
8810 N2: DAG.getValueType(ExtVT));
8811 } else
8812 Extend = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntermediateVT, Operand: Arrange);
8813
8814 if (IsStrict)
8815 return DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other),
8816 Ops: {Op.getOperand(i: 0), Extend}, Flags);
8817
8818 return DAG.getNode(Opcode: Opc, DL: dl, VT: Op.getValueType(), Operand: Extend);
8819}
8820
8821SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
8822 SelectionDAG &DAG) const {
8823 SDLoc dl(Op);
8824 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8825 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8826 bool IsStrict = Op->isStrictFPOpcode();
8827 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8828 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
8829
8830 // TODO: Any other flags to propagate?
8831 SDNodeFlags Flags;
8832 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8833
8834 EVT InVT = Src.getValueType();
8835 EVT OutVT = Op.getValueType();
8836 if (OutVT.isVector() && OutVT.isFloatingPoint() &&
8837 isOperationCustom(Op: Op.getOpcode(), VT: InVT))
8838 return LowerINT_TO_FPVector(Op, DAG, dl);
8839
8840 // Conversions to f128 are legal.
8841 if (Op.getValueType() == MVT::f128)
8842 return Subtarget.hasP9Vector() ? Op : SDValue();
8843
8844 // Don't handle ppc_fp128 here; let it be lowered to a libcall.
8845 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
8846 return SDValue();
8847
8848 if (Src.getValueType() == MVT::i1) {
8849 SDValue Sel = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: Op.getValueType(), N1: Src,
8850 N2: DAG.getConstantFP(Val: 1.0, DL: dl, VT: Op.getValueType()),
8851 N3: DAG.getConstantFP(Val: 0.0, DL: dl, VT: Op.getValueType()));
8852 if (IsStrict)
8853 return DAG.getMergeValues(Ops: {Sel, Chain}, dl);
8854 else
8855 return Sel;
8856 }
8857
8858 // If we have direct moves, we can do all the conversion, skip the store/load
8859 // however, without FPCVT we can't do most conversions.
8860 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) &&
8861 Subtarget.isPPC64() && Subtarget.hasFPCVT())
8862 return LowerINT_TO_FPDirectMove(Op, DAG, dl);
8863
8864 assert((IsSigned || Subtarget.hasFPCVT()) &&
8865 "UINT_TO_FP is supported only with FPCVT");
8866
8867 if (Src.getValueType() == MVT::i64) {
8868 SDValue SINT = Src;
8869 // When converting to single-precision, we actually need to convert
8870 // to double-precision first and then round to single-precision.
8871 // To avoid double-rounding effects during that operation, we have
8872 // to prepare the input operand. Bits that might be truncated when
8873 // converting to double-precision are replaced by a bit that won't
8874 // be lost at this stage, but is below the single-precision rounding
8875 // position.
8876 //
8877 // However, if afn is in effect, accept double
8878 // rounding to avoid the extra overhead.
8879 // FIXME: Currently INT_TO_FP can't support fast math flags because
8880 // of nneg flag, thus Op->getFlags().hasApproximateFuncs() is always
8881 // false.
8882 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT() &&
8883 !Op->getFlags().hasApproximateFuncs()) {
8884
8885 // Twiddle input to make sure the low 11 bits are zero. (If this
8886 // is the case, we are guaranteed the value will fit into the 53 bit
8887 // mantissa of an IEEE double-precision value without rounding.)
8888 // If any of those low 11 bits were not zero originally, make sure
8889 // bit 12 (value 2048) is set instead, so that the final rounding
8890 // to single-precision gets the correct result.
8891 SDValue Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64,
8892 N1: SINT, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8893 Round = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8894 N1: Round, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8895 Round = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i64, N1: Round, N2: SINT);
8896 Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64, N1: Round,
8897 N2: DAG.getSignedConstant(Val: -2048, DL: dl, VT: MVT::i64));
8898
8899 // However, we cannot use that value unconditionally: if the magnitude
8900 // of the input value is small, the bit-twiddling we did above might
8901 // end up visibly changing the output. Fortunately, in that case, we
8902 // don't need to twiddle bits since the original input will convert
8903 // exactly to double-precision floating-point already. Therefore,
8904 // construct a conditional to use the original value if the top 11
8905 // bits are all sign-bit copies, and use the rounded value computed
8906 // above otherwise.
8907 SDValue Cond = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: MVT::i64,
8908 N1: SINT, N2: DAG.getConstant(Val: 53, DL: dl, VT: MVT::i32));
8909 Cond = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8910 N1: Cond, N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64));
8911 Cond = DAG.getSetCC(
8912 DL: dl,
8913 VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: MVT::i64),
8914 LHS: Cond, RHS: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64), Cond: ISD::SETUGT);
8915
8916 SINT = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i64, N1: Cond, N2: Round, N3: SINT);
8917 }
8918
8919 ReuseLoadInfo RLI;
8920 SDValue Bits;
8921
8922 MachineFunction &MF = DAG.getMachineFunction();
8923 if (canReuseLoadAddress(Op: SINT, MemVT: MVT::i64, RLI, DAG)) {
8924 // Drop range metadata, as this metadata becomes invalid for f64 bit
8925 // reinterpretation of i64 values.
8926 Bits =
8927 DAG.getLoad(VT: MVT::f64, dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI, Alignment: RLI.Alignment,
8928 MMOFlags: RLI.MMOFlags(), Metadata: MMOMetadata(RLI.AAInfo));
8929 if (RLI.ResChain)
8930 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8931 } else if (Subtarget.hasLFIWAX() &&
8932 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::SEXTLOAD)) {
8933 MachineMemOperand *MMO = MF.getMachineMemOperand(
8934 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8935 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8936 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8937 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWAX, dl,
8938 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8939 Ops, MemVT: MVT::i32, MMO);
8940 if (RLI.ResChain)
8941 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8942 } else if (Subtarget.hasFPCVT() &&
8943 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::ZEXTLOAD)) {
8944 MachineMemOperand *MMO = MF.getMachineMemOperand(
8945 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8946 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8947 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8948 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWZX, dl,
8949 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8950 Ops, MemVT: MVT::i32, MMO);
8951 if (RLI.ResChain)
8952 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8953 } else if (((Subtarget.hasLFIWAX() &&
8954 SINT.getOpcode() == ISD::SIGN_EXTEND) ||
8955 (Subtarget.hasFPCVT() &&
8956 SINT.getOpcode() == ISD::ZERO_EXTEND)) &&
8957 SINT.getOperand(i: 0).getValueType() == MVT::i32) {
8958 MachineFrameInfo &MFI = MF.getFrameInfo();
8959 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8960
8961 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
8962 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
8963
8964 SDValue Store = DAG.getStore(Chain, dl, Val: SINT.getOperand(i: 0), Ptr: FIdx,
8965 PtrInfo: MachinePointerInfo::getFixedStack(
8966 MF&: DAG.getMachineFunction(), FI: FrameIdx));
8967 Chain = Store;
8968
8969 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
8970 "Expected an i32 store");
8971
8972 RLI.Ptr = FIdx;
8973 RLI.Chain = Chain;
8974 RLI.MPI =
8975 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
8976 RLI.Alignment = Align(4);
8977
8978 MachineMemOperand *MMO = MF.getMachineMemOperand(
8979 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8980 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8981 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8982 Bits = DAG.getMemIntrinsicNode(Opcode: SINT.getOpcode() == ISD::ZERO_EXTEND ?
8983 PPCISD::LFIWZX : PPCISD::LFIWAX,
8984 dl, VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8985 Ops, MemVT: MVT::i32, MMO);
8986 Chain = Bits.getValue(R: 1);
8987 } else
8988 Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: SINT);
8989
8990 SDValue FP = convertIntToFP(Op, Src: Bits, DAG, Subtarget, Chain);
8991 if (IsStrict)
8992 Chain = FP.getValue(R: 1);
8993
8994 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
8995 if (IsStrict)
8996 FP = DAG.getNode(
8997 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
8998 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)},
8999 Flags);
9000 else
9001 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9002 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9003 }
9004 return FP;
9005 }
9006
9007 assert(Src.getValueType() == MVT::i32 &&
9008 "Unhandled INT_TO_FP type in custom expander!");
9009 // Since we only generate this in 64-bit mode, we can take advantage of
9010 // 64-bit registers. In particular, sign extend the input value into the
9011 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack
9012 // then lfd it and fcfid it.
9013 MachineFunction &MF = DAG.getMachineFunction();
9014 MachineFrameInfo &MFI = MF.getFrameInfo();
9015 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9016
9017 SDValue Ld;
9018 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
9019 ReuseLoadInfo RLI;
9020 bool ReusingLoad;
9021 if (!(ReusingLoad = canReuseLoadAddress(Op: Src, MemVT: MVT::i32, RLI, DAG))) {
9022 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
9023 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9024
9025 SDValue Store = DAG.getStore(Chain, dl, Val: Src, Ptr: FIdx,
9026 PtrInfo: MachinePointerInfo::getFixedStack(
9027 MF&: DAG.getMachineFunction(), FI: FrameIdx));
9028 Chain = Store;
9029
9030 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
9031 "Expected an i32 store");
9032
9033 RLI.Ptr = FIdx;
9034 RLI.Chain = Chain;
9035 RLI.MPI =
9036 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
9037 RLI.Alignment = Align(4);
9038 }
9039
9040 MachineMemOperand *MMO = MF.getMachineMemOperand(
9041 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
9042 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
9043 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
9044 Ld = DAG.getMemIntrinsicNode(Opcode: IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
9045 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops,
9046 MemVT: MVT::i32, MMO);
9047 Chain = Ld.getValue(R: 1);
9048 if (ReusingLoad && RLI.ResChain) {
9049 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Ld.getValue(R: 1));
9050 }
9051 } else {
9052 assert(Subtarget.isPPC64() &&
9053 "i32->FP without LFIWAX supported only on PPC64");
9054
9055 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9056 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9057
9058 SDValue Ext64 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i64, Operand: Src);
9059
9060 // STD the extended value into the stack slot.
9061 SDValue Store = DAG.getStore(
9062 Chain, dl, Val: Ext64, Ptr: FIdx,
9063 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9064 Chain = Store;
9065
9066 // Load the value as a double.
9067 Ld = DAG.getLoad(
9068 VT: MVT::f64, dl, Chain, Ptr: FIdx,
9069 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9070 Chain = Ld.getValue(R: 1);
9071 }
9072
9073 // FCFID it and return it.
9074 SDValue FP = convertIntToFP(Op, Src: Ld, DAG, Subtarget, Chain);
9075 if (IsStrict)
9076 Chain = FP.getValue(R: 1);
9077 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
9078 if (IsStrict)
9079 FP = DAG.getNode(
9080 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
9081 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)}, Flags);
9082 else
9083 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9084 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9085 }
9086 return FP;
9087}
9088
9089SDValue PPCTargetLowering::LowerSET_ROUNDING(SDValue Op,
9090 SelectionDAG &DAG) const {
9091 SDLoc Dl(Op);
9092 MachineFunction &MF = DAG.getMachineFunction();
9093 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9094 SDValue Chain = Op.getOperand(i: 0);
9095
9096 // If requested mode is constant, just use simpler mtfsb/mffscrni
9097 if (auto *CVal = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
9098 uint64_t Mode = CVal->getZExtValue();
9099 assert(Mode < 4 && "Unsupported rounding mode!");
9100 unsigned InternalRnd = Mode ^ (~(Mode >> 1) & 1);
9101 if (Subtarget.isISA3_0())
9102 return SDValue(
9103 DAG.getMachineNode(
9104 Opcode: PPC::MFFSCRNI, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9105 Ops: {DAG.getConstant(Val: InternalRnd, DL: Dl, VT: MVT::i32, isTarget: true), Chain}),
9106 1);
9107 SDNode *SetHi = DAG.getMachineNode(
9108 Opcode: (InternalRnd & 2) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9109 Ops: {DAG.getConstant(Val: 30, DL: Dl, VT: MVT::i32, isTarget: true), Chain});
9110 SDNode *SetLo = DAG.getMachineNode(
9111 Opcode: (InternalRnd & 1) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9112 Ops: {DAG.getConstant(Val: 31, DL: Dl, VT: MVT::i32, isTarget: true), SDValue(SetHi, 0)});
9113 return SDValue(SetLo, 0);
9114 }
9115
9116 // Use x ^ (~(x >> 1) & 1) to transform LLVM rounding mode to Power format.
9117 SDValue One = DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32);
9118 SDValue SrcFlag = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
9119 N2: DAG.getConstant(Val: 3, DL: Dl, VT: MVT::i32));
9120 SDValue DstFlag = DAG.getNode(
9121 Opcode: ISD::XOR, DL: Dl, VT: MVT::i32, N1: SrcFlag,
9122 N2: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32,
9123 N1: DAG.getNOT(DL: Dl,
9124 Val: DAG.getNode(Opcode: ISD::SRL, DL: Dl, VT: MVT::i32, N1: SrcFlag, N2: One),
9125 VT: MVT::i32),
9126 N2: One));
9127 // For Power9, there's faster mffscrn, and we don't need to read FPSCR
9128 SDValue MFFS;
9129 if (!Subtarget.isISA3_0()) {
9130 MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: Dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9131 Chain = MFFS.getValue(R: 1);
9132 }
9133 SDValue NewFPSCR;
9134 if (Subtarget.isPPC64()) {
9135 if (Subtarget.isISA3_0()) {
9136 NewFPSCR = DAG.getAnyExtOrTrunc(Op: DstFlag, DL: Dl, VT: MVT::i64);
9137 } else {
9138 // Set the last two bits (rounding mode) of bitcasted FPSCR.
9139 SDNode *InsertRN = DAG.getMachineNode(
9140 Opcode: PPC::RLDIMI, dl: Dl, VT: MVT::i64,
9141 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::i64, Operand: MFFS),
9142 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: Dl, VT: MVT::i64, Operand: DstFlag),
9143 DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9144 DAG.getTargetConstant(Val: 62, DL: Dl, VT: MVT::i32)});
9145 NewFPSCR = SDValue(InsertRN, 0);
9146 }
9147 NewFPSCR = DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::f64, Operand: NewFPSCR);
9148 } else {
9149 // In 32-bit mode, store f64, load and update the lower half.
9150 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9151 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9152 SDValue Addr = Subtarget.isLittleEndian()
9153 ? StackSlot
9154 : DAG.getNode(Opcode: ISD::ADD, DL: Dl, VT: PtrVT, N1: StackSlot,
9155 N2: DAG.getConstant(Val: 4, DL: Dl, VT: PtrVT));
9156 if (Subtarget.isISA3_0()) {
9157 Chain = DAG.getStore(Chain, dl: Dl, Val: DstFlag, Ptr: Addr, PtrInfo: MachinePointerInfo());
9158 } else {
9159 Chain = DAG.getStore(Chain, dl: Dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9160 SDValue Tmp =
9161 DAG.getLoad(VT: MVT::i32, dl: Dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9162 Chain = Tmp.getValue(R: 1);
9163 Tmp = SDValue(DAG.getMachineNode(
9164 Opcode: PPC::RLWIMI, dl: Dl, VT: MVT::i32,
9165 Ops: {Tmp, DstFlag, DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9166 DAG.getTargetConstant(Val: 30, DL: Dl, VT: MVT::i32),
9167 DAG.getTargetConstant(Val: 31, DL: Dl, VT: MVT::i32)}),
9168 0);
9169 Chain = DAG.getStore(Chain, dl: Dl, Val: Tmp, Ptr: Addr, PtrInfo: MachinePointerInfo());
9170 }
9171 NewFPSCR =
9172 DAG.getLoad(VT: MVT::f64, dl: Dl, Chain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9173 Chain = NewFPSCR.getValue(R: 1);
9174 }
9175 if (Subtarget.isISA3_0())
9176 return SDValue(DAG.getMachineNode(Opcode: PPC::MFFSCRN, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9177 Ops: {NewFPSCR, Chain}),
9178 1);
9179 SDValue Zero = DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32, isTarget: true);
9180 SDNode *MTFSF = DAG.getMachineNode(
9181 Opcode: PPC::MTFSF, dl: Dl, VT: MVT::Other,
9182 Ops: {DAG.getConstant(Val: 255, DL: Dl, VT: MVT::i32, isTarget: true), NewFPSCR, Zero, Zero, Chain});
9183 return SDValue(MTFSF, 0);
9184}
9185
9186SDValue PPCTargetLowering::LowerGET_ROUNDING(SDValue Op,
9187 SelectionDAG &DAG) const {
9188 SDLoc dl(Op);
9189 /*
9190 The rounding mode is in bits 30:31 of FPSR, and has the following
9191 settings:
9192 00 Round to nearest
9193 01 Round to 0
9194 10 Round to +inf
9195 11 Round to -inf
9196
9197 GET_ROUNDING, on the other hand, expects the following:
9198 -1 Undefined
9199 0 Round to 0
9200 1 Round to nearest
9201 2 Round to +inf
9202 3 Round to -inf
9203
9204 To perform the conversion, we do:
9205 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1))
9206 */
9207
9208 MachineFunction &MF = DAG.getMachineFunction();
9209 EVT VT = Op.getValueType();
9210 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9211
9212 // Save FP Control Word to register
9213 SDValue Chain = Op.getOperand(i: 0);
9214 SDValue MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9215 Chain = MFFS.getValue(R: 1);
9216
9217 SDValue CWD;
9218 if (isTypeLegal(VT: MVT::i64)) {
9219 CWD = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
9220 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: MFFS));
9221 } else {
9222 // Save FP register to stack slot
9223 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9224 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9225 Chain = DAG.getStore(Chain, dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9226
9227 // Load FP Control Word from low 32 bits of stack slot.
9228 assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) &&
9229 "Stack slot adjustment is valid only on big endian subtargets!");
9230 SDValue Four = DAG.getConstant(Val: 4, DL: dl, VT: PtrVT);
9231 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackSlot, N2: Four);
9232 CWD = DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9233 Chain = CWD.getValue(R: 1);
9234 }
9235
9236 // Transform as necessary
9237 SDValue CWD1 =
9238 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9239 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
9240 SDValue CWD2 =
9241 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9242 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9243 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32,
9244 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9245 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9246 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
9247
9248 SDValue RetVal =
9249 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1, N2: CWD2);
9250
9251 RetVal =
9252 DAG.getNode(Opcode: (VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND),
9253 DL: dl, VT, Operand: RetVal);
9254
9255 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
9256}
9257
9258SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9259 EVT VT = Op.getValueType();
9260 uint64_t BitWidth = VT.getSizeInBits();
9261 SDLoc dl(Op);
9262 assert(Op.getNumOperands() == 3 &&
9263 VT == Op.getOperand(1).getValueType() &&
9264 "Unexpected SHL!");
9265
9266 // Expand into a bunch of logical ops. Note that these ops
9267 // depend on the PPC behavior for oversized shift amounts.
9268 SDValue Lo = Op.getOperand(i: 0);
9269 SDValue Hi = Op.getOperand(i: 1);
9270 SDValue Amt = Op.getOperand(i: 2);
9271 EVT AmtVT = Amt.getValueType();
9272
9273 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9274 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9275 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Amt);
9276 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Tmp1);
9277 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR , DL: dl, VT, N1: Tmp2, N2: Tmp3);
9278 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9279 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9280 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Tmp5);
9281 SDValue OutHi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9282 SDValue OutLo = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Amt);
9283 SDValue OutOps[] = { OutLo, OutHi };
9284 return DAG.getMergeValues(Ops: OutOps, dl);
9285}
9286
9287SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9288 EVT VT = Op.getValueType();
9289 SDLoc dl(Op);
9290 uint64_t BitWidth = VT.getSizeInBits();
9291 assert(Op.getNumOperands() == 3 &&
9292 VT == Op.getOperand(1).getValueType() &&
9293 "Unexpected SRL!");
9294
9295 // Expand into a bunch of logical ops. Note that these ops
9296 // depend on the PPC behavior for oversized shift amounts.
9297 SDValue Lo = Op.getOperand(i: 0);
9298 SDValue Hi = Op.getOperand(i: 1);
9299 SDValue Amt = Op.getOperand(i: 2);
9300 EVT AmtVT = Amt.getValueType();
9301
9302 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9303 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9304 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9305 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9306 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9307 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9308 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9309 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Tmp5);
9310 SDValue OutLo = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9311 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Amt);
9312 SDValue OutOps[] = { OutLo, OutHi };
9313 return DAG.getMergeValues(Ops: OutOps, dl);
9314}
9315
9316SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
9317 SDLoc dl(Op);
9318 EVT VT = Op.getValueType();
9319 uint64_t BitWidth = VT.getSizeInBits();
9320 assert(Op.getNumOperands() == 3 &&
9321 VT == Op.getOperand(1).getValueType() &&
9322 "Unexpected SRA!");
9323
9324 // Expand into a bunch of logical ops, followed by a select_cc.
9325 SDValue Lo = Op.getOperand(i: 0);
9326 SDValue Hi = Op.getOperand(i: 1);
9327 SDValue Amt = Op.getOperand(i: 2);
9328 EVT AmtVT = Amt.getValueType();
9329
9330 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9331 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9332 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9333 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9334 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9335 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9336 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9337 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Tmp5);
9338 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Amt);
9339 SDValue OutLo = DAG.getSelectCC(DL: dl, LHS: Tmp5, RHS: DAG.getConstant(Val: 0, DL: dl, VT: AmtVT),
9340 True: Tmp4, False: Tmp6, Cond: ISD::SETLE);
9341 SDValue OutOps[] = { OutLo, OutHi };
9342 return DAG.getMergeValues(Ops: OutOps, dl);
9343}
9344
9345SDValue PPCTargetLowering::LowerFunnelShift(SDValue Op,
9346 SelectionDAG &DAG) const {
9347 SDLoc dl(Op);
9348 EVT VT = Op.getValueType();
9349 unsigned BitWidth = VT.getSizeInBits();
9350
9351 bool IsFSHL = Op.getOpcode() == ISD::FSHL;
9352 SDValue X = Op.getOperand(i: 0);
9353 SDValue Y = Op.getOperand(i: 1);
9354 SDValue Z = Op.getOperand(i: 2);
9355 EVT AmtVT = Z.getValueType();
9356
9357 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
9358 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
9359 // This is simpler than TargetLowering::expandFunnelShift because we can rely
9360 // on PowerPC shift by BW being well defined.
9361 Z = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Z,
9362 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
9363 SDValue SubZ =
9364 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT, N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Z);
9365 X = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: X, N2: IsFSHL ? Z : SubZ);
9366 Y = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Y, N2: IsFSHL ? SubZ : Z);
9367 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: X, N2: Y);
9368}
9369
9370//===----------------------------------------------------------------------===//
9371// Vector related lowering.
9372//
9373
9374/// getCanonicalConstSplat - Build a canonical splat immediate of Val with an
9375/// element size of SplatSize. Cast the result to VT.
9376static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT,
9377 SelectionDAG &DAG, const SDLoc &dl) {
9378 static const MVT VTys[] = { // canonical VT to use for each size.
9379 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32
9380 };
9381
9382 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1];
9383
9384 // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize.
9385 if (Val == ((1LLU << (SplatSize * 8)) - 1)) {
9386 SplatSize = 1;
9387 Val = 0xFF;
9388 }
9389
9390 EVT CanonicalVT = VTys[SplatSize-1];
9391
9392 // Build a canonical splat for this value.
9393 // Explicitly truncate APInt here, as this API is used with a mix of
9394 // signed and unsigned values.
9395 return DAG.getBitcast(
9396 VT: ReqVT,
9397 V: DAG.getConstant(Val: APInt(64, Val).trunc(width: SplatSize * 8), DL: dl, VT: CanonicalVT));
9398}
9399
9400/// BuildIntrinsicOp - Return a unary operator intrinsic node with the
9401/// specified intrinsic ID.
9402static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG,
9403 const SDLoc &dl, EVT DestVT = MVT::Other) {
9404 if (DestVT == MVT::Other) DestVT = Op.getValueType();
9405 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9406 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op);
9407}
9408
9409/// BuildIntrinsicOp - Return a binary operator intrinsic node with the
9410/// specified intrinsic ID.
9411static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
9412 SelectionDAG &DAG, const SDLoc &dl,
9413 EVT DestVT = MVT::Other) {
9414 if (DestVT == MVT::Other) DestVT = LHS.getValueType();
9415 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9416 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: LHS, N3: RHS);
9417}
9418
9419/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the
9420/// specified intrinsic ID.
9421static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
9422 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl,
9423 EVT DestVT = MVT::Other) {
9424 if (DestVT == MVT::Other) DestVT = Op0.getValueType();
9425 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9426 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op0, N3: Op1, N4: Op2);
9427}
9428
9429/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified
9430/// amount. The result has the specified value type.
9431static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
9432 SelectionDAG &DAG, const SDLoc &dl) {
9433 // Force LHS/RHS to be the right type.
9434 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: LHS);
9435 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: RHS);
9436
9437 int Ops[16];
9438 for (unsigned i = 0; i != 16; ++i)
9439 Ops[i] = i + Amt;
9440 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: LHS, N2: RHS, Mask: Ops);
9441 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
9442}
9443
9444/// Do we have an efficient pattern in a .td file for this node?
9445///
9446/// \param V - pointer to the BuildVectorSDNode being matched
9447/// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves?
9448///
9449/// There are some patterns where it is beneficial to keep a BUILD_VECTOR
9450/// node as a BUILD_VECTOR node rather than expanding it. The patterns where
9451/// the opposite is true (expansion is beneficial) are:
9452/// - The node builds a vector out of integers that are not 32 or 64-bits
9453/// - The node builds a vector out of constants
9454/// - The node is a "load-and-splat"
9455/// In all other cases, we will choose to keep the BUILD_VECTOR.
9456static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V,
9457 bool HasDirectMove,
9458 bool HasP8Vector) {
9459 EVT VecVT = V->getValueType(ResNo: 0);
9460 bool RightType = VecVT == MVT::v2f64 ||
9461 (HasP8Vector && VecVT == MVT::v4f32) ||
9462 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32));
9463 if (!RightType)
9464 return false;
9465
9466 bool IsSplat = true;
9467 bool IsLoad = false;
9468 SDValue Op0 = V->getOperand(Num: 0);
9469
9470 // This function is called in a block that confirms the node is not a constant
9471 // splat. So a constant BUILD_VECTOR here means the vector is built out of
9472 // different constants.
9473 if (V->isConstant())
9474 return false;
9475 for (int i = 0, e = V->getNumOperands(); i < e; ++i) {
9476 if (V->getOperand(Num: i).isUndef())
9477 return false;
9478 // We want to expand nodes that represent load-and-splat even if the
9479 // loaded value is a floating point truncation or conversion to int.
9480 if (V->getOperand(Num: i).getOpcode() == ISD::LOAD ||
9481 (V->getOperand(Num: i).getOpcode() == ISD::FP_ROUND &&
9482 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9483 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_SINT &&
9484 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9485 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_UINT &&
9486 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD))
9487 IsLoad = true;
9488 // If the operands are different or the input is not a load and has more
9489 // uses than just this BV node, then it isn't a splat.
9490 if (V->getOperand(Num: i) != Op0 ||
9491 (!IsLoad && !V->isOnlyUserOf(N: V->getOperand(Num: i).getNode())))
9492 IsSplat = false;
9493 }
9494 return !(IsSplat && IsLoad);
9495}
9496
9497// Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128.
9498SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
9499
9500 SDLoc dl(Op);
9501 SDValue Op0 = Op->getOperand(Num: 0);
9502
9503 if (!Subtarget.isPPC64() || (Op0.getOpcode() != ISD::BUILD_PAIR) ||
9504 (Op.getValueType() != MVT::f128))
9505 return SDValue();
9506
9507 SDValue Lo = Op0.getOperand(i: 0);
9508 SDValue Hi = Op0.getOperand(i: 1);
9509 if ((Lo.getValueType() != MVT::i64) || (Hi.getValueType() != MVT::i64))
9510 return SDValue();
9511
9512 if (!Subtarget.isLittleEndian())
9513 std::swap(a&: Lo, b&: Hi);
9514
9515 return DAG.getNode(Opcode: PPCISD::BUILD_FP128, DL: dl, VT: MVT::f128, N1: Lo, N2: Hi);
9516}
9517
9518static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) {
9519 const SDValue *InputLoad = &Op;
9520 while (InputLoad->getOpcode() == ISD::BITCAST)
9521 InputLoad = &InputLoad->getOperand(i: 0);
9522 if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR ||
9523 InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) {
9524 IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED;
9525 InputLoad = &InputLoad->getOperand(i: 0);
9526 }
9527 if (InputLoad->getOpcode() != ISD::LOAD)
9528 return nullptr;
9529 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9530 return ISD::isNormalLoad(N: LD) ? InputLoad : nullptr;
9531}
9532
9533// Convert the argument APFloat to a single precision APFloat if there is no
9534// loss in information during the conversion to single precision APFloat and the
9535// resulting number is not a denormal number. Return true if successful.
9536bool llvm::convertToNonDenormSingle(APFloat &ArgAPFloat) {
9537 APFloat APFloatToConvert = ArgAPFloat;
9538 bool LosesInfo = true;
9539 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9540 losesInfo: &LosesInfo);
9541 bool Success = (!LosesInfo && !APFloatToConvert.isDenormal());
9542 if (Success)
9543 ArgAPFloat = APFloatToConvert;
9544 return Success;
9545}
9546
9547// Bitcast the argument APInt to a double and convert it to a single precision
9548// APFloat, bitcast the APFloat to an APInt and assign it to the original
9549// argument if there is no loss in information during the conversion from
9550// double to single precision APFloat and the resulting number is not a denormal
9551// number. Return true if successful.
9552bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) {
9553 double DpValue = ArgAPInt.bitsToDouble();
9554 APFloat APFloatDp(DpValue);
9555 bool Success = convertToNonDenormSingle(ArgAPFloat&: APFloatDp);
9556 if (Success)
9557 ArgAPInt = APFloatDp.bitcastToAPInt();
9558 return Success;
9559}
9560
9561// Nondestructive check for convertTonNonDenormSingle.
9562bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) {
9563 // Only convert if it loses info, since XXSPLTIDP should
9564 // handle the other case.
9565 APFloat APFloatToConvert = ArgAPFloat;
9566 bool LosesInfo = true;
9567 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9568 losesInfo: &LosesInfo);
9569
9570 return (!LosesInfo && !APFloatToConvert.isDenormal());
9571}
9572
9573static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op,
9574 unsigned &Opcode) {
9575 LoadSDNode *InputNode = dyn_cast<LoadSDNode>(Val: Op.getOperand(i: 0));
9576 if (!InputNode || !Subtarget.hasVSX() || !ISD::isUNINDEXEDLoad(N: InputNode))
9577 return false;
9578
9579 EVT Ty = Op->getValueType(ResNo: 0);
9580 // For v2f64, v4f32 and v4i32 types, we require the load to be non-extending
9581 // as we cannot handle extending loads for these types.
9582 if ((Ty == MVT::v2f64 || Ty == MVT::v4f32 || Ty == MVT::v4i32) &&
9583 ISD::isNON_EXTLoad(N: InputNode))
9584 return true;
9585
9586 EVT MemVT = InputNode->getMemoryVT();
9587 // For v8i16 and v16i8 types, extending loads can be handled as long as the
9588 // memory VT is the same vector element VT type.
9589 // The loads feeding into the v8i16 and v16i8 types will be extending because
9590 // scalar i8/i16 are not legal types.
9591 if ((Ty == MVT::v8i16 || Ty == MVT::v16i8) && ISD::isEXTLoad(N: InputNode) &&
9592 (MemVT == Ty.getVectorElementType()))
9593 return true;
9594
9595 if (Ty == MVT::v2i64) {
9596 // Check the extend type, when the input type is i32, and the output vector
9597 // type is v2i64.
9598 if (MemVT == MVT::i32) {
9599 if (ISD::isZEXTLoad(N: InputNode))
9600 Opcode = PPCISD::ZEXT_LD_SPLAT;
9601 if (ISD::isSEXTLoad(N: InputNode))
9602 Opcode = PPCISD::SEXT_LD_SPLAT;
9603 }
9604 return true;
9605 }
9606 return false;
9607}
9608
9609bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN,
9610 bool IsLittleEndian) {
9611 assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector");
9612
9613 BitMask.clearAllBits();
9614 EVT VT = BVN.getValueType(ResNo: 0);
9615 unsigned VTSize = VT.getSizeInBits();
9616 APInt ConstValue(VTSize, 0);
9617
9618 unsigned EltWidth = VT.getScalarSizeInBits();
9619
9620 unsigned BitPos = 0;
9621 for (auto OpVal : BVN.op_values()) {
9622 auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal);
9623
9624 if (!CN)
9625 return false;
9626 // The elements in a vector register are ordered in reverse byte order
9627 // between little-endian and big-endian modes.
9628 ConstValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth),
9629 bitPosition: IsLittleEndian ? BitPos : VTSize - EltWidth - BitPos);
9630 BitPos += EltWidth;
9631 }
9632
9633 for (unsigned J = 0; J < 16; ++J) {
9634 APInt ExtractValue = ConstValue.extractBits(numBits: 8, bitPosition: J * 8);
9635 if (ExtractValue != 0x00 && ExtractValue != 0xFF)
9636 return false;
9637 if (ExtractValue == 0xFF)
9638 BitMask.setBit(J);
9639 }
9640 return true;
9641}
9642
9643// If this is a case we can't handle, return null and let the default
9644// expansion code take care of it. If we CAN select this case, and if it
9645// selects to a single instruction, return Op. Otherwise, if we can codegen
9646// this case more efficiently than a constant pool load, lower it to the
9647// sequence of ops that should be used.
9648SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
9649 SelectionDAG &DAG) const {
9650 SDLoc dl(Op);
9651 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: Op.getNode());
9652 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
9653
9654 if (Subtarget.hasP10Vector()) {
9655 APInt BitMask(32, 0);
9656 // If the value of the vector is all zeros or all ones,
9657 // we do not convert it to MTVSRBMI.
9658 // The xxleqv instruction sets a vector with all ones.
9659 // The xxlxor instruction sets a vector with all zeros.
9660 if (isValidMtVsrBmi(BitMask, BVN&: *BVN, IsLittleEndian: Subtarget.isLittleEndian()) &&
9661 BitMask != 0 && BitMask != 0xffff) {
9662 SDValue SDConstant = DAG.getTargetConstant(Val: BitMask, DL: dl, VT: MVT::i32);
9663 MachineSDNode *MSDNode =
9664 DAG.getMachineNode(Opcode: PPC::MTVSRBMI, dl, VT: MVT::v16i8, Op1: SDConstant);
9665 SDValue SDV = SDValue(MSDNode, 0);
9666 EVT DVT = BVN->getValueType(ResNo: 0);
9667 EVT SVT = SDV.getValueType();
9668 if (SVT != DVT) {
9669 SDV = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DVT, Operand: SDV);
9670 }
9671 return SDV;
9672 }
9673 // Recognize build vector patterns to emit VSX vector instructions
9674 // instead of loading value from memory.
9675 if (SDValue VecPat = combineBVLoadsSpecialValue(Operand: Op, DAG))
9676 return VecPat;
9677 }
9678 // Check if this is a splat of a constant value.
9679 APInt APSplatBits, APSplatUndef;
9680 unsigned SplatBitSize = 0;
9681 bool HasAnyUndefs;
9682 bool BVNIsConstantSplat =
9683 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
9684 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
9685
9686 // If it is a splat of a double, check if we can shrink it to a 32 bit
9687 // non-denormal float which when converted back to double gives us the same
9688 // double. This is to exploit the XXSPLTIDP instruction.
9689 // If we lose precision, we use XXSPLTI32DX.
9690 if (BVNIsConstantSplat && (SplatBitSize == 64) &&
9691 Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
9692 // Check the type first to short-circuit so we don't modify APSplatBits if
9693 // this block isn't executed.
9694 if ((Op->getValueType(ResNo: 0) == MVT::v2f64) &&
9695 convertToNonDenormSingle(ArgAPInt&: APSplatBits)) {
9696 SDValue SplatNode = DAG.getNode(
9697 Opcode: PPCISD::XXSPLTI_SP_TO_DP, DL: dl, VT: MVT::v2f64,
9698 Operand: DAG.getTargetConstant(Val: APSplatBits.getZExtValue(), DL: dl, VT: MVT::i32));
9699 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9700 } else {
9701 // We may lose precision, so we have to use XXSPLTI32DX.
9702
9703 uint32_t Hi = Hi_32(Value: APSplatBits.getZExtValue());
9704 uint32_t Lo = Lo_32(Value: APSplatBits.getZExtValue());
9705 SDValue SplatNode = DAG.getUNDEF(VT: MVT::v2i64);
9706
9707 if (!Hi || !Lo)
9708 // If either load is 0, then we should generate XXLXOR to set to 0.
9709 SplatNode = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::v2i64);
9710
9711 if (Hi)
9712 SplatNode = DAG.getNode(
9713 Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9714 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32),
9715 N3: DAG.getTargetConstant(Val: Hi, DL: dl, VT: MVT::i32));
9716
9717 if (Lo)
9718 SplatNode =
9719 DAG.getNode(Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9720 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i32),
9721 N3: DAG.getTargetConstant(Val: Lo, DL: dl, VT: MVT::i32));
9722
9723 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9724 }
9725 }
9726
9727 if (SDValue V =
9728 LowerVecSplatSmallFP(Op, DAG, BVNIsConstantSplat, SplatBitSize))
9729 return V;
9730
9731 bool IsSplat64 = false;
9732 uint64_t SplatBits = 0;
9733 int32_t SextVal = 0;
9734 if (BVNIsConstantSplat && SplatBitSize <= 64) {
9735 SplatBits = APSplatBits.getZExtValue();
9736 if (SplatBitSize <= 32) {
9737 SextVal = SignExtend32(X: SplatBits, B: SplatBitSize);
9738 } else if (SplatBitSize == 64 && Subtarget.hasP8Altivec()) {
9739 int64_t Splat64Val = static_cast<int64_t>(SplatBits);
9740 bool P9Vector = Subtarget.hasP9Vector();
9741 int32_t Hi = P9Vector ? 127 : 15;
9742 int32_t Lo = P9Vector ? -128 : -16;
9743 IsSplat64 = Splat64Val >= Lo && Splat64Val <= Hi;
9744 SextVal = static_cast<int32_t>(SplatBits);
9745 }
9746 }
9747
9748 if (!BVNIsConstantSplat || (SplatBitSize > 32 && !IsSplat64)) {
9749 unsigned NewOpcode = PPCISD::LD_SPLAT;
9750
9751 // Handle load-and-splat patterns as we have instructions that will do this
9752 // in one go.
9753 if (DAG.isSplatValue(V: Op, AllowUndefs: true) &&
9754 isValidSplatLoad(Subtarget, Op, Opcode&: NewOpcode)) {
9755 const SDValue *InputLoad = &Op.getOperand(i: 0);
9756 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9757
9758 // If the input load is an extending load, it will be an i32 -> i64
9759 // extending load and isValidSplatLoad() will update NewOpcode.
9760 unsigned MemorySize = LD->getMemoryVT().getScalarSizeInBits();
9761 unsigned ElementSize =
9762 MemorySize * ((NewOpcode == PPCISD::LD_SPLAT) ? 1 : 2);
9763
9764 assert(((ElementSize == 2 * MemorySize)
9765 ? (NewOpcode == PPCISD::ZEXT_LD_SPLAT ||
9766 NewOpcode == PPCISD::SEXT_LD_SPLAT)
9767 : (NewOpcode == PPCISD::LD_SPLAT)) &&
9768 "Unmatched element size and opcode!\n");
9769
9770 // Checking for a single use of this load, we have to check for vector
9771 // width (128 bits) / ElementSize uses (since each operand of the
9772 // BUILD_VECTOR is a separate use of the value.
9773 unsigned NumUsesOfInputLD = 128 / ElementSize;
9774 for (SDValue BVInOp : Op->ops())
9775 if (BVInOp.isUndef())
9776 NumUsesOfInputLD--;
9777
9778 // Exclude somes case where LD_SPLAT is worse than scalar_to_vector:
9779 // Below cases should also happen for "lfiwzx/lfiwax + LE target + index
9780 // 1" and "lxvrhx + BE target + index 7" and "lxvrbx + BE target + index
9781 // 15", but function IsValidSplatLoad() now will only return true when
9782 // the data at index 0 is not nullptr. So we will not get into trouble for
9783 // these cases.
9784 //
9785 // case 1 - lfiwzx/lfiwax
9786 // 1.1: load result is i32 and is sign/zero extend to i64;
9787 // 1.2: build a v2i64 vector type with above loaded value;
9788 // 1.3: the vector has only one value at index 0, others are all undef;
9789 // 1.4: on BE target, so that lfiwzx/lfiwax does not need any permute.
9790 if (NumUsesOfInputLD == 1 &&
9791 (Op->getValueType(ResNo: 0) == MVT::v2i64 && NewOpcode != PPCISD::LD_SPLAT &&
9792 !Subtarget.isLittleEndian() && Subtarget.hasVSX() &&
9793 Subtarget.hasLFIWAX()))
9794 return SDValue();
9795
9796 // case 2 - lxvr[hb]x
9797 // 2.1: load result is at most i16;
9798 // 2.2: build a vector with above loaded value;
9799 // 2.3: the vector has only one value at index 0, others are all undef;
9800 // 2.4: on LE target, so that lxvr[hb]x does not need any permute.
9801 if (NumUsesOfInputLD == 1 && Subtarget.isLittleEndian() &&
9802 Subtarget.isISA3_1() && ElementSize <= 16)
9803 return SDValue();
9804
9805 assert(NumUsesOfInputLD > 0 && "No uses of input LD of a build_vector?");
9806 if (InputLoad->getNode()->hasNUsesOfValue(NUses: NumUsesOfInputLD, Value: 0) &&
9807 Subtarget.hasVSX()) {
9808 SDValue Ops[] = {
9809 LD->getChain(), // Chain
9810 LD->getBasePtr(), // Ptr
9811 DAG.getValueType(Op.getValueType()) // VT
9812 };
9813 SDValue LdSplt = DAG.getMemIntrinsicNode(
9814 Opcode: NewOpcode, dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other), Ops,
9815 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
9816 // Replace all uses of the output chain of the original load with the
9817 // output chain of the new load.
9818 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1),
9819 To: LdSplt.getValue(R: 1));
9820 return LdSplt;
9821 }
9822 }
9823
9824 // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to
9825 // 32-bits can be lowered to VSX instructions under certain conditions.
9826 // Without VSX, there is no pattern more efficient than expanding the node.
9827 if (Subtarget.hasVSX() && Subtarget.isPPC64() &&
9828 haveEfficientBuildVectorPattern(V: BVN, HasDirectMove: Subtarget.hasDirectMove(),
9829 HasP8Vector: Subtarget.hasP8Vector()))
9830 return Op;
9831 return SDValue();
9832 }
9833
9834 uint64_t SplatUndef = APSplatUndef.getZExtValue();
9835 unsigned SplatSize = SplatBitSize / 8;
9836
9837 // First, handle single instruction cases.
9838
9839 // All zeros?
9840 if (SplatBits == 0) {
9841 // Canonicalize all zero vectors to be v4i32.
9842 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) {
9843 SDValue Z = DAG.getConstant(Val: 0, DL: dl, VT: MVT::v4i32);
9844 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Z);
9845 }
9846 return Op;
9847 }
9848
9849 // We have XXSPLTIW for constant splats four bytes wide.
9850 // Given vector length is a multiple of 4, 2-byte splats can be replaced
9851 // with 4-byte splats. We replicate the SplatBits in case of 2-byte splat to
9852 // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be
9853 // turned into a 4-byte splat of 0xABABABAB.
9854 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 2)
9855 return getCanonicalConstSplat(Val: SplatBits | (SplatBits << 16), SplatSize: SplatSize * 2,
9856 VT: Op.getValueType(), DAG, dl);
9857
9858 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 4)
9859 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9860 dl);
9861
9862 // We have XXSPLTIB for constant splats one byte wide.
9863 if (Subtarget.hasP9Vector() && SplatSize == 1)
9864 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9865 dl);
9866
9867 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
9868 // Use VSPLTIW/VUPKLSW for v2i64 in range [-16,15].
9869 if (SextVal >= -16 && SextVal <= 15) {
9870 // SplatSize may be 1, 2, 4, or 8. Use size 4 instead of 8 for the splat to
9871 // generate a splat word with extend for size 8.
9872 unsigned UseSize = SplatSize == 8 ? 4 : SplatSize;
9873 SDValue Res =
9874 getCanonicalConstSplat(Val: SextVal, SplatSize: UseSize, VT: Op.getValueType(), DAG, dl);
9875 if (SplatSize != 8)
9876 return Res;
9877 SDValue IntrinsicOp =
9878 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vupklsw,
9879 Op: DAG.getBitcast(VT: MVT::v4i32, V: Res), DAG, dl, DestVT: MVT::v2i64);
9880 return DAG.getBitcast(VT: Op.getValueType(), V: IntrinsicOp);
9881 }
9882
9883 // Two instruction sequences.
9884
9885 if (Subtarget.hasP9Vector() && SextVal >= -128 && SextVal <= 127) {
9886 SDValue C = DAG.getConstant(Val: (unsigned char)SextVal, DL: dl, VT: MVT::i32);
9887 SmallVector<SDValue, 16> Ops(16, C);
9888 SDValue BV = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops);
9889 unsigned IID;
9890 EVT VT;
9891 switch (SplatSize) {
9892 default:
9893 llvm_unreachable("Unexpected type for vector constant.");
9894 case 2:
9895 IID = Intrinsic::ppc_altivec_vupklsb;
9896 VT = MVT::v8i16;
9897 break;
9898 case 4:
9899 IID = Intrinsic::ppc_altivec_vextsb2w;
9900 VT = MVT::v4i32;
9901 break;
9902 case 8:
9903 IID = Intrinsic::ppc_altivec_vextsb2d;
9904 VT = MVT::v2i64;
9905 break;
9906 }
9907 SDValue Extend = BuildIntrinsicOp(IID, Op: BV, DAG, dl, DestVT: VT);
9908 return DAG.getBitcast(VT: Op->getValueType(ResNo: 0), V: Extend);
9909 }
9910 assert(!IsSplat64 && "Unhandled 64-bit splat pattern");
9911
9912 // If this value is in the range [-32,30] and is even, use:
9913 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2)
9914 // If this value is in the range [17,31] and is odd, use:
9915 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16)
9916 // If this value is in the range [-31,-17] and is odd, use:
9917 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16)
9918 // Note the last two are three-instruction sequences.
9919 if (SextVal >= -32 && SextVal <= 31) {
9920 // To avoid having these optimizations undone by constant folding,
9921 // we convert to a pseudo that will be expanded later into one of
9922 // the above forms.
9923 SDValue Elt = DAG.getSignedConstant(Val: SextVal, DL: dl, VT: MVT::i32);
9924 EVT VT = (SplatSize == 1 ? MVT::v16i8 :
9925 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32));
9926 SDValue EltSize = DAG.getConstant(Val: SplatSize, DL: dl, VT: MVT::i32);
9927 SDValue RetVal = DAG.getNode(Opcode: PPCISD::VADD_SPLAT, DL: dl, VT, N1: Elt, N2: EltSize);
9928 if (VT == Op.getValueType())
9929 return RetVal;
9930 else
9931 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: RetVal);
9932 }
9933
9934 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is
9935 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important
9936 // for fneg/fabs.
9937 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) {
9938 // Make -1 and vspltisw -1:
9939 SDValue OnesV = getCanonicalConstSplat(Val: -1, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
9940
9941 // Make the VSLW intrinsic, computing 0x8000_0000.
9942 SDValue Res = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: OnesV,
9943 RHS: OnesV, DAG, dl);
9944
9945 // xor by OnesV to invert it.
9946 Res = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::v4i32, N1: Res, N2: OnesV);
9947 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9948 }
9949
9950 // Check to see if this is a wide variety of vsplti*, binop self cases.
9951 static const signed char SplatCsts[] = {
9952 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7,
9953 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16
9954 };
9955
9956 for (unsigned idx = 0; idx < std::size(SplatCsts); ++idx) {
9957 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for
9958 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1'
9959 int i = SplatCsts[idx];
9960
9961 // Figure out what shift amount will be used by altivec if shifted by i in
9962 // this splat size.
9963 unsigned TypeShiftAmt = i & (SplatBitSize-1);
9964
9965 // vsplti + shl self.
9966 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
9967 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9968 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9969 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
9970 Intrinsic::ppc_altivec_vslw
9971 };
9972 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9973 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9974 }
9975
9976 // vsplti + srl self.
9977 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
9978 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9979 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9980 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0,
9981 Intrinsic::ppc_altivec_vsrw
9982 };
9983 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9984 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9985 }
9986
9987 // vsplti + rol self.
9988 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) |
9989 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) {
9990 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9991 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9992 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0,
9993 Intrinsic::ppc_altivec_vrlw
9994 };
9995 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9996 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9997 }
9998
9999 // t = vsplti c, result = vsldoi t, t, 1
10000 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
10001 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10002 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1;
10003 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10004 }
10005 // t = vsplti c, result = vsldoi t, t, 2
10006 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
10007 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10008 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2;
10009 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10010 }
10011 // t = vsplti c, result = vsldoi t, t, 3
10012 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
10013 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10014 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3;
10015 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10016 }
10017 }
10018
10019 return SDValue();
10020}
10021
10022/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
10023/// the specified operations to build the shuffle.
10024static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
10025 SDValue RHS, SelectionDAG &DAG,
10026 const SDLoc &dl) {
10027 unsigned OpNum = (PFEntry >> 26) & 0x0F;
10028 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
10029 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
10030
10031 enum {
10032 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
10033 OP_VMRGHW,
10034 OP_VMRGLW,
10035 OP_VSPLTISW0,
10036 OP_VSPLTISW1,
10037 OP_VSPLTISW2,
10038 OP_VSPLTISW3,
10039 OP_VSLDOI4,
10040 OP_VSLDOI8,
10041 OP_VSLDOI12
10042 };
10043
10044 if (OpNum == OP_COPY) {
10045 if (LHSID == (1*9+2)*9+3) return LHS;
10046 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
10047 return RHS;
10048 }
10049
10050 SDValue OpLHS, OpRHS;
10051 OpLHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
10052 OpRHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
10053
10054 int ShufIdxs[16];
10055 switch (OpNum) {
10056 default: llvm_unreachable("Unknown i32 permute!");
10057 case OP_VMRGHW:
10058 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3;
10059 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19;
10060 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7;
10061 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23;
10062 break;
10063 case OP_VMRGLW:
10064 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11;
10065 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27;
10066 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15;
10067 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31;
10068 break;
10069 case OP_VSPLTISW0:
10070 for (unsigned i = 0; i != 16; ++i)
10071 ShufIdxs[i] = (i&3)+0;
10072 break;
10073 case OP_VSPLTISW1:
10074 for (unsigned i = 0; i != 16; ++i)
10075 ShufIdxs[i] = (i&3)+4;
10076 break;
10077 case OP_VSPLTISW2:
10078 for (unsigned i = 0; i != 16; ++i)
10079 ShufIdxs[i] = (i&3)+8;
10080 break;
10081 case OP_VSPLTISW3:
10082 for (unsigned i = 0; i != 16; ++i)
10083 ShufIdxs[i] = (i&3)+12;
10084 break;
10085 case OP_VSLDOI4:
10086 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 4, VT: OpLHS.getValueType(), DAG, dl);
10087 case OP_VSLDOI8:
10088 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 8, VT: OpLHS.getValueType(), DAG, dl);
10089 case OP_VSLDOI12:
10090 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 12, VT: OpLHS.getValueType(), DAG, dl);
10091 }
10092 EVT VT = OpLHS.getValueType();
10093 OpLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpLHS);
10094 OpRHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpRHS);
10095 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OpLHS, N2: OpRHS, Mask: ShufIdxs);
10096 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
10097}
10098
10099/// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled
10100/// by the VINSERTB instruction introduced in ISA 3.0, else just return default
10101/// SDValue.
10102SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N,
10103 SelectionDAG &DAG) const {
10104 const unsigned BytesInVector = 16;
10105 bool IsLE = Subtarget.isLittleEndian();
10106 SDLoc dl(N);
10107 SDValue V1 = N->getOperand(Num: 0);
10108 SDValue V2 = N->getOperand(Num: 1);
10109 unsigned ShiftElts = 0, InsertAtByte = 0;
10110 bool Swap = false;
10111
10112 // Shifts required to get the byte we want at element 7.
10113 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1,
10114 0, 15, 14, 13, 12, 11, 10, 9};
10115 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0,
10116 1, 2, 3, 4, 5, 6, 7, 8};
10117
10118 ArrayRef<int> Mask = N->getMask();
10119 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
10120
10121 // For each mask element, find out if we're just inserting something
10122 // from V2 into V1 or vice versa.
10123 // Possible permutations inserting an element from V2 into V1:
10124 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10125 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10126 // ...
10127 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X
10128 // Inserting from V1 into V2 will be similar, except mask range will be
10129 // [16,31].
10130
10131 bool FoundCandidate = false;
10132 // If both vector operands for the shuffle are the same vector, the mask
10133 // will contain only elements from the first one and the second one will be
10134 // undef.
10135 unsigned VINSERTBSrcElem = IsLE ? 8 : 7;
10136 // Go through the mask of half-words to find an element that's being moved
10137 // from one vector to the other.
10138 for (unsigned i = 0; i < BytesInVector; ++i) {
10139 unsigned CurrentElement = Mask[i];
10140 // If 2nd operand is undefined, we should only look for element 7 in the
10141 // Mask.
10142 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem)
10143 continue;
10144
10145 bool OtherElementsInOrder = true;
10146 // Examine the other elements in the Mask to see if they're in original
10147 // order.
10148 for (unsigned j = 0; j < BytesInVector; ++j) {
10149 if (j == i)
10150 continue;
10151 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be
10152 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined,
10153 // in which we always assume we're always picking from the 1st operand.
10154 int MaskOffset =
10155 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0;
10156 if (Mask[j] != OriginalOrder[j] + MaskOffset) {
10157 OtherElementsInOrder = false;
10158 break;
10159 }
10160 }
10161 // If other elements are in original order, we record the number of shifts
10162 // we need to get the element we want into element 7. Also record which byte
10163 // in the vector we should insert into.
10164 if (OtherElementsInOrder) {
10165 // If 2nd operand is undefined, we assume no shifts and no swapping.
10166 if (V2.isUndef()) {
10167 ShiftElts = 0;
10168 Swap = false;
10169 } else {
10170 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4.
10171 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF]
10172 : BigEndianShifts[CurrentElement & 0xF];
10173 Swap = CurrentElement < BytesInVector;
10174 }
10175 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i;
10176 FoundCandidate = true;
10177 break;
10178 }
10179 }
10180
10181 if (!FoundCandidate)
10182 return SDValue();
10183
10184 // Candidate found, construct the proper SDAG sequence with VINSERTB,
10185 // optionally with VECSHL if shift is required.
10186 if (Swap)
10187 std::swap(a&: V1, b&: V2);
10188 if (V2.isUndef())
10189 V2 = V1;
10190 if (ShiftElts) {
10191 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10192 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10193 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: Shl,
10194 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10195 }
10196 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: V2,
10197 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10198}
10199
10200/// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled
10201/// by the VINSERTH instruction introduced in ISA 3.0, else just return default
10202/// SDValue.
10203SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N,
10204 SelectionDAG &DAG) const {
10205 const unsigned NumHalfWords = 8;
10206 const unsigned BytesInVector = NumHalfWords * 2;
10207 // Check that the shuffle is on half-words.
10208 if (!isNByteElemShuffleMask(N, Width: 2, StepLen: 1))
10209 return SDValue();
10210
10211 bool IsLE = Subtarget.isLittleEndian();
10212 SDLoc dl(N);
10213 SDValue V1 = N->getOperand(Num: 0);
10214 SDValue V2 = N->getOperand(Num: 1);
10215 unsigned ShiftElts = 0, InsertAtByte = 0;
10216 bool Swap = false;
10217
10218 // Shifts required to get the half-word we want at element 3.
10219 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5};
10220 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4};
10221
10222 uint32_t Mask = 0;
10223 uint32_t OriginalOrderLow = 0x1234567;
10224 uint32_t OriginalOrderHigh = 0x89ABCDEF;
10225 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a
10226 // 32-bit space, only need 4-bit nibbles per element.
10227 for (unsigned i = 0; i < NumHalfWords; ++i) {
10228 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10229 Mask |= ((uint32_t)(N->getMaskElt(Idx: i * 2) / 2) << MaskShift);
10230 }
10231
10232 // For each mask element, find out if we're just inserting something
10233 // from V2 into V1 or vice versa. Possible permutations inserting an element
10234 // from V2 into V1:
10235 // X, 1, 2, 3, 4, 5, 6, 7
10236 // 0, X, 2, 3, 4, 5, 6, 7
10237 // 0, 1, X, 3, 4, 5, 6, 7
10238 // 0, 1, 2, X, 4, 5, 6, 7
10239 // 0, 1, 2, 3, X, 5, 6, 7
10240 // 0, 1, 2, 3, 4, X, 6, 7
10241 // 0, 1, 2, 3, 4, 5, X, 7
10242 // 0, 1, 2, 3, 4, 5, 6, X
10243 // Inserting from V1 into V2 will be similar, except mask range will be [8,15].
10244
10245 bool FoundCandidate = false;
10246 // Go through the mask of half-words to find an element that's being moved
10247 // from one vector to the other.
10248 for (unsigned i = 0; i < NumHalfWords; ++i) {
10249 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10250 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF;
10251 uint32_t MaskOtherElts = ~(0xF << MaskShift);
10252 uint32_t TargetOrder = 0x0;
10253
10254 // If both vector operands for the shuffle are the same vector, the mask
10255 // will contain only elements from the first one and the second one will be
10256 // undef.
10257 if (V2.isUndef()) {
10258 ShiftElts = 0;
10259 unsigned VINSERTHSrcElem = IsLE ? 4 : 3;
10260 TargetOrder = OriginalOrderLow;
10261 Swap = false;
10262 // Skip if not the correct element or mask of other elements don't equal
10263 // to our expected order.
10264 if (MaskOneElt == VINSERTHSrcElem &&
10265 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10266 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10267 FoundCandidate = true;
10268 break;
10269 }
10270 } else { // If both operands are defined.
10271 // Target order is [8,15] if the current mask is between [0,7].
10272 TargetOrder =
10273 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow;
10274 // Skip if mask of other elements don't equal our expected order.
10275 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10276 // We only need the last 3 bits for the number of shifts.
10277 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7]
10278 : BigEndianShifts[MaskOneElt & 0x7];
10279 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10280 Swap = MaskOneElt < NumHalfWords;
10281 FoundCandidate = true;
10282 break;
10283 }
10284 }
10285 }
10286
10287 if (!FoundCandidate)
10288 return SDValue();
10289
10290 // Candidate found, construct the proper SDAG sequence with VINSERTH,
10291 // optionally with VECSHL if shift is required.
10292 if (Swap)
10293 std::swap(a&: V1, b&: V2);
10294 if (V2.isUndef())
10295 V2 = V1;
10296 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10297 if (ShiftElts) {
10298 // Double ShiftElts because we're left shifting on v16i8 type.
10299 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10300 N3: DAG.getConstant(Val: 2 * ShiftElts, DL: dl, VT: MVT::i32));
10301 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: Shl);
10302 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10303 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10304 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10305 }
10306 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V2);
10307 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10308 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10309 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10310}
10311
10312/// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be
10313/// handled by the XXSPLTI32DX instruction introduced in ISA 3.1, otherwise
10314/// return the default SDValue.
10315SDValue PPCTargetLowering::lowerToXXSPLTI32DX(ShuffleVectorSDNode *SVN,
10316 SelectionDAG &DAG) const {
10317 // The LHS and RHS may be bitcasts to v16i8 as we canonicalize shuffles
10318 // to v16i8. Peek through the bitcasts to get the actual operands.
10319 SDValue LHS = peekThroughBitcasts(V: SVN->getOperand(Num: 0));
10320 SDValue RHS = peekThroughBitcasts(V: SVN->getOperand(Num: 1));
10321
10322 auto ShuffleMask = SVN->getMask();
10323 SDValue VecShuffle(SVN, 0);
10324 SDLoc DL(SVN);
10325
10326 // Check that we have a four byte shuffle.
10327 if (!isNByteElemShuffleMask(N: SVN, Width: 4, StepLen: 1))
10328 return SDValue();
10329
10330 // Canonicalize the RHS being a BUILD_VECTOR when lowering to xxsplti32dx.
10331 if (RHS->getOpcode() != ISD::BUILD_VECTOR) {
10332 std::swap(a&: LHS, b&: RHS);
10333 VecShuffle = peekThroughBitcasts(V: DAG.getCommutedVectorShuffle(SV: *SVN));
10334 ShuffleVectorSDNode *CommutedSV = dyn_cast<ShuffleVectorSDNode>(Val&: VecShuffle);
10335 if (!CommutedSV)
10336 return SDValue();
10337 ShuffleMask = CommutedSV->getMask();
10338 }
10339
10340 // Ensure that the RHS is a vector of constants.
10341 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: RHS.getNode());
10342 if (!BVN)
10343 return SDValue();
10344
10345 // Check if RHS is a splat of 4-bytes (or smaller).
10346 APInt APSplatValue, APSplatUndef;
10347 unsigned SplatBitSize;
10348 bool HasAnyUndefs;
10349 if (!BVN->isConstantSplat(SplatValue&: APSplatValue, SplatUndef&: APSplatUndef, SplatBitSize,
10350 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian()) ||
10351 SplatBitSize > 32)
10352 return SDValue();
10353
10354 // Check that the shuffle mask matches the semantics of XXSPLTI32DX.
10355 // The instruction splats a constant C into two words of the source vector
10356 // producing { C, Unchanged, C, Unchanged } or { Unchanged, C, Unchanged, C }.
10357 // Thus we check that the shuffle mask is the equivalent of
10358 // <0, [4-7], 2, [4-7]> or <[4-7], 1, [4-7], 3> respectively.
10359 // Note: the check above of isNByteElemShuffleMask() ensures that the bytes
10360 // within each word are consecutive, so we only need to check the first byte.
10361 SDValue Index;
10362 bool IsLE = Subtarget.isLittleEndian();
10363 if ((ShuffleMask[0] == 0 && ShuffleMask[8] == 8) &&
10364 (ShuffleMask[4] % 4 == 0 && ShuffleMask[12] % 4 == 0 &&
10365 ShuffleMask[4] > 15 && ShuffleMask[12] > 15))
10366 Index = DAG.getTargetConstant(Val: IsLE ? 0 : 1, DL, VT: MVT::i32);
10367 else if ((ShuffleMask[4] == 4 && ShuffleMask[12] == 12) &&
10368 (ShuffleMask[0] % 4 == 0 && ShuffleMask[8] % 4 == 0 &&
10369 ShuffleMask[0] > 15 && ShuffleMask[8] > 15))
10370 Index = DAG.getTargetConstant(Val: IsLE ? 1 : 0, DL, VT: MVT::i32);
10371 else
10372 return SDValue();
10373
10374 // If the splat is narrower than 32-bits, we need to get the 32-bit value
10375 // for XXSPLTI32DX.
10376 unsigned SplatVal = APSplatValue.getZExtValue();
10377 for (; SplatBitSize < 32; SplatBitSize <<= 1)
10378 SplatVal |= (SplatVal << SplatBitSize);
10379
10380 SDValue SplatNode = DAG.getNode(
10381 Opcode: PPCISD::XXSPLTI32DX, DL, VT: MVT::v2i64, N1: DAG.getBitcast(VT: MVT::v2i64, V: LHS),
10382 N2: Index, N3: DAG.getTargetConstant(Val: SplatVal, DL, VT: MVT::i32));
10383 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: SplatNode);
10384}
10385
10386/// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8).
10387/// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is
10388/// a multiple of 8. Otherwise convert it to a scalar rotation(i128)
10389/// i.e (or (shl x, C1), (srl x, 128-C1)).
10390SDValue PPCTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
10391 assert(Op.getOpcode() == ISD::ROTL && "Should only be called for ISD::ROTL");
10392 assert(Op.getValueType() == MVT::v1i128 &&
10393 "Only set v1i128 as custom, other type shouldn't reach here!");
10394 SDLoc dl(Op);
10395 SDValue N0 = peekThroughBitcasts(V: Op.getOperand(i: 0));
10396 SDValue N1 = peekThroughBitcasts(V: Op.getOperand(i: 1));
10397 unsigned SHLAmt = N1.getConstantOperandVal(i: 0);
10398 if (SHLAmt % 8 == 0) {
10399 std::array<int, 16> Mask;
10400 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
10401 std::rotate(first: Mask.begin(), middle: Mask.begin() + SHLAmt / 8, last: Mask.end());
10402 if (SDValue Shuffle =
10403 DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: DAG.getBitcast(VT: MVT::v16i8, V: N0),
10404 N2: DAG.getUNDEF(VT: MVT::v16i8), Mask))
10405 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: Shuffle);
10406 }
10407 SDValue ArgVal = DAG.getBitcast(VT: MVT::i128, V: N0);
10408 SDValue SHLOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ArgVal,
10409 N2: DAG.getConstant(Val: SHLAmt, DL: dl, VT: MVT::i32));
10410 SDValue SRLOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: ArgVal,
10411 N2: DAG.getConstant(Val: 128 - SHLAmt, DL: dl, VT: MVT::i32));
10412 SDValue OROp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i128, N1: SHLOp, N2: SRLOp);
10413 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: OROp);
10414}
10415
10416/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this
10417/// is a shuffle we can handle in a single instruction, return it. Otherwise,
10418/// return the code it can be lowered into. Worst case, it can always be
10419/// lowered into a vperm.
10420SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
10421 SelectionDAG &DAG) const {
10422 SDLoc dl(Op);
10423 SDValue V1 = Op.getOperand(i: 0);
10424 SDValue V2 = Op.getOperand(i: 1);
10425 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10426
10427 // Any nodes that were combined in the target-independent combiner prior
10428 // to vector legalization will not be sent to the target combine. Try to
10429 // combine it here.
10430 if (SDValue NewShuffle = combineVectorShuffle(SVN: SVOp, DAG)) {
10431 if (!isa<ShuffleVectorSDNode>(Val: NewShuffle))
10432 return NewShuffle;
10433 Op = NewShuffle;
10434 SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10435 V1 = Op.getOperand(i: 0);
10436 V2 = Op.getOperand(i: 1);
10437 }
10438 EVT VT = Op.getValueType();
10439 bool isLittleEndian = Subtarget.isLittleEndian();
10440
10441 unsigned ShiftElts, InsertAtByte;
10442 bool Swap = false;
10443
10444 // If this is a load-and-splat, we can do that with a single instruction
10445 // in some cases. However if the load has multiple uses, we don't want to
10446 // combine it because that will just produce multiple loads.
10447 bool IsPermutedLoad = false;
10448 const SDValue *InputLoad = getNormalLoadInput(Op: V1, IsPermuted&: IsPermutedLoad);
10449 if (InputLoad && Subtarget.hasVSX() && V2.isUndef() &&
10450 (PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) || PPC::isSplatShuffleMask(N: SVOp, EltSize: 8)) &&
10451 InputLoad->hasOneUse()) {
10452 bool IsFourByte = PPC::isSplatShuffleMask(N: SVOp, EltSize: 4);
10453 int SplatIdx =
10454 PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: IsFourByte ? 4 : 8, DAG);
10455
10456 // The splat index for permuted loads will be in the left half of the vector
10457 // which is strictly wider than the loaded value by 8 bytes. So we need to
10458 // adjust the splat index to point to the correct address in memory.
10459 if (IsPermutedLoad) {
10460 assert((isLittleEndian || IsFourByte) &&
10461 "Unexpected size for permuted load on big endian target");
10462 SplatIdx += IsFourByte ? 2 : 1;
10463 assert((SplatIdx < (IsFourByte ? 4 : 2)) &&
10464 "Splat of a value outside of the loaded memory");
10465 }
10466
10467 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
10468 // For 4-byte load-and-splat, we need Power9.
10469 if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) {
10470 uint64_t Offset = 0;
10471 if (IsFourByte)
10472 Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4;
10473 else
10474 Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8;
10475
10476 // If the width of the load is the same as the width of the splat,
10477 // loading with an offset would load the wrong memory.
10478 if (LD->getValueType(ResNo: 0).getSizeInBits() == (IsFourByte ? 32 : 64))
10479 Offset = 0;
10480
10481 SDValue BasePtr = LD->getBasePtr();
10482 if (Offset != 0)
10483 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
10484 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: Offset, DL: dl));
10485 SDValue Ops[] = {
10486 LD->getChain(), // Chain
10487 BasePtr, // BasePtr
10488 DAG.getValueType(Op.getValueType()) // VT
10489 };
10490 SDVTList VTL =
10491 DAG.getVTList(VT1: IsFourByte ? MVT::v4i32 : MVT::v2i64, VT2: MVT::Other);
10492 SDValue LdSplt =
10493 DAG.getMemIntrinsicNode(Opcode: PPCISD::LD_SPLAT, dl, VTList: VTL,
10494 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
10495 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1), To: LdSplt.getValue(R: 1));
10496 if (LdSplt.getValueType() != SVOp->getValueType(ResNo: 0))
10497 LdSplt = DAG.getBitcast(VT: SVOp->getValueType(ResNo: 0), V: LdSplt);
10498 return LdSplt;
10499 }
10500 }
10501
10502 // All v2i64 and v2f64 shuffles are legal
10503 if (VT == MVT::v2i64 || VT == MVT::v2f64)
10504 return Op;
10505
10506 if (Subtarget.hasP9Vector() &&
10507 PPC::isXXINSERTWMask(N: SVOp, ShiftElts, InsertAtByte, Swap,
10508 IsLE: isLittleEndian)) {
10509 if (V2.isUndef())
10510 V2 = V1;
10511 else if (Swap)
10512 std::swap(a&: V1, b&: V2);
10513 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10514 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2);
10515 if (ShiftElts) {
10516 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv2, N2: Conv2,
10517 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10518 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Shl,
10519 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10520 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10521 }
10522 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10523 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10524 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10525 }
10526
10527 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
10528 SDValue SplatInsertNode;
10529 if ((SplatInsertNode = lowerToXXSPLTI32DX(SVN: SVOp, DAG)))
10530 return SplatInsertNode;
10531 }
10532
10533 if (Subtarget.hasP9Altivec()) {
10534 SDValue NewISDNode;
10535 if ((NewISDNode = lowerToVINSERTH(N: SVOp, DAG)))
10536 return NewISDNode;
10537
10538 if ((NewISDNode = lowerToVINSERTB(N: SVOp, DAG)))
10539 return NewISDNode;
10540 }
10541
10542 if (Subtarget.hasVSX() &&
10543 PPC::isXXSLDWIShuffleMask(N: SVOp, ShiftElts, Swap, IsLE: isLittleEndian)) {
10544 if (Swap)
10545 std::swap(a&: V1, b&: V2);
10546 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10547 SDValue Conv2 =
10548 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2.isUndef() ? V1 : V2);
10549
10550 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10551 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10552 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Shl);
10553 }
10554
10555 if (Subtarget.hasVSX() &&
10556 PPC::isXXPERMDIShuffleMask(N: SVOp, DM&: ShiftElts, Swap, IsLE: isLittleEndian)) {
10557 if (Swap)
10558 std::swap(a&: V1, b&: V2);
10559 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10560 SDValue Conv2 =
10561 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V2.isUndef() ? V1 : V2);
10562
10563 SDValue PermDI = DAG.getNode(Opcode: PPCISD::XXPERMDI, DL: dl, VT: MVT::v2i64, N1: Conv1, N2: Conv2,
10564 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10565 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: PermDI);
10566 }
10567
10568 if (Subtarget.hasP9Vector()) {
10569 if (PPC::isXXBRHShuffleMask(N: SVOp)) {
10570 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10571 SDValue ReveHWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v8i16, Operand: Conv);
10572 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveHWord);
10573 } else if (PPC::isXXBRWShuffleMask(N: SVOp)) {
10574 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10575 SDValue ReveWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v4i32, Operand: Conv);
10576 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveWord);
10577 } else if (PPC::isXXBRDShuffleMask(N: SVOp)) {
10578 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10579 SDValue ReveDWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Conv);
10580 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveDWord);
10581 } else if (PPC::isXXBRQShuffleMask(N: SVOp)) {
10582 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: V1);
10583 SDValue ReveQWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v1i128, Operand: Conv);
10584 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveQWord);
10585 }
10586 }
10587
10588 if (Subtarget.hasVSX()) {
10589 if (V2.isUndef() && PPC::isSplatShuffleMask(N: SVOp, EltSize: 4)) {
10590 int SplatIdx = PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: 4, DAG);
10591
10592 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10593 SDValue Splat = DAG.getNode(Opcode: PPCISD::XXSPLT, DL: dl, VT: MVT::v4i32, N1: Conv,
10594 N2: DAG.getConstant(Val: SplatIdx, DL: dl, VT: MVT::i32));
10595 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Splat);
10596 }
10597
10598 // Left shifts of 8 bytes are actually swaps. Convert accordingly.
10599 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) == 8) {
10600 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: V1);
10601 SDValue Swap = DAG.getNode(Opcode: PPCISD::SWAP_NO_CHAIN, DL: dl, VT: MVT::v2f64, Operand: Conv);
10602 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Swap);
10603 }
10604 }
10605
10606 // Cases that are handled by instructions that take permute immediates
10607 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
10608 // selected by the instruction selector.
10609 if (V2.isUndef()) {
10610 if (PPC::isSplatShuffleMask(N: SVOp, EltSize: 1) ||
10611 PPC::isSplatShuffleMask(N: SVOp, EltSize: 2) ||
10612 PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) ||
10613 PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10614 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10615 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) != -1 ||
10616 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10617 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10618 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10619 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10620 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10621 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10622 (Subtarget.hasP8Altivec() && (
10623 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10624 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind: 1, DAG) ||
10625 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind: 1, DAG)))) {
10626 return Op;
10627 }
10628 }
10629
10630 // Altivec has a variety of "shuffle immediates" that take two vector inputs
10631 // and produce a fixed permutation. If any of these match, do not lower to
10632 // VPERM.
10633 unsigned int ShuffleKind = isLittleEndian ? 2 : 0;
10634 if (PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10635 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10636 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind, DAG) != -1 ||
10637 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10638 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10639 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10640 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10641 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10642 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10643 (Subtarget.hasP8Altivec() && (
10644 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10645 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind, DAG) ||
10646 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind, DAG))))
10647 return Op;
10648
10649 // Check to see if this is a shuffle of 4-byte values. If so, we can use our
10650 // perfect shuffle table to emit an optimal matching sequence.
10651 ArrayRef<int> PermMask = SVOp->getMask();
10652
10653 if (!DisablePerfectShuffle && !isLittleEndian) {
10654 unsigned PFIndexes[4];
10655 bool isFourElementShuffle = true;
10656 for (unsigned i = 0; i != 4 && isFourElementShuffle;
10657 ++i) { // Element number
10658 unsigned EltNo = 8; // Start out undef.
10659 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte.
10660 if (PermMask[i * 4 + j] < 0)
10661 continue; // Undef, ignore it.
10662
10663 unsigned ByteSource = PermMask[i * 4 + j];
10664 if ((ByteSource & 3) != j) {
10665 isFourElementShuffle = false;
10666 break;
10667 }
10668
10669 if (EltNo == 8) {
10670 EltNo = ByteSource / 4;
10671 } else if (EltNo != ByteSource / 4) {
10672 isFourElementShuffle = false;
10673 break;
10674 }
10675 }
10676 PFIndexes[i] = EltNo;
10677 }
10678
10679 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the
10680 // perfect shuffle vector to determine if it is cost effective to do this as
10681 // discrete instructions, or whether we should use a vperm.
10682 // For now, we skip this for little endian until such time as we have a
10683 // little-endian perfect shuffle table.
10684 if (isFourElementShuffle) {
10685 // Compute the index in the perfect shuffle table.
10686 unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
10687 PFIndexes[2] * 9 + PFIndexes[3];
10688
10689 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
10690 unsigned Cost = (PFEntry >> 30);
10691
10692 // Determining when to avoid vperm is tricky. Many things affect the cost
10693 // of vperm, particularly how many times the perm mask needs to be
10694 // computed. For example, if the perm mask can be hoisted out of a loop or
10695 // is already used (perhaps because there are multiple permutes with the
10696 // same shuffle mask?) the vperm has a cost of 1. OTOH, hoisting the
10697 // permute mask out of the loop requires an extra register.
10698 //
10699 // As a compromise, we only emit discrete instructions if the shuffle can
10700 // be generated in 3 or fewer operations. When we have loop information
10701 // available, if this block is within a loop, we should avoid using vperm
10702 // for 3-operation perms and use a constant pool load instead.
10703 if (Cost < 3)
10704 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
10705 }
10706 }
10707
10708 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant
10709 // vector that will get spilled to the constant pool.
10710 if (V2.isUndef()) V2 = V1;
10711
10712 return LowerVPERM(Op, DAG, PermMask, VT, V1, V2);
10713}
10714
10715SDValue PPCTargetLowering::LowerVPERM(SDValue Op, SelectionDAG &DAG,
10716 ArrayRef<int> PermMask, EVT VT,
10717 SDValue V1, SDValue V2) const {
10718 unsigned Opcode = PPCISD::VPERM;
10719 EVT ValType = V1.getValueType();
10720 SDLoc dl(Op);
10721 bool NeedSwap = false;
10722 bool isLittleEndian = Subtarget.isLittleEndian();
10723 bool isPPC64 = Subtarget.isPPC64();
10724
10725 if (Subtarget.hasVSX() && Subtarget.hasP9Vector() &&
10726 (V1->hasOneUse() || V2->hasOneUse())) {
10727 LLVM_DEBUG(dbgs() << "At least one of two input vectors are dead - using "
10728 "XXPERM instead\n");
10729 Opcode = PPCISD::XXPERM;
10730
10731 // The second input to XXPERM is also an output so if the second input has
10732 // multiple uses then copying is necessary, as a result we want the
10733 // single-use operand to be used as the second input to prevent copying.
10734 if ((!isLittleEndian && !V2->hasOneUse() && V1->hasOneUse()) ||
10735 (isLittleEndian && !V1->hasOneUse() && V2->hasOneUse())) {
10736 std::swap(a&: V1, b&: V2);
10737 NeedSwap = !NeedSwap;
10738 }
10739 }
10740
10741 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except
10742 // that it is in input element units, not in bytes. Convert now.
10743
10744 // For little endian, the order of the input vectors is reversed, and
10745 // the permutation mask is complemented with respect to 31. This is
10746 // necessary to produce proper semantics with the big-endian-based vperm
10747 // instruction.
10748 EVT EltVT = V1.getValueType().getVectorElementType();
10749 unsigned BytesPerElement = EltVT.getSizeInBits() / 8;
10750
10751 bool V1HasXXSWAPD = V1->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10752 bool V2HasXXSWAPD = V2->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10753
10754 /*
10755 Vectors will be appended like so: [ V1 | v2 ]
10756 XXSWAPD on V1:
10757 [ A | B | C | D ] -> [ C | D | A | B ]
10758 0-3 4-7 8-11 12-15 0-3 4-7 8-11 12-15
10759 i.e. index of A, B += 8, and index of C, D -= 8.
10760 XXSWAPD on V2:
10761 [ E | F | G | H ] -> [ G | H | E | F ]
10762 16-19 20-23 24-27 28-31 16-19 20-23 24-27 28-31
10763 i.e. index of E, F += 8, index of G, H -= 8
10764 Swap V1 and V2:
10765 [ V1 | V2 ] -> [ V2 | V1 ]
10766 0-15 16-31 0-15 16-31
10767 i.e. index of V1 += 16, index of V2 -= 16
10768 */
10769
10770 SmallVector<SDValue, 16> ResultMask;
10771 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
10772 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i];
10773
10774 if (V1HasXXSWAPD) {
10775 if (SrcElt < 8)
10776 SrcElt += 8;
10777 else if (SrcElt < 16)
10778 SrcElt -= 8;
10779 }
10780 if (V2HasXXSWAPD) {
10781 if (SrcElt > 23)
10782 SrcElt -= 8;
10783 else if (SrcElt > 15)
10784 SrcElt += 8;
10785 }
10786 if (NeedSwap) {
10787 if (SrcElt < 16)
10788 SrcElt += 16;
10789 else
10790 SrcElt -= 16;
10791 }
10792 for (unsigned j = 0; j != BytesPerElement; ++j)
10793 if (isLittleEndian)
10794 ResultMask.push_back(
10795 Elt: DAG.getConstant(Val: 31 - (SrcElt * BytesPerElement + j), DL: dl, VT: MVT::i32));
10796 else
10797 ResultMask.push_back(
10798 Elt: DAG.getConstant(Val: SrcElt * BytesPerElement + j, DL: dl, VT: MVT::i32));
10799 }
10800
10801 if (V1HasXXSWAPD) {
10802 dl = SDLoc(V1->getOperand(Num: 0));
10803 V1 = V1->getOperand(Num: 0)->getOperand(Num: 1);
10804 }
10805 if (V2HasXXSWAPD) {
10806 dl = SDLoc(V2->getOperand(Num: 0));
10807 V2 = V2->getOperand(Num: 0)->getOperand(Num: 1);
10808 }
10809
10810 if (isPPC64 && (V1HasXXSWAPD || V2HasXXSWAPD)) {
10811 if (ValType != MVT::v2f64)
10812 V1 = DAG.getBitcast(VT: MVT::v2f64, V: V1);
10813 if (V2.getValueType() != MVT::v2f64)
10814 V2 = DAG.getBitcast(VT: MVT::v2f64, V: V2);
10815 }
10816
10817 ShufflesHandledWithVPERM++;
10818 SDValue VPermMask = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: ResultMask);
10819 LLVM_DEBUG({
10820 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10821 if (Opcode == PPCISD::XXPERM) {
10822 dbgs() << "Emitting a XXPERM for the following shuffle:\n";
10823 } else {
10824 dbgs() << "Emitting a VPERM for the following shuffle:\n";
10825 }
10826 SVOp->dump();
10827 dbgs() << "With the following permute control vector:\n";
10828 VPermMask.dump();
10829 });
10830
10831 if (Opcode == PPCISD::XXPERM)
10832 VPermMask = DAG.getBitcast(VT: MVT::v4i32, V: VPermMask);
10833
10834 // Only need to place items backwards in LE,
10835 // the mask was properly calculated.
10836 if (isLittleEndian)
10837 std::swap(a&: V1, b&: V2);
10838
10839 SDValue VPERMNode =
10840 DAG.getNode(Opcode, DL: dl, VT: V1.getValueType(), N1: V1, N2: V2, N3: VPermMask);
10841
10842 VPERMNode = DAG.getBitcast(VT: ValType, V: VPERMNode);
10843 return VPERMNode;
10844}
10845
10846/// getVectorCompareInfo - Given an intrinsic, return false if it is not a
10847/// vector comparison. If it is, return true and fill in Opc/isDot with
10848/// information about the intrinsic.
10849static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc,
10850 bool &isDot, const PPCSubtarget &Subtarget) {
10851 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 0);
10852 CompareOpc = -1;
10853 isDot = false;
10854 switch (IntrinsicID) {
10855 default:
10856 return false;
10857 // Comparison predicates.
10858 case Intrinsic::ppc_altivec_vcmpbfp_p:
10859 CompareOpc = 966;
10860 isDot = true;
10861 break;
10862 case Intrinsic::ppc_altivec_vcmpeqfp_p:
10863 CompareOpc = 198;
10864 isDot = true;
10865 break;
10866 case Intrinsic::ppc_altivec_vcmpequb_p:
10867 CompareOpc = 6;
10868 isDot = true;
10869 break;
10870 case Intrinsic::ppc_altivec_vcmpequh_p:
10871 CompareOpc = 70;
10872 isDot = true;
10873 break;
10874 case Intrinsic::ppc_altivec_vcmpequw_p:
10875 CompareOpc = 134;
10876 isDot = true;
10877 break;
10878 case Intrinsic::ppc_altivec_vcmpequd_p:
10879 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10880 CompareOpc = 199;
10881 isDot = true;
10882 } else
10883 return false;
10884 break;
10885 case Intrinsic::ppc_altivec_vcmpneb_p:
10886 case Intrinsic::ppc_altivec_vcmpneh_p:
10887 case Intrinsic::ppc_altivec_vcmpnew_p:
10888 case Intrinsic::ppc_altivec_vcmpnezb_p:
10889 case Intrinsic::ppc_altivec_vcmpnezh_p:
10890 case Intrinsic::ppc_altivec_vcmpnezw_p:
10891 if (Subtarget.hasP9Altivec()) {
10892 switch (IntrinsicID) {
10893 default:
10894 llvm_unreachable("Unknown comparison intrinsic.");
10895 case Intrinsic::ppc_altivec_vcmpneb_p:
10896 CompareOpc = 7;
10897 break;
10898 case Intrinsic::ppc_altivec_vcmpneh_p:
10899 CompareOpc = 71;
10900 break;
10901 case Intrinsic::ppc_altivec_vcmpnew_p:
10902 CompareOpc = 135;
10903 break;
10904 case Intrinsic::ppc_altivec_vcmpnezb_p:
10905 CompareOpc = 263;
10906 break;
10907 case Intrinsic::ppc_altivec_vcmpnezh_p:
10908 CompareOpc = 327;
10909 break;
10910 case Intrinsic::ppc_altivec_vcmpnezw_p:
10911 CompareOpc = 391;
10912 break;
10913 }
10914 isDot = true;
10915 } else
10916 return false;
10917 break;
10918 case Intrinsic::ppc_altivec_vcmpgefp_p:
10919 CompareOpc = 454;
10920 isDot = true;
10921 break;
10922 case Intrinsic::ppc_altivec_vcmpgtfp_p:
10923 CompareOpc = 710;
10924 isDot = true;
10925 break;
10926 case Intrinsic::ppc_altivec_vcmpgtsb_p:
10927 CompareOpc = 774;
10928 isDot = true;
10929 break;
10930 case Intrinsic::ppc_altivec_vcmpgtsh_p:
10931 CompareOpc = 838;
10932 isDot = true;
10933 break;
10934 case Intrinsic::ppc_altivec_vcmpgtsw_p:
10935 CompareOpc = 902;
10936 isDot = true;
10937 break;
10938 case Intrinsic::ppc_altivec_vcmpgtsd_p:
10939 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10940 CompareOpc = 967;
10941 isDot = true;
10942 } else
10943 return false;
10944 break;
10945 case Intrinsic::ppc_altivec_vcmpgtub_p:
10946 CompareOpc = 518;
10947 isDot = true;
10948 break;
10949 case Intrinsic::ppc_altivec_vcmpgtuh_p:
10950 CompareOpc = 582;
10951 isDot = true;
10952 break;
10953 case Intrinsic::ppc_altivec_vcmpgtuw_p:
10954 CompareOpc = 646;
10955 isDot = true;
10956 break;
10957 case Intrinsic::ppc_altivec_vcmpgtud_p:
10958 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10959 CompareOpc = 711;
10960 isDot = true;
10961 } else
10962 return false;
10963 break;
10964
10965 case Intrinsic::ppc_altivec_vcmpequq:
10966 case Intrinsic::ppc_altivec_vcmpgtsq:
10967 case Intrinsic::ppc_altivec_vcmpgtuq:
10968 if (!Subtarget.isISA3_1())
10969 return false;
10970 switch (IntrinsicID) {
10971 default:
10972 llvm_unreachable("Unknown comparison intrinsic.");
10973 case Intrinsic::ppc_altivec_vcmpequq:
10974 CompareOpc = 455;
10975 break;
10976 case Intrinsic::ppc_altivec_vcmpgtsq:
10977 CompareOpc = 903;
10978 break;
10979 case Intrinsic::ppc_altivec_vcmpgtuq:
10980 CompareOpc = 647;
10981 break;
10982 }
10983 break;
10984
10985 // VSX predicate comparisons use the same infrastructure
10986 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10987 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10988 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
10989 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
10990 case Intrinsic::ppc_vsx_xvcmpgesp_p:
10991 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
10992 if (Subtarget.hasVSX()) {
10993 switch (IntrinsicID) {
10994 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10995 CompareOpc = 99;
10996 break;
10997 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10998 CompareOpc = 115;
10999 break;
11000 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
11001 CompareOpc = 107;
11002 break;
11003 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
11004 CompareOpc = 67;
11005 break;
11006 case Intrinsic::ppc_vsx_xvcmpgesp_p:
11007 CompareOpc = 83;
11008 break;
11009 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
11010 CompareOpc = 75;
11011 break;
11012 }
11013 isDot = true;
11014 } else
11015 return false;
11016 break;
11017
11018 // Normal Comparisons.
11019 case Intrinsic::ppc_altivec_vcmpbfp:
11020 CompareOpc = 966;
11021 break;
11022 case Intrinsic::ppc_altivec_vcmpeqfp:
11023 CompareOpc = 198;
11024 break;
11025 case Intrinsic::ppc_altivec_vcmpequb:
11026 CompareOpc = 6;
11027 break;
11028 case Intrinsic::ppc_altivec_vcmpequh:
11029 CompareOpc = 70;
11030 break;
11031 case Intrinsic::ppc_altivec_vcmpequw:
11032 CompareOpc = 134;
11033 break;
11034 case Intrinsic::ppc_altivec_vcmpequd:
11035 if (Subtarget.hasP8Altivec())
11036 CompareOpc = 199;
11037 else
11038 return false;
11039 break;
11040 case Intrinsic::ppc_altivec_vcmpneb:
11041 case Intrinsic::ppc_altivec_vcmpneh:
11042 case Intrinsic::ppc_altivec_vcmpnew:
11043 case Intrinsic::ppc_altivec_vcmpnezb:
11044 case Intrinsic::ppc_altivec_vcmpnezh:
11045 case Intrinsic::ppc_altivec_vcmpnezw:
11046 if (Subtarget.hasP9Altivec())
11047 switch (IntrinsicID) {
11048 default:
11049 llvm_unreachable("Unknown comparison intrinsic.");
11050 case Intrinsic::ppc_altivec_vcmpneb:
11051 CompareOpc = 7;
11052 break;
11053 case Intrinsic::ppc_altivec_vcmpneh:
11054 CompareOpc = 71;
11055 break;
11056 case Intrinsic::ppc_altivec_vcmpnew:
11057 CompareOpc = 135;
11058 break;
11059 case Intrinsic::ppc_altivec_vcmpnezb:
11060 CompareOpc = 263;
11061 break;
11062 case Intrinsic::ppc_altivec_vcmpnezh:
11063 CompareOpc = 327;
11064 break;
11065 case Intrinsic::ppc_altivec_vcmpnezw:
11066 CompareOpc = 391;
11067 break;
11068 }
11069 else
11070 return false;
11071 break;
11072 case Intrinsic::ppc_altivec_vcmpgefp:
11073 CompareOpc = 454;
11074 break;
11075 case Intrinsic::ppc_altivec_vcmpgtfp:
11076 CompareOpc = 710;
11077 break;
11078 case Intrinsic::ppc_altivec_vcmpgtsb:
11079 CompareOpc = 774;
11080 break;
11081 case Intrinsic::ppc_altivec_vcmpgtsh:
11082 CompareOpc = 838;
11083 break;
11084 case Intrinsic::ppc_altivec_vcmpgtsw:
11085 CompareOpc = 902;
11086 break;
11087 case Intrinsic::ppc_altivec_vcmpgtsd:
11088 if (Subtarget.hasP8Altivec())
11089 CompareOpc = 967;
11090 else
11091 return false;
11092 break;
11093 case Intrinsic::ppc_altivec_vcmpgtub:
11094 CompareOpc = 518;
11095 break;
11096 case Intrinsic::ppc_altivec_vcmpgtuh:
11097 CompareOpc = 582;
11098 break;
11099 case Intrinsic::ppc_altivec_vcmpgtuw:
11100 CompareOpc = 646;
11101 break;
11102 case Intrinsic::ppc_altivec_vcmpgtud:
11103 if (Subtarget.hasP8Altivec())
11104 CompareOpc = 711;
11105 else
11106 return false;
11107 break;
11108 case Intrinsic::ppc_altivec_vcmpequq_p:
11109 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11110 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11111 if (!Subtarget.isISA3_1())
11112 return false;
11113 switch (IntrinsicID) {
11114 default:
11115 llvm_unreachable("Unknown comparison intrinsic.");
11116 case Intrinsic::ppc_altivec_vcmpequq_p:
11117 CompareOpc = 455;
11118 break;
11119 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11120 CompareOpc = 903;
11121 break;
11122 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11123 CompareOpc = 647;
11124 break;
11125 }
11126 isDot = true;
11127 break;
11128 }
11129 return true;
11130}
11131
11132/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom
11133/// lower, do it, otherwise return null.
11134SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11135 SelectionDAG &DAG) const {
11136 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11137
11138 SDLoc dl(Op);
11139 // Note: BCD instructions expect the immediate operand in vector form (v4i32),
11140 // but the builtin provides it as a scalar. To satisfy the instruction
11141 // encoding, we splat the scalar across all lanes using SPLAT_VECTOR.
11142 auto MapNodeWithSplatVector =
11143 [&](unsigned Opcode,
11144 std::initializer_list<SDValue> ExtraOps = {}) -> SDValue {
11145 SDValue SplatVal =
11146 DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: MVT::v4i32, Operand: Op.getOperand(i: 2));
11147
11148 SmallVector<SDValue, 4> Ops{SplatVal, Op.getOperand(i: 1)};
11149 Ops.append(in_start: ExtraOps.begin(), in_end: ExtraOps.end());
11150 return DAG.getNode(Opcode, DL: dl, VT: MVT::v16i8, Ops);
11151 };
11152
11153 switch (IntrinsicID) {
11154 case Intrinsic::thread_pointer:
11155 // Reads the thread pointer register, used for __builtin_thread_pointer.
11156 if (Subtarget.isPPC64())
11157 return DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
11158 return DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
11159
11160 case Intrinsic::ppc_rldimi: {
11161 assert(Subtarget.isPPC64() && "rldimi is only available in 64-bit!");
11162 SDValue Src = Op.getOperand(i: 1);
11163 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11164 if (Mask.isZero())
11165 return Op.getOperand(i: 2);
11166 if (Mask.isAllOnes())
11167 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src, N2: Op.getOperand(i: 3));
11168 uint64_t SH = Op.getConstantOperandVal(i: 3);
11169 unsigned MB = 0, ME = 0;
11170 if (!isRunOfOnes64(Val: Mask.getZExtValue(), MB, ME))
11171 report_fatal_error(reason: "invalid rldimi mask!");
11172 // rldimi requires ME=63-SH, otherwise rotation is needed before rldimi.
11173 if (ME < 63 - SH) {
11174 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11175 N2: DAG.getConstant(Val: ME + SH + 1, DL: dl, VT: MVT::i32));
11176 } else if (ME > 63 - SH) {
11177 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11178 N2: DAG.getConstant(Val: ME + SH - 63, DL: dl, VT: MVT::i32));
11179 }
11180 return SDValue(
11181 DAG.getMachineNode(Opcode: PPC::RLDIMI, dl, VT: MVT::i64,
11182 Ops: {Op.getOperand(i: 2), Src,
11183 DAG.getTargetConstant(Val: 63 - ME, DL: dl, VT: MVT::i32),
11184 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32)}),
11185 0);
11186 }
11187
11188 case Intrinsic::ppc_rlwimi: {
11189 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11190 if (Mask.isZero())
11191 return Op.getOperand(i: 2);
11192 if (Mask.isAllOnes())
11193 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
11194 N2: Op.getOperand(i: 3));
11195 unsigned MB = 0, ME = 0;
11196 if (!isRunOfOnes(Val: Mask.getZExtValue(), MB, ME))
11197 report_fatal_error(reason: "invalid rlwimi mask!");
11198 return SDValue(DAG.getMachineNode(
11199 Opcode: PPC::RLWIMI, dl, VT: MVT::i32,
11200 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1), Op.getOperand(i: 3),
11201 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11202 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11203 0);
11204 }
11205
11206 case Intrinsic::ppc_bcdshift:
11207 return MapNodeWithSplatVector(PPCISD::BCDSHIFT, {Op.getOperand(i: 3)});
11208 case Intrinsic::ppc_bcdshiftround:
11209 return MapNodeWithSplatVector(PPCISD::BCDSHIFTROUND, {Op.getOperand(i: 3)});
11210 case Intrinsic::ppc_bcdtruncate:
11211 return MapNodeWithSplatVector(PPCISD::BCDTRUNC, {Op.getOperand(i: 3)});
11212 case Intrinsic::ppc_bcdunsignedtruncate:
11213 return MapNodeWithSplatVector(PPCISD::BCDUTRUNC);
11214 case Intrinsic::ppc_bcdunsignedshift:
11215 return MapNodeWithSplatVector(PPCISD::BCDUSHIFT);
11216
11217 case Intrinsic::ppc_rlwnm: {
11218 if (Op.getConstantOperandVal(i: 3) == 0)
11219 return DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
11220 unsigned MB = 0, ME = 0;
11221 if (!isRunOfOnes(Val: Op.getConstantOperandVal(i: 3), MB, ME))
11222 report_fatal_error(reason: "invalid rlwnm mask!");
11223 return SDValue(
11224 DAG.getMachineNode(Opcode: PPC::RLWNM, dl, VT: MVT::i32,
11225 Ops: {Op.getOperand(i: 1), Op.getOperand(i: 2),
11226 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11227 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11228 0);
11229 }
11230
11231 case Intrinsic::ppc_mma_disassemble_acc: {
11232 if (Subtarget.isISAFuture()) {
11233 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11234 SDValue WideVec =
11235 SDValue(DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes,
11236 Ops: Op.getOperand(i: 1)),
11237 0);
11238 SmallVector<SDValue, 4> RetOps;
11239 SDValue Value = SDValue(WideVec.getNode(), 0);
11240 SDValue Value2 = SDValue(WideVec.getNode(), 1);
11241
11242 SDValue Extract;
11243 Extract = DAG.getNode(
11244 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11245 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11246 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11247 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11248 RetOps.push_back(Elt: Extract);
11249 Extract = DAG.getNode(
11250 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11251 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11252 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11253 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11254 RetOps.push_back(Elt: Extract);
11255 Extract = DAG.getNode(
11256 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11257 N1: Subtarget.isLittleEndian() ? Value : Value2,
11258 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11259 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11260 RetOps.push_back(Elt: Extract);
11261 Extract = DAG.getNode(
11262 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11263 N1: Subtarget.isLittleEndian() ? Value : Value2,
11264 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11265 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11266 RetOps.push_back(Elt: Extract);
11267 return DAG.getMergeValues(Ops: RetOps, dl);
11268 }
11269 [[fallthrough]];
11270 }
11271 case Intrinsic::ppc_vsx_disassemble_pair: {
11272 int NumVecs = 2;
11273 SDValue WideVec = Op.getOperand(i: 1);
11274 if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) {
11275 NumVecs = 4;
11276 WideVec = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: WideVec);
11277 }
11278 SmallVector<SDValue, 4> RetOps;
11279 for (int VecNo = 0; VecNo < NumVecs; VecNo++) {
11280 SDValue Extract = DAG.getNode(
11281 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: WideVec,
11282 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo
11283 : VecNo,
11284 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11285 RetOps.push_back(Elt: Extract);
11286 }
11287 return DAG.getMergeValues(Ops: RetOps, dl);
11288 }
11289
11290 case Intrinsic::ppc_build_dmr: {
11291 SmallVector<SDValue, 8> Pairs;
11292 SmallVector<SDValue, 8> Chains;
11293 for (int i = 1; i < 9; i += 2) {
11294 SDValue Hi = Op.getOperand(i);
11295 SDValue Lo = Op.getOperand(i: i + 1);
11296 if (Hi->getOpcode() == ISD::LOAD)
11297 Chains.push_back(Elt: Hi.getValue(R: 1));
11298 if (Lo->getOpcode() == ISD::LOAD)
11299 Chains.push_back(Elt: Lo.getValue(R: 1));
11300 Pairs.push_back(
11301 Elt: DAG.getNode(Opcode: PPCISD::PAIR_BUILD, DL: dl, VT: MVT::v256i1, Ops: {Hi, Lo}));
11302 }
11303 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
11304 SDValue Value = DMFInsert1024(Pairs, dl: SDLoc(Op), DAG);
11305 return DAG.getMergeValues(Ops: {Value, TF}, dl);
11306 }
11307
11308 case Intrinsic::ppc_mma_dmxxextfdmr512: {
11309 assert(Subtarget.isISAFuture() && "dmxxextfdmr512 requires ISA Future");
11310 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11311 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11312 "Specify P of 0 or 1 for lower or upper 512 bytes");
11313 unsigned HiLo = Idx->getSExtValue();
11314 unsigned Opcode;
11315 unsigned Subx;
11316 if (HiLo == 0) {
11317 Opcode = PPC::DMXXEXTFDMR512;
11318 Subx = PPC::sub_wacc_lo;
11319 } else {
11320 Opcode = PPC::DMXXEXTFDMR512_HI;
11321 Subx = PPC::sub_wacc_hi;
11322 }
11323 SDValue Subreg(
11324 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
11325 Op1: Op.getOperand(i: 1),
11326 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11327 0);
11328 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11329 return SDValue(DAG.getMachineNode(Opcode, dl, ResultTys: ReturnTypes, Ops: Subreg), 0);
11330 }
11331
11332 case Intrinsic::ppc_mma_dmxxextfdmr256: {
11333 assert(Subtarget.isISAFuture() && "dmxxextfdmr256 requires ISA Future");
11334 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11335 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11336 "Specify a dmr row pair 0-3");
11337 unsigned IdxVal = Idx->getSExtValue();
11338 unsigned Subx;
11339 switch (IdxVal) {
11340 case 0:
11341 Subx = PPC::sub_dmrrowp0;
11342 break;
11343 case 1:
11344 Subx = PPC::sub_dmrrowp1;
11345 break;
11346 case 2:
11347 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11348 break;
11349 case 3:
11350 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11351 break;
11352 }
11353 SDValue Subreg(
11354 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v256i1,
11355 Op1: Op.getOperand(i: 1),
11356 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11357 0);
11358 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11359 return SDValue(
11360 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR256, dl, VT: MVT::v256i1, Ops: {Subreg, P}),
11361 0);
11362 }
11363
11364 case Intrinsic::ppc_mma_dmxxinstdmr512: {
11365 assert(Subtarget.isISAFuture() && "dmxxinstdmr512 requires ISA Future");
11366 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 4));
11367 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11368 "Specify P of 0 or 1 for lower or upper 512 bytes");
11369 unsigned HiLo = Idx->getSExtValue();
11370 unsigned Opcode;
11371 unsigned Subx;
11372 if (HiLo == 0) {
11373 Opcode = PPCISD::INST512;
11374 Subx = PPC::sub_wacc_lo;
11375 } else {
11376 Opcode = PPCISD::INST512HI;
11377 Subx = PPC::sub_wacc_hi;
11378 }
11379 SDValue Wacc = DAG.getNode(Opcode, DL: dl, VT: MVT::v512i1, N1: Op.getOperand(i: 2),
11380 N2: Op.getOperand(i: 3));
11381 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11382 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11383 Op1: Op.getOperand(i: 1), Op2: Wacc, Op3: SubReg),
11384 0);
11385 }
11386
11387 case Intrinsic::ppc_mma_dmxxinstdmr256: {
11388 assert(Subtarget.isISAFuture() && "dmxxinstdmr256 requires ISA Future");
11389 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11390 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11391 "Specify a dmr row pair 0-3");
11392 unsigned IdxVal = Idx->getSExtValue();
11393 unsigned Subx;
11394 switch (IdxVal) {
11395 case 0:
11396 Subx = PPC::sub_dmrrowp0;
11397 break;
11398 case 1:
11399 Subx = PPC::sub_dmrrowp1;
11400 break;
11401 case 2:
11402 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11403 break;
11404 case 3:
11405 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11406 break;
11407 }
11408 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11409 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11410 SDValue DMRRowp =
11411 DAG.getNode(Opcode: PPCISD::INST256, DL: dl, VT: MVT::v256i1, N1: Op.getOperand(i: 2), N2: P);
11412 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11413 Op1: Op.getOperand(i: 1), Op2: DMRRowp, Op3: SubReg),
11414 0);
11415 }
11416
11417 case Intrinsic::ppc_mma_xxmfacc:
11418 case Intrinsic::ppc_mma_xxmtacc: {
11419 // Allow pre-isa-future subtargets to lower as normal.
11420 if (!Subtarget.isISAFuture())
11421 return SDValue();
11422 // The intrinsics for xxmtacc and xxmfacc take one argument of
11423 // type v512i1, for future cpu the corresponding wacc instruction
11424 // dmxx[inst|extf]dmr512 is always generated for type v512i1, negating
11425 // the need to produce the xxm[t|f]acc.
11426 SDValue WideVec = Op.getOperand(i: 1);
11427 DAG.ReplaceAllUsesWith(From: Op, To: WideVec);
11428 return SDValue();
11429 }
11430
11431 case Intrinsic::ppc_unpack_longdouble: {
11432 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11433 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11434 "Argument of long double unpack must be 0 or 1!");
11435 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::f64, N1: Op.getOperand(i: 1),
11436 N2: DAG.getConstant(Val: !!(Idx->getSExtValue()), DL: dl,
11437 VT: Idx->getValueType(ResNo: 0)));
11438 }
11439
11440 case Intrinsic::ppc_compare_exp_lt:
11441 case Intrinsic::ppc_compare_exp_gt:
11442 case Intrinsic::ppc_compare_exp_eq:
11443 case Intrinsic::ppc_compare_exp_uo: {
11444 unsigned Pred;
11445 switch (IntrinsicID) {
11446 case Intrinsic::ppc_compare_exp_lt:
11447 Pred = PPC::PRED_LT;
11448 break;
11449 case Intrinsic::ppc_compare_exp_gt:
11450 Pred = PPC::PRED_GT;
11451 break;
11452 case Intrinsic::ppc_compare_exp_eq:
11453 Pred = PPC::PRED_EQ;
11454 break;
11455 case Intrinsic::ppc_compare_exp_uo:
11456 Pred = PPC::PRED_UN;
11457 break;
11458 }
11459 return SDValue(
11460 DAG.getMachineNode(
11461 Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11462 Ops: {SDValue(DAG.getMachineNode(Opcode: PPC::XSCMPEXPDP, dl, VT: MVT::i32,
11463 Op1: Op.getOperand(i: 1), Op2: Op.getOperand(i: 2)),
11464 0),
11465 DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11466 DAG.getTargetConstant(Val: Pred, DL: dl, VT: MVT::i32)}),
11467 0);
11468 }
11469 case Intrinsic::ppc_test_data_class: {
11470 EVT OpVT = Op.getOperand(i: 1).getValueType();
11471 unsigned CmprOpc = OpVT == MVT::f128 ? PPC::XSTSTDCQP
11472 : (OpVT == MVT::f64 ? PPC::XSTSTDCDP
11473 : PPC::XSTSTDCSP);
11474 // Lower __builtin_ppc_test_data_class(value, mask) to XSTSTDC* instruction.
11475 // The XSTSTDC* instructions test if a floating-point value matches any of
11476 // the data classes specified in the mask, setting CR field bits
11477 // accordingly. We need to extract the EQ bit (bit 2) from the CR field and
11478 // convert it to an integer result (1 if match, 0 if no match).
11479 //
11480 // Note: Operands are swapped because XSTSTDC* expects (mask, value) but the
11481 // intrinsic provides (value, mask) as Op.getOperand(1) and
11482 // Op.getOperand(2).
11483 SDValue TestDataClass =
11484 SDValue(DAG.getMachineNode(Opcode: CmprOpc, dl, VT: MVT::i32,
11485 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1)}),
11486 0);
11487 if (Subtarget.isISA3_1()) {
11488 // ISA 3.1+: Use SETBC instruction to directly convert CR bit to integer.
11489 // This is more efficient than the SELECT_CC approach used in earlier
11490 // ISAs.
11491 SDValue SubRegIdx = DAG.getTargetConstant(Val: PPC::sub_eq, DL: dl, VT: MVT::i32);
11492 SDValue CRBit =
11493 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11494 Op1: TestDataClass, Op2: SubRegIdx),
11495 0);
11496
11497 return DAG.getNode(Opcode: PPCISD::SETBC, DL: dl, VT: MVT::i32, Operand: CRBit);
11498 }
11499
11500 // Pre-ISA 3.1: Use SELECT_CC to convert CR field to integer (1 or 0).
11501 return SDValue(
11502 DAG.getMachineNode(Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11503 Ops: {TestDataClass, DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32),
11504 DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11505 DAG.getTargetConstant(Val: PPC::PRED_EQ, DL: dl, VT: MVT::i32)}),
11506 0);
11507 }
11508 case Intrinsic::ppc_fnmsub: {
11509 EVT VT = Op.getOperand(i: 1).getValueType();
11510 if (!Subtarget.hasVSX() || (!Subtarget.hasFloat128() && VT == MVT::f128))
11511 return DAG.getNode(
11512 Opcode: ISD::FNEG, DL: dl, VT,
11513 Operand: DAG.getNode(Opcode: ISD::FMA, DL: dl, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11514 N3: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op.getOperand(i: 3))));
11515 return DAG.getNode(Opcode: PPCISD::FNMSUB, DL: dl, VT, N1: Op.getOperand(i: 1),
11516 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11517 }
11518 case Intrinsic::ppc_convert_f128_to_ppcf128:
11519 case Intrinsic::ppc_convert_ppcf128_to_f128: {
11520 RTLIB::Libcall LC = IntrinsicID == Intrinsic::ppc_convert_ppcf128_to_f128
11521 ? RTLIB::CONVERT_PPCF128_F128
11522 : RTLIB::CONVERT_F128_PPCF128;
11523 MakeLibCallOptions CallOptions;
11524 std::pair<SDValue, SDValue> Result =
11525 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op.getOperand(i: 1), CallOptions,
11526 dl, Chain: SDValue());
11527 return Result.first;
11528 }
11529 case Intrinsic::ppc_maxfe:
11530 case Intrinsic::ppc_maxfl:
11531 case Intrinsic::ppc_maxfs:
11532 case Intrinsic::ppc_minfe:
11533 case Intrinsic::ppc_minfl:
11534 case Intrinsic::ppc_minfs: {
11535 EVT VT = Op.getValueType();
11536 assert(
11537 all_of(Op->ops().drop_front(4),
11538 [VT](const SDUse &Use) { return Use.getValueType() == VT; }) &&
11539 "ppc_[max|min]f[e|l|s] must have uniform type arguments");
11540 (void)VT;
11541 ISD::CondCode CC = ISD::SETGT;
11542 if (IntrinsicID == Intrinsic::ppc_minfe ||
11543 IntrinsicID == Intrinsic::ppc_minfl ||
11544 IntrinsicID == Intrinsic::ppc_minfs)
11545 CC = ISD::SETLT;
11546 unsigned I = Op.getNumOperands() - 2, Cnt = I;
11547 SDValue Res = Op.getOperand(i: I);
11548 for (--I; Cnt != 0; --Cnt, I = (--I == 0 ? (Op.getNumOperands() - 1) : I)) {
11549 Res =
11550 DAG.getSelectCC(DL: dl, LHS: Res, RHS: Op.getOperand(i: I), True: Res, False: Op.getOperand(i: I), Cond: CC);
11551 }
11552 return Res;
11553 }
11554 }
11555
11556 // If this is a lowered altivec predicate compare, CompareOpc is set to the
11557 // opcode number of the comparison.
11558 int CompareOpc;
11559 bool isDot;
11560 if (!getVectorCompareInfo(Intrin: Op, CompareOpc, isDot, Subtarget))
11561 return SDValue(); // Don't custom lower most intrinsics.
11562
11563 // If this is a non-dot comparison, make the VCMP node and we are done.
11564 if (!isDot) {
11565 SDValue Tmp = DAG.getNode(Opcode: PPCISD::VCMP, DL: dl, VT: Op.getOperand(i: 2).getValueType(),
11566 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11567 N3: DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32));
11568 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Tmp);
11569 }
11570
11571 // Create the PPCISD altivec 'dot' comparison node.
11572 SDValue Ops[] = {
11573 Op.getOperand(i: 2), // LHS
11574 Op.getOperand(i: 3), // RHS
11575 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
11576 };
11577 EVT VTs[] = { Op.getOperand(i: 2).getValueType(), MVT::Glue };
11578 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
11579
11580 // Unpack the result based on how the target uses it.
11581 unsigned BitNo; // Bit # of CR6.
11582 bool InvertBit; // Invert result?
11583 unsigned Bitx;
11584 unsigned SetOp;
11585 switch (Op.getConstantOperandVal(i: 1)) {
11586 default: // Can't happen, don't crash on invalid number though.
11587 case 0: // Return the value of the EQ bit of CR6.
11588 BitNo = 0;
11589 InvertBit = false;
11590 Bitx = PPC::sub_eq;
11591 SetOp = PPCISD::SETBC;
11592 break;
11593 case 1: // Return the inverted value of the EQ bit of CR6.
11594 BitNo = 0;
11595 InvertBit = true;
11596 Bitx = PPC::sub_eq;
11597 SetOp = PPCISD::SETBCR;
11598 break;
11599 case 2: // Return the value of the LT bit of CR6.
11600 BitNo = 2;
11601 InvertBit = false;
11602 Bitx = PPC::sub_lt;
11603 SetOp = PPCISD::SETBC;
11604 break;
11605 case 3: // Return the inverted value of the LT bit of CR6.
11606 BitNo = 2;
11607 InvertBit = true;
11608 Bitx = PPC::sub_lt;
11609 SetOp = PPCISD::SETBCR;
11610 break;
11611 }
11612
11613 SDValue GlueOp = CompNode.getValue(R: 1);
11614 if (Subtarget.isISA3_1()) {
11615 SDValue SubRegIdx = DAG.getTargetConstant(Val: Bitx, DL: dl, VT: MVT::i32);
11616 SDValue CR6Reg = DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32);
11617 SDValue CRBit =
11618 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11619 Op1: CR6Reg, Op2: SubRegIdx, Op3: GlueOp),
11620 0);
11621 return DAG.getNode(Opcode: SetOp, DL: dl, VT: MVT::i32, Operand: CRBit);
11622 }
11623
11624 // Now that we have the comparison, emit a copy from the CR to a GPR.
11625 // This is flagged to the above dot comparison.
11626 SDValue Flags = DAG.getNode(Opcode: PPCISD::MFOCRF, DL: dl, VT: MVT::i32,
11627 N1: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32), N2: GlueOp);
11628
11629 // Shift the bit into the low position.
11630 Flags = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: Flags,
11631 N2: DAG.getConstant(Val: 8 - (3 - BitNo), DL: dl, VT: MVT::i32));
11632 // Isolate the bit.
11633 Flags = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Flags,
11634 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11635
11636 // If we are supposed to, toggle the bit.
11637 if (InvertBit)
11638 Flags = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Flags,
11639 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11640 return Flags;
11641}
11642
11643SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
11644 SelectionDAG &DAG) const {
11645 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to
11646 // the beginning of the argument list.
11647 int ArgStart = isa<ConstantSDNode>(Val: Op.getOperand(i: 0)) ? 0 : 1;
11648 SDLoc DL(Op);
11649 switch (Op.getConstantOperandVal(i: ArgStart)) {
11650 case Intrinsic::ppc_cfence: {
11651 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument.");
11652 SDValue Val = Op.getOperand(i: ArgStart + 1);
11653 EVT Ty = Val.getValueType();
11654 if (Ty == MVT::i128) {
11655 // FIXME: Testing one of two paired registers is sufficient to guarantee
11656 // ordering?
11657 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: Val);
11658 }
11659 unsigned Opcode = Subtarget.isPPC64() ? PPC::CFENCE8 : PPC::CFENCE;
11660 return SDValue(
11661 DAG.getMachineNode(
11662 Opcode, dl: DL, VT: MVT::Other,
11663 Op1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getScalarIntVT(), Operand: Val),
11664 Op2: Op.getOperand(i: 0)),
11665 0);
11666 }
11667 case Intrinsic::ppc_disassemble_dmr: {
11668 assert(ArgStart == 1 &&
11669 "llvm.ppc.disassemble.dmr must carry a chain argument.");
11670 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: Op.getOperand(i: ArgStart + 2),
11671 Ptr: Op.getOperand(i: ArgStart + 1), PtrInfo: MachinePointerInfo());
11672 }
11673 default:
11674 break;
11675 }
11676 return SDValue();
11677}
11678
11679// Lower scalar BSWAP64 to xxbrd.
11680SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
11681 SDLoc dl(Op);
11682 if (!Subtarget.isPPC64())
11683 return Op;
11684
11685 if (Subtarget.hasP9Vector()) {
11686 // MTVSRDD
11687 Op = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: MVT::v2i64, N1: Op.getOperand(i: 0),
11688 N2: Op.getOperand(i: 0));
11689 // XXBRD
11690 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Op);
11691 // MFVSRD
11692 int VectorIndex = 0;
11693 if (Subtarget.isLittleEndian())
11694 VectorIndex = 1;
11695 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Op,
11696 N2: DAG.getTargetConstant(Val: VectorIndex, DL: dl, VT: MVT::i32));
11697 return Op;
11698 }
11699
11700 // For Power8, use parallel rotate instructions for faster bswap64.
11701 SDValue Input = Op.getOperand(i: 0);
11702 // Helper to create rotate-and-insert operations (RLWIMI/RLDIMI).
11703 auto CreateRotateInsert =
11704 [&](unsigned Opcode, MVT VT, SDValue Dest, SDValue Src, unsigned RotAmt,
11705 unsigned MaskBegin,
11706 std::optional<unsigned> MaskEnd = std::nullopt) -> SDValue {
11707 SmallVector<SDValue, 5> Ops = {
11708 Dest, Src, DAG.getTargetConstant(Val: RotAmt, DL: dl, VT: MVT::i32),
11709 DAG.getTargetConstant(Val: MaskBegin, DL: dl, VT: MVT::i32)};
11710 if (MaskEnd.has_value())
11711 Ops.push_back(Elt: DAG.getTargetConstant(Val: *MaskEnd, DL: dl, VT: MVT::i32));
11712
11713 return SDValue(DAG.getMachineNode(Opcode, dl, VT, Ops), 0);
11714 };
11715
11716 // Helper to perform 32-bit byte swap using rotl(8) + 2x rlwimi.
11717 auto Swap32 = [&](SDValue Val32) -> SDValue {
11718 SDValue Rot = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Val32,
11719 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
11720 // Insert bits [24:31] from Val32 into Rot at position [0:7].
11721 SDValue Swap =
11722 CreateRotateInsert(PPC::RLWIMI, MVT::i32, Rot, Val32, 24, 0, 7);
11723 // Insert bits [16:23] from Val32 into Swap at position [16:23].
11724 return CreateRotateInsert(PPC::RLWIMI, MVT::i32, Swap, Val32, 24, 16, 23);
11725 };
11726 // Extract and swap high and low 32-bit halves independently for parallelism.
11727 SDValue Hi32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
11728 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Input,
11729 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64)));
11730 SDValue Lo32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Input);
11731
11732 // Combine swapped halves: rotate LoSwap left by 32 bits and insert into
11733 // HiSwap to swap their positions, completing the 64-bit byte reversal.
11734 SDValue HiSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Hi32));
11735 SDValue LoSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Lo32));
11736
11737 return CreateRotateInsert(PPC::RLDIMI, MVT::i64, HiSwap, LoSwap, 32, 0);
11738}
11739
11740// ATOMIC_CMP_SWAP for i8/i16 needs to zero-extend its input since it will be
11741// compared to a value that is atomically loaded (atomic loads zero-extend).
11742SDValue PPCTargetLowering::LowerATOMIC_CMP_SWAP(SDValue Op,
11743 SelectionDAG &DAG) const {
11744 assert(Op.getOpcode() == ISD::ATOMIC_CMP_SWAP &&
11745 "Expecting an atomic compare-and-swap here.");
11746 SDLoc dl(Op);
11747 auto *AtomicNode = cast<AtomicSDNode>(Val: Op.getNode());
11748 EVT MemVT = AtomicNode->getMemoryVT();
11749 if (MemVT.getSizeInBits() >= 32)
11750 return Op;
11751
11752 SDValue CmpOp = Op.getOperand(i: 2);
11753 // If this is already correctly zero-extended, leave it alone.
11754 auto HighBits = APInt::getHighBitsSet(numBits: 32, hiBitsSet: 32 - MemVT.getSizeInBits());
11755 if (DAG.MaskedValueIsZero(Op: CmpOp, Mask: HighBits))
11756 return Op;
11757
11758 // Clear the high bits of the compare operand.
11759 unsigned MaskVal = (1 << MemVT.getSizeInBits()) - 1;
11760 SDValue NewCmpOp =
11761 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: CmpOp,
11762 N2: DAG.getConstant(Val: MaskVal, DL: dl, VT: MVT::i32));
11763
11764 // Replace the existing compare operand with the properly zero-extended one.
11765 SmallVector<SDValue, 4> Ops;
11766 for (int i = 0, e = AtomicNode->getNumOperands(); i < e; i++)
11767 Ops.push_back(Elt: AtomicNode->getOperand(Num: i));
11768 Ops[2] = NewCmpOp;
11769 MachineMemOperand *MMO = AtomicNode->getMemOperand();
11770 SDVTList Tys = DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other);
11771 auto NodeTy =
11772 (MemVT == MVT::i8) ? PPCISD::ATOMIC_CMP_SWAP_8 : PPCISD::ATOMIC_CMP_SWAP_16;
11773 return DAG.getMemIntrinsicNode(Opcode: NodeTy, dl, VTList: Tys, Ops, MemVT, MMO);
11774}
11775
11776SDValue PPCTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
11777 SelectionDAG &DAG) const {
11778 AtomicSDNode *N = cast<AtomicSDNode>(Val: Op.getNode());
11779 EVT MemVT = N->getMemoryVT();
11780 assert(MemVT.getSimpleVT() == MVT::i128 &&
11781 "Expect quadword atomic operations");
11782 SDLoc dl(N);
11783 unsigned Opc = N->getOpcode();
11784 switch (Opc) {
11785 case ISD::ATOMIC_LOAD: {
11786 // Lower quadword atomic load to int_ppc_atomic_load_i128 which will be
11787 // lowered to ppc instructions by pattern matching instruction selector.
11788 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
11789 SmallVector<SDValue, 4> Ops{
11790 N->getOperand(Num: 0),
11791 DAG.getConstant(Val: Intrinsic::ppc_atomic_load_i128, DL: dl, VT: MVT::i32)};
11792 for (int I = 1, E = N->getNumOperands(); I < E; ++I)
11793 Ops.push_back(Elt: N->getOperand(Num: I));
11794 SDValue LoadedVal = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys,
11795 Ops, MemVT, MMO: N->getMemOperand());
11796 SDValue ValLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal);
11797 SDValue ValHi =
11798 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal.getValue(R: 1));
11799 ValHi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ValHi,
11800 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11801 SDValue Val =
11802 DAG.getNode(Opcode: ISD::OR, DL: dl, ResultTys: {MVT::i128, MVT::Other}, Ops: {ValLo, ValHi});
11803 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, ResultTys: {MVT::i128, MVT::Other},
11804 Ops: {Val, LoadedVal.getValue(R: 2)});
11805 }
11806 case ISD::ATOMIC_STORE: {
11807 // Lower quadword atomic store to int_ppc_atomic_store_i128 which will be
11808 // lowered to ppc instructions by pattern matching instruction selector.
11809 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
11810 SmallVector<SDValue, 4> Ops{
11811 N->getOperand(Num: 0),
11812 DAG.getConstant(Val: Intrinsic::ppc_atomic_store_i128, DL: dl, VT: MVT::i32)};
11813 SDValue Val = N->getOperand(Num: 1);
11814 SDValue ValLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: Val);
11815 SDValue ValHi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: Val,
11816 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11817 ValHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: ValHi);
11818 Ops.push_back(Elt: ValLo);
11819 Ops.push_back(Elt: ValHi);
11820 Ops.push_back(Elt: N->getOperand(Num: 2));
11821 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops, MemVT,
11822 MMO: N->getMemOperand());
11823 }
11824 default:
11825 llvm_unreachable("Unexpected atomic opcode");
11826 }
11827}
11828
11829static SDValue getDataClassTest(SDValue Op, FPClassTest Mask, const SDLoc &Dl,
11830 SelectionDAG &DAG,
11831 const PPCSubtarget &Subtarget) {
11832 assert(Mask <= fcAllFlags && "Invalid fp_class flags!");
11833
11834 enum DataClassMask {
11835 DC_NAN = 1 << 6,
11836 DC_NEG_INF = 1 << 4,
11837 DC_POS_INF = 1 << 5,
11838 DC_NEG_ZERO = 1 << 2,
11839 DC_POS_ZERO = 1 << 3,
11840 DC_NEG_SUBNORM = 1,
11841 DC_POS_SUBNORM = 1 << 1,
11842 };
11843
11844 EVT VT = Op.getValueType();
11845
11846 unsigned TestOp = VT == MVT::f128 ? PPC::XSTSTDCQP
11847 : VT == MVT::f64 ? PPC::XSTSTDCDP
11848 : PPC::XSTSTDCSP;
11849
11850 if (Mask == fcAllFlags)
11851 return DAG.getBoolConstant(V: true, DL: Dl, VT: MVT::i1, OpVT: VT);
11852 if (Mask == 0)
11853 return DAG.getBoolConstant(V: false, DL: Dl, VT: MVT::i1, OpVT: VT);
11854
11855 // When it's cheaper or necessary to test reverse flags.
11856 if ((Mask & fcNormal) == fcNormal || Mask == ~fcQNan || Mask == ~fcSNan) {
11857 SDValue Rev = getDataClassTest(Op, Mask: ~Mask, Dl, DAG, Subtarget);
11858 return DAG.getNOT(DL: Dl, Val: Rev, VT: MVT::i1);
11859 }
11860
11861 // Power doesn't support testing whether a value is 'normal'. Test the rest
11862 // first, and test if it's 'not not-normal' with expected sign.
11863 if (Mask & fcNormal) {
11864 SDValue Rev(DAG.getMachineNode(
11865 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11866 Op1: DAG.getTargetConstant(Val: DC_NAN | DC_NEG_INF | DC_POS_INF |
11867 DC_NEG_ZERO | DC_POS_ZERO |
11868 DC_NEG_SUBNORM | DC_POS_SUBNORM,
11869 DL: Dl, VT: MVT::i32),
11870 Op2: Op),
11871 0);
11872 // Sign are stored in CR bit 0, result are in CR bit 2.
11873 SDValue Sign(
11874 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11875 Op2: DAG.getTargetConstant(Val: PPC::sub_lt, DL: Dl, VT: MVT::i32)),
11876 0);
11877 SDValue Normal(DAG.getNOT(
11878 DL: Dl,
11879 Val: SDValue(DAG.getMachineNode(
11880 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11881 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11882 0),
11883 VT: MVT::i1));
11884 if (Mask & fcPosNormal)
11885 Sign = DAG.getNOT(DL: Dl, Val: Sign, VT: MVT::i1);
11886 SDValue Result = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: Sign, N2: Normal);
11887 if (Mask == fcPosNormal || Mask == fcNegNormal)
11888 return Result;
11889
11890 return DAG.getNode(
11891 Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11892 N1: getDataClassTest(Op, Mask: Mask & ~fcNormal, Dl, DAG, Subtarget), N2: Result);
11893 }
11894
11895 // The instruction doesn't differentiate between signaling or quiet NaN. Test
11896 // the rest first, and test if it 'is NaN and is signaling/quiet'.
11897 if ((Mask & fcNan) == fcQNan || (Mask & fcNan) == fcSNan) {
11898 bool IsQuiet = Mask & fcQNan;
11899 SDValue NanCheck = getDataClassTest(Op, Mask: fcNan, Dl, DAG, Subtarget);
11900
11901 // Quietness is determined by the first bit in fraction field.
11902 uint64_t QuietMask = 0;
11903 SDValue HighWord;
11904 if (VT == MVT::f128) {
11905 HighWord = DAG.getNode(
11906 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: DAG.getBitcast(VT: MVT::v4i32, V: Op),
11907 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 3 : 0, DL: Dl));
11908 QuietMask = 0x8000;
11909 } else if (VT == MVT::f64) {
11910 if (Subtarget.isPPC64()) {
11911 HighWord = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::i32,
11912 N1: DAG.getBitcast(VT: MVT::i64, V: Op),
11913 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11914 } else {
11915 SDValue Vec = DAG.getBitcast(
11916 VT: MVT::v4i32, V: DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: Dl, VT: MVT::v2f64, Operand: Op));
11917 HighWord = DAG.getNode(
11918 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: Vec,
11919 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 1 : 0, DL: Dl));
11920 }
11921 QuietMask = 0x80000;
11922 } else if (VT == MVT::f32) {
11923 HighWord = DAG.getBitcast(VT: MVT::i32, V: Op);
11924 QuietMask = 0x400000;
11925 }
11926 SDValue NanRes = DAG.getSetCC(
11927 DL: Dl, VT: MVT::i1,
11928 LHS: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: HighWord,
11929 N2: DAG.getConstant(Val: QuietMask, DL: Dl, VT: MVT::i32)),
11930 RHS: DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32), Cond: IsQuiet ? ISD::SETNE : ISD::SETEQ);
11931 NanRes = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: NanCheck, N2: NanRes);
11932 if (Mask == fcQNan || Mask == fcSNan)
11933 return NanRes;
11934
11935 return DAG.getNode(Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11936 N1: getDataClassTest(Op, Mask: Mask & ~fcNan, Dl, DAG, Subtarget),
11937 N2: NanRes);
11938 }
11939
11940 unsigned NativeMask = 0;
11941 if ((Mask & fcNan) == fcNan)
11942 NativeMask |= DC_NAN;
11943 if (Mask & fcNegInf)
11944 NativeMask |= DC_NEG_INF;
11945 if (Mask & fcPosInf)
11946 NativeMask |= DC_POS_INF;
11947 if (Mask & fcNegZero)
11948 NativeMask |= DC_NEG_ZERO;
11949 if (Mask & fcPosZero)
11950 NativeMask |= DC_POS_ZERO;
11951 if (Mask & fcNegSubnormal)
11952 NativeMask |= DC_NEG_SUBNORM;
11953 if (Mask & fcPosSubnormal)
11954 NativeMask |= DC_POS_SUBNORM;
11955 return SDValue(
11956 DAG.getMachineNode(
11957 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1,
11958 Op1: SDValue(DAG.getMachineNode(
11959 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11960 Op1: DAG.getTargetConstant(Val: NativeMask, DL: Dl, VT: MVT::i32), Op2: Op),
11961 0),
11962 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11963 0);
11964}
11965
11966SDValue PPCTargetLowering::LowerIS_FPCLASS(SDValue Op,
11967 SelectionDAG &DAG) const {
11968 assert(Subtarget.hasP9Vector() && "Test data class requires Power9");
11969 SDValue LHS = Op.getOperand(i: 0);
11970 uint64_t RHSC = Op.getConstantOperandVal(i: 1);
11971 SDLoc Dl(Op);
11972 FPClassTest Category = static_cast<FPClassTest>(RHSC);
11973 if (LHS.getValueType() == MVT::ppcf128) {
11974 // The higher part determines the value class.
11975 LHS = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::f64, N1: LHS,
11976 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11977 }
11978
11979 return getDataClassTest(Op: LHS, Mask: Category, Dl, DAG, Subtarget);
11980}
11981
11982// Adjust the length value for a load/store with length to account for the
11983// instructions requiring a left justified length, and for non-byte element
11984// types requiring scaling by element size.
11985static SDValue AdjustLength(SDValue Val, unsigned Bits, bool Left,
11986 SelectionDAG &DAG) {
11987 SDLoc dl(Val);
11988 EVT VT = Val->getValueType(ResNo: 0);
11989 unsigned LeftAdj = Left ? VT.getSizeInBits() - 8 : 0;
11990 unsigned TypeAdj = llvm::countr_zero<uint32_t>(Val: Bits / 8);
11991 SDValue SHLAmt = DAG.getConstant(Val: LeftAdj + TypeAdj, DL: dl, VT);
11992 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Val, N2: SHLAmt);
11993}
11994
11995SDValue PPCTargetLowering::LowerVP_LOAD(SDValue Op, SelectionDAG &DAG) const {
11996 auto VPLD = cast<VPLoadSDNode>(Val&: Op);
11997 bool Future = Subtarget.isISAFuture();
11998 SDLoc dl(Op);
11999 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(3).getNode(), true) &&
12000 "Mask predication not supported");
12001 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12002 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPLD->getOperand(Num: 4));
12003 unsigned IID = Future ? Intrinsic::ppc_vsx_lxvrl : Intrinsic::ppc_vsx_lxvl;
12004 unsigned EltBits = Op->getValueType(ResNo: 0).getScalarType().getSizeInBits();
12005 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
12006 SDValue Ops[] = {VPLD->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
12007 VPLD->getOperand(Num: 1), Len};
12008 SDVTList Tys = DAG.getVTList(VT1: Op->getValueType(ResNo: 0), VT2: MVT::Other);
12009 SDValue VPL =
12010 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys, Ops,
12011 MemVT: VPLD->getMemoryVT(), MMO: VPLD->getMemOperand());
12012 return VPL;
12013}
12014
12015SDValue PPCTargetLowering::LowerVP_STORE(SDValue Op, SelectionDAG &DAG) const {
12016 auto VPST = cast<VPStoreSDNode>(Val&: Op);
12017 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(4).getNode(), true) &&
12018 "Mask predication not supported");
12019 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12020 SDLoc dl(Op);
12021 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPST->getOperand(Num: 5));
12022 unsigned EltBits =
12023 Op->getOperand(Num: 1).getValueType().getScalarType().getSizeInBits();
12024 bool Future = Subtarget.isISAFuture();
12025 unsigned IID = Future ? Intrinsic::ppc_vsx_stxvrl : Intrinsic::ppc_vsx_stxvl;
12026 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
12027 SDValue Ops[] = {
12028 VPST->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
12029 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: VPST->getOperand(Num: 1)),
12030 VPST->getOperand(Num: 2), Len};
12031 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12032 SDValue VPS =
12033 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12034 MemVT: VPST->getMemoryVT(), MMO: VPST->getMemOperand());
12035 return VPS;
12036}
12037
12038SDValue PPCTargetLowering::LowerPartialReduce(SDValue Op,
12039 SelectionDAG &DAG) const {
12040 SDValue Acc = Op.getOperand(i: 0);
12041 SDValue Op1 = Op.getOperand(i: 1);
12042 SDValue Op2 = Op.getOperand(i: 2);
12043
12044 assert(Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA &&
12045 "Unexpected partial reduction");
12046
12047 if (Acc.getValueType() != MVT::v4i32)
12048 return SDValue();
12049 if (Op1.getValueType() != MVT::v16i32 || Op1.getOpcode() != ISD::SIGN_EXTEND)
12050 return SDValue();
12051 SDValue Op1Input = Op1.getOperand(i: 0);
12052 if (Op1Input.getValueType() != MVT::v16i8 || !llvm::isOneOrOneSplat(V: Op2))
12053 return SDValue();
12054
12055 SDLoc dl(Op);
12056 SDValue Ones = DAG.getConstant(Val: 1, DL: dl, VT: MVT::v16i8);
12057 return DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_SUMLA, DL: dl, VT: MVT::v4i32, N1: Acc, N2: Op1Input,
12058 N3: Ones);
12059}
12060
12061SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
12062 SelectionDAG &DAG) const {
12063 SDLoc dl(Op);
12064
12065 MachineFunction &MF = DAG.getMachineFunction();
12066 SDValue Op0 = Op.getOperand(i: 0);
12067 EVT ValVT = Op0.getValueType();
12068 unsigned EltSize = Op.getValueType().getScalarSizeInBits();
12069 if (isa<ConstantSDNode>(Val: Op0) && EltSize <= 32) {
12070 int64_t IntVal = Op.getConstantOperandVal(i: 0);
12071 if (IntVal >= -16 && IntVal <= 15)
12072 return getCanonicalConstSplat(Val: IntVal, SplatSize: EltSize / 8, VT: Op.getValueType(), DAG,
12073 dl);
12074 }
12075
12076 ReuseLoadInfo RLI;
12077 if (Subtarget.hasLFIWAX() && Subtarget.hasVSX() &&
12078 Op.getValueType() == MVT::v4i32 && Op0.getOpcode() == ISD::LOAD &&
12079 Op0.getValueType() == MVT::i32 && Op0.hasOneUse() &&
12080 canReuseLoadAddress(Op: Op0, MemVT: MVT::i32, RLI, DAG, ET: ISD::NON_EXTLOAD)) {
12081
12082 MachineMemOperand *MMO = MF.getMachineMemOperand(
12083 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
12084 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
12085 SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
12086 SDValue Bits = DAG.getMemIntrinsicNode(
12087 Opcode: PPCISD::LD_SPLAT, dl, VTList: DAG.getVTList(VT1: MVT::v4i32, VT2: MVT::Other), Ops,
12088 MemVT: MVT::i32, MMO);
12089 if (RLI.ResChain)
12090 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
12091 return Bits.getValue(R: 0);
12092 }
12093
12094 // Create a stack slot that is 16-byte aligned.
12095 MachineFrameInfo &MFI = MF.getFrameInfo();
12096 int FrameIdx = MFI.CreateStackObject(Size: 16, Alignment: Align(16), isSpillSlot: false);
12097 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12098 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
12099
12100 SDValue Val = Op0;
12101 // P10 hardware store forwarding requires that a single store contains all
12102 // the data for the load. P10 is able to merge a pair of adjacent stores. Try
12103 // to avoid load hit store on P10 when running binaries compiled for older
12104 // processors by generating two mergeable scalar stores to forward with the
12105 // vector load.
12106 if (!DisableP10StoreForward && Subtarget.isPPC64() &&
12107 !Subtarget.isLittleEndian() && ValVT.isInteger() &&
12108 ValVT.getSizeInBits() <= 64) {
12109 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: Val);
12110 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: MVT::i64, DL: DAG.getDataLayout());
12111 SDValue ShiftBy = DAG.getConstant(
12112 Val: 64 - Op.getValueType().getScalarSizeInBits(), DL: dl, VT: ShiftAmountTy);
12113 Val = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: Val, N2: ShiftBy);
12114 SDValue Plus8 =
12115 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FIdx, N2: DAG.getConstant(Val: 8, DL: dl, VT: PtrVT));
12116 SDValue Store2 =
12117 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: Plus8, PtrInfo: MachinePointerInfo());
12118 SDValue Store = DAG.getStore(Chain: Store2, dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12119 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx,
12120 PtrInfo: MachinePointerInfo());
12121 }
12122
12123 // Store the input value into Value#0 of the stack slot.
12124 SDValue Store =
12125 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12126 // Load it out.
12127 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12128}
12129
12130SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
12131 SelectionDAG &DAG) const {
12132 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
12133 "Should only be called for ISD::INSERT_VECTOR_ELT");
12134
12135 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
12136
12137 EVT VT = Op.getValueType();
12138 SDLoc dl(Op);
12139 SDValue V1 = Op.getOperand(i: 0);
12140 SDValue V2 = Op.getOperand(i: 1);
12141
12142 if (VT == MVT::v2f64 && C)
12143 return Op;
12144
12145 if (Subtarget.hasP9Vector()) {
12146 // A f32 load feeding into a v4f32 insert_vector_elt is handled in this way
12147 // because on P10, it allows this specific insert_vector_elt load pattern to
12148 // utilize the refactored load and store infrastructure in order to exploit
12149 // prefixed loads.
12150 // On targets with inexpensive direct moves (Power9 and up), a
12151 // (insert_vector_elt v4f32:$vec, (f32 load)) is always better as an integer
12152 // load since a single precision load will involve conversion to double
12153 // precision on the load followed by another conversion to single precision.
12154 if ((VT == MVT::v4f32) && (V2.getValueType() == MVT::f32) &&
12155 (isa<LoadSDNode>(Val: V2))) {
12156 SDValue BitcastVector = DAG.getBitcast(VT: MVT::v4i32, V: V1);
12157 SDValue BitcastLoad = DAG.getBitcast(VT: MVT::i32, V: V2);
12158 SDValue InsVecElt =
12159 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v4i32, N1: BitcastVector,
12160 N2: BitcastLoad, N3: Op.getOperand(i: 2));
12161 return DAG.getBitcast(VT: MVT::v4f32, V: InsVecElt);
12162 }
12163 }
12164
12165 if (Subtarget.isISA3_1()) {
12166 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && !Subtarget.isPPC64())
12167 return SDValue();
12168 // On P10, we have legal lowering for constant and variable indices for
12169 // all vectors.
12170 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
12171 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64)
12172 return Op;
12173 }
12174
12175 // Before P10, we have legal lowering for constant indices but not for
12176 // variable ones.
12177 if (!C)
12178 return SDValue();
12179
12180 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types.
12181 if (VT == MVT::v8i16 || VT == MVT::v16i8) {
12182 SDValue Mtvsrz = DAG.getNode(Opcode: PPCISD::MTVSRZ, DL: dl, VT, Operand: V2);
12183 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8;
12184 unsigned InsertAtElement = C->getZExtValue();
12185 unsigned InsertAtByte = InsertAtElement * BytesInEachElement;
12186 if (Subtarget.isLittleEndian()) {
12187 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte;
12188 }
12189 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT, N1: V1, N2: Mtvsrz,
12190 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
12191 }
12192 return Op;
12193}
12194
12195SDValue PPCTargetLowering::LowerDMFVectorLoad(SDValue Op,
12196 SelectionDAG &DAG) const {
12197 SDLoc dl(Op);
12198 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12199 SDValue LoadChain = LN->getChain();
12200 SDValue BasePtr = LN->getBasePtr();
12201 EVT VT = Op.getValueType();
12202 bool IsV1024i1 = VT == MVT::v1024i1;
12203 bool IsV2048i1 = VT == MVT::v2048i1;
12204
12205 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12206 // Dense Math dmr pair registers, respectively.
12207 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12208 (void)IsV2048i1;
12209 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12210 "Dense Math support required.");
12211 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12212
12213 SmallVector<SDValue, 8> Loads;
12214 SmallVector<SDValue, 8> LoadChains;
12215
12216 SDValue IntrinID = DAG.getConstant(Val: Intrinsic::ppc_vsx_lxvp, DL: dl, VT: MVT::i32);
12217 SDValue LoadOps[] = {LoadChain, IntrinID, BasePtr};
12218 MachineMemOperand *MMO = LN->getMemOperand();
12219 unsigned NumVecs = VT.getSizeInBits() / 256;
12220 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12221 MachineMemOperand *NewMMO =
12222 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12223 if (Idx > 0) {
12224 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12225 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12226 LoadOps[2] = BasePtr;
12227 }
12228 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
12229 VTList: DAG.getVTList(VT1: MVT::v256i1, VT2: MVT::Other),
12230 Ops: LoadOps, MemVT: MVT::v256i1, MMO: NewMMO);
12231 LoadChains.push_back(Elt: Ld.getValue(R: 1));
12232 Loads.push_back(Elt: Ld);
12233 }
12234
12235 if (Subtarget.isLittleEndian()) {
12236 std::reverse(first: Loads.begin(), last: Loads.end());
12237 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12238 }
12239
12240 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12241 SDValue Value = DMFInsert1024(Pairs: Loads, dl, DAG);
12242
12243 if (IsV1024i1) {
12244 return DAG.getMergeValues(Ops: {Value, TF}, dl);
12245 }
12246
12247 // Handle Loads for V2048i1 which represents a dmr pair.
12248 SmallVector<SDValue, 4> MoreLoads{Loads[4], Loads[5], Loads[6], Loads[7]};
12249 SDValue Dmr1Value = DMFInsert1024(Pairs: MoreLoads, dl, DAG);
12250
12251 SDValue Dmr0Sub = DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32);
12252 SDValue Dmr1Sub = DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32);
12253
12254 SDValue DmrPRC = DAG.getTargetConstant(Val: PPC::DMRpRCRegClassID, DL: dl, VT: MVT::i32);
12255 const SDValue DmrPOps[] = {DmrPRC, Value, Dmr0Sub, Dmr1Value, Dmr1Sub};
12256
12257 SDValue DmrPValue = SDValue(
12258 DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v2048i1, Ops: DmrPOps), 0);
12259
12260 return DAG.getMergeValues(Ops: {DmrPValue, TF}, dl);
12261}
12262
12263SDValue PPCTargetLowering::DMFInsert1024(const SmallVectorImpl<SDValue> &Pairs,
12264 const SDLoc &dl,
12265 SelectionDAG &DAG) const {
12266 SDValue Lo =
12267 DAG.getNode(Opcode: PPCISD::INST512, DL: dl, VT: MVT::v512i1, N1: Pairs[0], N2: Pairs[1]);
12268 SDValue LoSub = DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32);
12269 SDValue Hi =
12270 DAG.getNode(Opcode: PPCISD::INST512HI, DL: dl, VT: MVT::v512i1, N1: Pairs[2], N2: Pairs[3]);
12271 SDValue HiSub = DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32);
12272 SDValue RC = DAG.getTargetConstant(Val: PPC::DMRRCRegClassID, DL: dl, VT: MVT::i32);
12273
12274 return SDValue(DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v1024i1,
12275 Ops: {RC, Lo, LoSub, Hi, HiSub}),
12276 0);
12277}
12278
12279SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
12280 SelectionDAG &DAG) const {
12281 SDLoc dl(Op);
12282 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12283 SDValue LoadChain = LN->getChain();
12284 SDValue BasePtr = LN->getBasePtr();
12285 EVT VT = Op.getValueType();
12286
12287 if (VT == MVT::v1024i1 || VT == MVT::v2048i1)
12288 return LowerDMFVectorLoad(Op, DAG);
12289
12290 if (VT != MVT::v256i1 && VT != MVT::v512i1)
12291 return Op;
12292
12293 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12294 assert((VT != MVT::v512i1 || Subtarget.hasMMA()) &&
12295 "Type unsupported without MMA");
12296 assert((VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12297 "Type unsupported without paired vector support");
12298
12299 // For v256i1 on ISA Future, let the load go through to instruction selection
12300 // where it will be matched to lxvp/plxvp by the instruction patterns.
12301 if (VT == MVT::v256i1 && Subtarget.isISAFuture())
12302 return Op;
12303
12304 // For other cases, create 2 or 4 v16i8 loads to load the pair or accumulator
12305 // value in 2 or 4 vsx registers.
12306 Align Alignment = LN->getAlign();
12307 SmallVector<SDValue, 4> Loads;
12308 SmallVector<SDValue, 4> LoadChains;
12309 unsigned NumVecs = VT.getSizeInBits() / 128;
12310 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12311 SDValue Load =
12312 DAG.getLoad(VT: MVT::v16i8, dl, Chain: LoadChain, Ptr: BasePtr,
12313 PtrInfo: LN->getPointerInfo().getWithOffset(O: Idx * 16),
12314 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12315 MMOFlags: LN->getMemOperand()->getFlags(), Metadata: LN->getAAInfo());
12316 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12317 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12318 Loads.push_back(Elt: Load);
12319 LoadChains.push_back(Elt: Load.getValue(R: 1));
12320 }
12321 if (Subtarget.isLittleEndian()) {
12322 std::reverse(first: Loads.begin(), last: Loads.end());
12323 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12324 }
12325 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12326 SDValue Value =
12327 DAG.getNode(Opcode: VT == MVT::v512i1 ? PPCISD::ACC_BUILD : PPCISD::PAIR_BUILD,
12328 DL: dl, VT, Ops: Loads);
12329 SDValue RetOps[] = {Value, TF};
12330 return DAG.getMergeValues(Ops: RetOps, dl);
12331}
12332
12333SDValue PPCTargetLowering::LowerDMFVectorStore(SDValue Op,
12334 SelectionDAG &DAG) const {
12335
12336 SDLoc dl(Op);
12337 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12338 SDValue StoreChain = SN->getChain();
12339 SDValue BasePtr = SN->getBasePtr();
12340 SmallVector<SDValue, 8> Values;
12341 SmallVector<SDValue, 8> Stores;
12342 EVT VT = SN->getValue().getValueType();
12343 bool IsV1024i1 = VT == MVT::v1024i1;
12344 bool IsV2048i1 = VT == MVT::v2048i1;
12345
12346 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12347 // Dense Math dmr pair registers, respectively.
12348 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12349 (void)IsV2048i1;
12350 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12351 "Dense Math support required.");
12352 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12353
12354 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12355 if (IsV1024i1) {
12356 SDValue Lo(DAG.getMachineNode(
12357 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12358 Op1: Op.getOperand(i: 1),
12359 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12360 0);
12361 SDValue Hi(DAG.getMachineNode(
12362 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12363 Op1: Op.getOperand(i: 1),
12364 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12365 0);
12366 MachineSDNode *ExtNode =
12367 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Lo);
12368 Values.push_back(Elt: SDValue(ExtNode, 0));
12369 Values.push_back(Elt: SDValue(ExtNode, 1));
12370 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Hi);
12371 Values.push_back(Elt: SDValue(ExtNode, 0));
12372 Values.push_back(Elt: SDValue(ExtNode, 1));
12373 } else {
12374 // This corresponds to v2048i1 which represents a dmr pair.
12375 SDValue Dmr0(
12376 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12377 Op1: Op.getOperand(i: 1),
12378 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32)),
12379 0);
12380
12381 SDValue Dmr1(
12382 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12383 Op1: Op.getOperand(i: 1),
12384 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32)),
12385 0);
12386
12387 SDValue Dmr0Lo(DAG.getMachineNode(
12388 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12389 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12390 0);
12391
12392 SDValue Dmr0Hi(DAG.getMachineNode(
12393 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12394 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12395 0);
12396
12397 SDValue Dmr1Lo(DAG.getMachineNode(
12398 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12399 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12400 0);
12401
12402 SDValue Dmr1Hi(DAG.getMachineNode(
12403 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12404 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12405 0);
12406
12407 MachineSDNode *ExtNode =
12408 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr0Lo);
12409 Values.push_back(Elt: SDValue(ExtNode, 0));
12410 Values.push_back(Elt: SDValue(ExtNode, 1));
12411 ExtNode =
12412 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr0Hi);
12413 Values.push_back(Elt: SDValue(ExtNode, 0));
12414 Values.push_back(Elt: SDValue(ExtNode, 1));
12415 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr1Lo);
12416 Values.push_back(Elt: SDValue(ExtNode, 0));
12417 Values.push_back(Elt: SDValue(ExtNode, 1));
12418 ExtNode =
12419 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr1Hi);
12420 Values.push_back(Elt: SDValue(ExtNode, 0));
12421 Values.push_back(Elt: SDValue(ExtNode, 1));
12422 }
12423
12424 if (Subtarget.isLittleEndian())
12425 std::reverse(first: Values.begin(), last: Values.end());
12426
12427 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12428 SmallVector<SDValue, 4> Ops{
12429 StoreChain, DAG.getConstant(Val: Intrinsic::ppc_vsx_stxvp, DL: dl, VT: MVT::i32),
12430 Values[0], BasePtr};
12431 MachineMemOperand *MMO = SN->getMemOperand();
12432 unsigned NumVecs = VT.getSizeInBits() / 256;
12433 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12434 MachineMemOperand *NewMMO =
12435 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12436 if (Idx > 0) {
12437 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12438 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12439 Ops[3] = BasePtr;
12440 }
12441 Ops[2] = Values[Idx];
12442 SDValue St = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12443 MemVT: MVT::v256i1, MMO: NewMMO);
12444 Stores.push_back(Elt: St);
12445 }
12446
12447 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12448 return TF;
12449}
12450
12451SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
12452 SelectionDAG &DAG) const {
12453 SDLoc dl(Op);
12454 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12455 SDValue StoreChain = SN->getChain();
12456 SDValue BasePtr = SN->getBasePtr();
12457 SDValue Value = SN->getValue();
12458 SDValue Value2 = SN->getValue();
12459 EVT StoreVT = Value.getValueType();
12460
12461 if (StoreVT == MVT::v1024i1 || StoreVT == MVT::v2048i1)
12462 return LowerDMFVectorStore(Op, DAG);
12463
12464 if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1)
12465 return Op;
12466
12467 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12468 assert((StoreVT != MVT::v512i1 || Subtarget.hasMMA()) &&
12469 "Type unsupported without MMA");
12470 assert((StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12471 "Type unsupported without paired vector support");
12472
12473 // For v256i1 on ISA Future, let the store go through to instruction selection
12474 // where it will be matched to stxvp/pstxvp by the instruction patterns.
12475 if (StoreVT == MVT::v256i1 && Subtarget.isISAFuture() &&
12476 !DisableAutoPairedVecSt)
12477 return Op;
12478
12479 // For other cases, create 2 or 4 v16i8 stores to store the pair or
12480 // accumulator underlying registers individually.
12481 Align Alignment = SN->getAlign();
12482 SmallVector<SDValue, 4> Stores;
12483 unsigned NumVecs = 2;
12484 if (StoreVT == MVT::v512i1) {
12485 if (Subtarget.isISAFuture()) {
12486 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12487 MachineSDNode *ExtNode = DAG.getMachineNode(
12488 Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Op.getOperand(i: 1));
12489
12490 Value = SDValue(ExtNode, 0);
12491 Value2 = SDValue(ExtNode, 1);
12492 } else
12493 Value = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: Value);
12494 NumVecs = 4;
12495 }
12496 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12497 unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx;
12498 SDValue Elt;
12499 if (Subtarget.isISAFuture()) {
12500 VecNum = Subtarget.isLittleEndian() ? 1 - (Idx % 2) : (Idx % 2);
12501 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
12502 N1: Idx > 1 ? Value2 : Value,
12503 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12504 } else
12505 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: Value,
12506 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12507
12508 SDValue Store =
12509 DAG.getStore(Chain: StoreChain, dl, Val: Elt, Ptr: BasePtr,
12510 PtrInfo: SN->getPointerInfo().getWithOffset(O: Idx * 16),
12511 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12512 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
12513 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12514 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12515 Stores.push_back(Elt: Store);
12516 }
12517 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12518 return TF;
12519}
12520
12521SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
12522 SDLoc dl(Op);
12523 if (Op.getValueType() == MVT::v4i32) {
12524 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12525
12526 SDValue Zero = getCanonicalConstSplat(Val: 0, SplatSize: 1, VT: MVT::v4i32, DAG, dl);
12527 // +16 as shift amt.
12528 SDValue Neg16 = getCanonicalConstSplat(Val: -16, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
12529 SDValue RHSSwap = // = vrlw RHS, 16
12530 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vrlw, LHS: RHS, RHS: Neg16, DAG, dl);
12531
12532 // Shrinkify inputs to v8i16.
12533 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: LHS);
12534 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHS);
12535 RHSSwap = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHSSwap);
12536
12537 // Low parts multiplied together, generating 32-bit results (we ignore the
12538 // top parts).
12539 SDValue LoProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmulouh,
12540 LHS, RHS, DAG, dl, DestVT: MVT::v4i32);
12541
12542 SDValue HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmsumuhm,
12543 Op0: LHS, Op1: RHSSwap, Op2: Zero, DAG, dl, DestVT: MVT::v4i32);
12544 // Shift the high parts up 16 bits.
12545 HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: HiProd,
12546 RHS: Neg16, DAG, dl);
12547 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: LoProd, N2: HiProd);
12548 } else if (Op.getValueType() == MVT::v16i8) {
12549 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12550 bool isLittleEndian = Subtarget.isLittleEndian();
12551
12552 // Multiply the even 8-bit parts, producing 16-bit sums.
12553 SDValue EvenParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuleub,
12554 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12555 EvenParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: EvenParts);
12556
12557 // Multiply the odd 8-bit parts, producing 16-bit sums.
12558 SDValue OddParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuloub,
12559 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12560 OddParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OddParts);
12561
12562 // Merge the results together. Because vmuleub and vmuloub are
12563 // instructions with a big-endian bias, we must reverse the
12564 // element numbering and reverse the meaning of "odd" and "even"
12565 // when generating little endian code.
12566 int Ops[16];
12567 for (unsigned i = 0; i != 8; ++i) {
12568 if (isLittleEndian) {
12569 Ops[i*2 ] = 2*i;
12570 Ops[i*2+1] = 2*i+16;
12571 } else {
12572 Ops[i*2 ] = 2*i+1;
12573 Ops[i*2+1] = 2*i+1+16;
12574 }
12575 }
12576 if (isLittleEndian)
12577 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OddParts, N2: EvenParts, Mask: Ops);
12578 else
12579 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: EvenParts, N2: OddParts, Mask: Ops);
12580 } else {
12581 llvm_unreachable("Unknown mul to lower!");
12582 }
12583}
12584
12585SDValue PPCTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
12586 bool IsStrict = Op->isStrictFPOpcode();
12587 if (Op.getOperand(i: IsStrict ? 1 : 0).getValueType() == MVT::f128 &&
12588 !Subtarget.hasP9Vector())
12589 return SDValue();
12590
12591 return Op;
12592}
12593
12594// Custom lowering for fpext vf32 to v2f64
12595SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
12596
12597 assert(Op.getOpcode() == ISD::FP_EXTEND &&
12598 "Should only be called for ISD::FP_EXTEND");
12599
12600 // FIXME: handle extends from half precision float vectors on P9.
12601 // We only want to custom lower an extend from v2f32 to v2f64.
12602 if (Op.getValueType() != MVT::v2f64 ||
12603 Op.getOperand(i: 0).getValueType() != MVT::v2f32)
12604 return SDValue();
12605
12606 SDLoc dl(Op);
12607 SDValue Op0 = Op.getOperand(i: 0);
12608
12609 switch (Op0.getOpcode()) {
12610 default:
12611 return SDValue();
12612 case ISD::EXTRACT_SUBVECTOR: {
12613 assert(Op0.getNumOperands() == 2 &&
12614 isa<ConstantSDNode>(Op0->getOperand(1)) &&
12615 "Node should have 2 operands with second one being a constant!");
12616
12617 if (Op0.getOperand(i: 0).getValueType() != MVT::v4f32)
12618 return SDValue();
12619
12620 // Custom lower is only done for high or low doubleword.
12621 int Idx = Op0.getConstantOperandVal(i: 1);
12622 if (Idx % 2 != 0)
12623 return SDValue();
12624
12625 // Since input is v4f32, at this point Idx is either 0 or 2.
12626 // Shift to get the doubleword position we want.
12627 int DWord = Idx >> 1;
12628
12629 // High and low word positions are different on little endian.
12630 if (Subtarget.isLittleEndian())
12631 DWord ^= 0x1;
12632
12633 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64,
12634 N1: Op0.getOperand(i: 0), N2: DAG.getConstant(Val: DWord, DL: dl, VT: MVT::i32));
12635 }
12636 case ISD::FADD:
12637 case ISD::FMUL:
12638 case ISD::FSUB: {
12639 SDValue NewLoad[2];
12640 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) {
12641 // Ensure both input are loads.
12642 SDValue LdOp = Op0.getOperand(i);
12643 if (LdOp.getOpcode() != ISD::LOAD)
12644 return SDValue();
12645 // Generate new load node.
12646 LoadSDNode *LD = cast<LoadSDNode>(Val&: LdOp);
12647 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12648 NewLoad[i] = DAG.getMemIntrinsicNode(
12649 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12650 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12651 }
12652 SDValue NewOp =
12653 DAG.getNode(Opcode: Op0.getOpcode(), DL: SDLoc(Op0), VT: MVT::v4f32, N1: NewLoad[0],
12654 N2: NewLoad[1], Flags: Op0.getNode()->getFlags());
12655 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewOp,
12656 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12657 }
12658 case ISD::LOAD: {
12659 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op0);
12660 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12661 SDValue NewLd = DAG.getMemIntrinsicNode(
12662 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12663 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12664 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewLd,
12665 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12666 }
12667 }
12668 llvm_unreachable("ERROR:Should return for all cases within swtich.");
12669}
12670
12671static SDValue ConvertCarryValueToCarryFlag(EVT SumType, SDValue Value,
12672 SelectionDAG &DAG,
12673 const PPCSubtarget &STI) {
12674 SDLoc DL(Value);
12675 if (STI.useCRBits())
12676 Value = DAG.getNode(Opcode: ISD::SELECT, DL, VT: SumType, N1: Value,
12677 N2: DAG.getConstant(Val: 1, DL, VT: SumType),
12678 N3: DAG.getConstant(Val: 0, DL, VT: SumType));
12679 else
12680 Value = DAG.getZExtOrTrunc(Op: Value, DL, VT: SumType);
12681 SDValue Sum = DAG.getNode(Opcode: PPCISD::ADDC, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32),
12682 N1: Value, N2: DAG.getAllOnesConstant(DL, VT: SumType));
12683 return Sum.getValue(R: 1);
12684}
12685
12686static SDValue ConvertCarryFlagToCarryValue(EVT SumType, SDValue Flag,
12687 EVT CarryType, SelectionDAG &DAG,
12688 const PPCSubtarget &STI) {
12689 SDLoc DL(Flag);
12690 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SumType);
12691 SDValue Carry = DAG.getNode(
12692 Opcode: PPCISD::ADDE, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32), N1: Zero, N2: Zero, N3: Flag);
12693 if (STI.useCRBits())
12694 return DAG.getSetCC(DL, VT: CarryType, LHS: Carry, RHS: Zero, Cond: ISD::SETNE);
12695 return DAG.getZExtOrTrunc(Op: Carry, DL, VT: CarryType);
12696}
12697
12698SDValue PPCTargetLowering::LowerADDSUBO(SDValue Op, SelectionDAG &DAG) const {
12699
12700 SDLoc DL(Op);
12701 SDNode *N = Op.getNode();
12702 EVT VT = N->getValueType(ResNo: 0);
12703 EVT CarryType = N->getValueType(ResNo: 1);
12704 unsigned Opc = N->getOpcode();
12705 bool IsAdd = Opc == ISD::UADDO;
12706 Opc = IsAdd ? PPCISD::ADDC : PPCISD::SUBC;
12707 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12708 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
12709 SDValue Carry = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType,
12710 DAG, STI: Subtarget);
12711 if (!IsAdd)
12712 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Carry,
12713 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
12714 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: Carry);
12715}
12716
12717SDValue PPCTargetLowering::LowerADDSUBO_CARRY(SDValue Op,
12718 SelectionDAG &DAG) const {
12719 SDLoc DL(Op);
12720 SDNode *N = Op.getNode();
12721 unsigned Opc = N->getOpcode();
12722 EVT VT = N->getValueType(ResNo: 0);
12723 EVT CarryType = N->getValueType(ResNo: 1);
12724 SDValue CarryOp = N->getOperand(Num: 2);
12725 bool IsAdd = Opc == ISD::UADDO_CARRY;
12726 Opc = IsAdd ? PPCISD::ADDE : PPCISD::SUBE;
12727 if (!IsAdd)
12728 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12729 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12730 CarryOp = ConvertCarryValueToCarryFlag(SumType: VT, Value: CarryOp, DAG, STI: Subtarget);
12731 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12732 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: CarryOp);
12733 CarryOp = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType, DAG,
12734 STI: Subtarget);
12735 if (!IsAdd)
12736 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12737 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12738 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: CarryOp);
12739}
12740
12741SDValue PPCTargetLowering::LowerSSUBO(SDValue Op, SelectionDAG &DAG) const {
12742
12743 SDLoc dl(Op);
12744 SDValue LHS = Op.getOperand(i: 0);
12745 SDValue RHS = Op.getOperand(i: 1);
12746 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12747
12748 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
12749
12750 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: RHS, N2: LHS);
12751 SDValue Xor2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sub, N2: LHS);
12752
12753 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Xor1, N2: Xor2);
12754
12755 SDValue Overflow =
12756 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: And,
12757 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12758
12759 SDValue OverflowTrunc =
12760 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12761
12762 return DAG.getMergeValues(Ops: {Sub, OverflowTrunc}, dl);
12763}
12764
12765/// Implements signed add with overflow detection using the rule:
12766/// (x eqv y) & (sum xor x), where the overflow bit is extracted from the sign
12767SDValue PPCTargetLowering::LowerSADDO(SDValue Op, SelectionDAG &DAG) const {
12768
12769 SDLoc dl(Op);
12770 SDValue LHS = Op.getOperand(i: 0);
12771 SDValue RHS = Op.getOperand(i: 1);
12772 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12773
12774 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
12775
12776 // Compute ~(x xor y)
12777 SDValue XorXY = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
12778 SDValue EqvXY = DAG.getNOT(DL: dl, Val: XorXY, VT);
12779 // Compute (s xor x)
12780 SDValue SumXorX = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sum, N2: LHS);
12781
12782 // overflow = (x eqv y) & (s xor x)
12783 SDValue OverflowInSign = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: EqvXY, N2: SumXorX);
12784
12785 // Shift sign bit down to LSB
12786 SDValue Overflow =
12787 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: OverflowInSign,
12788 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12789 // Truncate to the overflow type (i1)
12790 SDValue OverflowTrunc =
12791 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12792
12793 return DAG.getMergeValues(Ops: {Sum, OverflowTrunc}, dl);
12794}
12795
12796/// Lower ABDU with negation pattern using branchless carry arithmetic.
12797/// Recognizes: abdu(a, sub(0, x)) and transforms to:
12798/// a - (0 - x) = a + x (mod 2^n)
12799/// Uses SUBC to compute result without branches.
12800SDValue PPCTargetLowering::LowerABDU(SDValue Op, SelectionDAG &DAG) const {
12801 SDLoc DL(Op);
12802 SDValue LHS = Op.getOperand(i: 0);
12803 SDValue RHS = Op.getOperand(i: 1);
12804 EVT OpVT = LHS.getValueType();
12805 EVT VT = Op.getValueType();
12806 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
12807
12808 // If the subtract doesn't overflow then just use abs(sub()).
12809 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: LHS, N1: RHS))
12810 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12811 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: RHS));
12812
12813 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: RHS, N1: LHS))
12814 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12815 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: RHS, N2: LHS));
12816
12817 // General path: use SUBC (or ADDC when RHS is 0-X) to get
12818 // subtract-with-flags, then CMOV to select a-b or b-a. ADDC/SUBC produce the
12819 // flags we need.
12820 unsigned Opcode = PPCISD::SUBC;
12821
12822 // Check if RHS is a negation (0 - X). If so, we can use ADDC instead of SUBC:
12823 // a - (0 - x) = a + x (mod 2^n)
12824 // Same semantics as in LowerCMP; apply same safety checks.
12825 if (RHS.getOpcode() == ISD::SUB) {
12826 SDValue SubLHS = RHS.getOperand(i: 0);
12827 SDValue SubRHS = RHS.getOperand(i: 1);
12828
12829 if (isNullConstant(V: SubLHS) && DAG.isKnownNeverZero(Op: SubRHS)) {
12830 Opcode = PPCISD::ADDC;
12831 RHS = SubRHS;
12832 }
12833 }
12834
12835 // On PPC64, carry ops use the full 64-bit register. Operands are type-legal
12836 // i32 here; widen only for the carry path (fast abs(sub) stays at VT).
12837 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12838 LHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: LHS);
12839 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: RHS);
12840 OpVT = MVT::i64;
12841 }
12842
12843 // Calculate LHS - RHS and capture the carry (CA)
12844 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12845 SDValue Res = DAG.getNode(Opcode, DL, VTList: VTs, N1: LHS, N2: RHS);
12846 SDValue CA0 = Res.getValue(R: 1);
12847
12848 // t2 = A - B + CA0 using SUBE.
12849 SDValue ZeroOrNeg1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Res, N2: Res, N3: CA0);
12850
12851 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: OpVT, N1: Res, N2: ZeroOrNeg1);
12852
12853 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: Xor, N2: ZeroOrNeg1);
12854
12855 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
12856 return Res;
12857}
12858
12859// Lower unsigned 3-way compare producing -1/0/1.
12860SDValue PPCTargetLowering::LowerUCMP(SDValue Op, SelectionDAG &DAG) const {
12861 SDLoc DL(Op);
12862 SDValue A = DAG.getFreeze(V: Op.getOperand(i: 0));
12863 SDValue B = DAG.getFreeze(V: Op.getOperand(i: 1));
12864 EVT OpVT = A.getValueType();
12865 EVT ResVT = Op.getValueType();
12866
12867 // On PPC64, i32 carries are affected by the upper 32 bits of the registers.
12868 // We must zero-extend to i64 to ensure the carry reflects the 32-bit unsigned
12869 // comparison.
12870 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12871 A = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: A);
12872 B = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: B);
12873 OpVT = MVT::i64;
12874 }
12875
12876 // First compute diff = A - B.
12877 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: A, N2: B);
12878
12879 // Generate B - A using SUBC to capture carry.
12880 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12881 SDValue SubC = DAG.getNode(Opcode: PPCISD::SUBC, DL, VTList: VTs, N1: B, N2: A);
12882 SDValue CA0 = SubC.getValue(R: 1);
12883
12884 // t2 = A - B + CA0 using SUBE.
12885 SDValue SubE1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: A, N2: B, N3: CA0);
12886 SDValue CA1 = SubE1.getValue(R: 1);
12887
12888 // res = diff - t2 + CA1 using SUBE (produces desired -1/0/1).
12889 SDValue ResPair = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Diff, N2: SubE1, N3: CA1);
12890
12891 // Extract the first result and truncate to result type if needed.
12892 return DAG.getSExtOrTrunc(Op: ResPair.getValue(R: 0), DL, VT: ResVT);
12893}
12894
12895// Lower signed 3-way compare producing -1/0/1.
12896// For PowerPC with SETB (P9+), use the default expansion.
12897// For pre-P9, we generate a custom sequence using comparison and bit
12898// extraction.
12899// cmpw cr7, LHS, RHS
12900// mfocrf r, CR7
12901// rlwinm LT, r, 29, 31, 31 (extract LT bit from CR7)
12902// rlwinm GT, r, 30, 31, 31 (extract GT bit from CR7)
12903// subf result, LT, GT (GT - LT = -1/0/1)
12904SDValue PPCTargetLowering::LowerSCMP(SDValue Op, SelectionDAG &DAG) const {
12905 SDLoc DL(Op);
12906 SDValue LHS = Op.getOperand(i: 0);
12907 SDValue RHS = Op.getOperand(i: 1);
12908 EVT VT = LHS.getValueType();
12909 EVT ResVT = Op.getValueType();
12910
12911 // For P9+, use the default expansion which will use SETB instruction
12912 if (Subtarget.isISA3_0())
12913 return SDValue();
12914
12915 // For pre-P9, generate custom sequence without SETB
12916 bool Is64BitCmp = (VT == MVT::i64);
12917
12918 // On PPC64, always use 64-bit operations to avoid extra sign-extension
12919 // Even if result type is i32, it will likely be extended to i64 by calling
12920 // convention
12921 bool Use64BitOps = Subtarget.isPPC64();
12922
12923 // Perform comparison
12924 unsigned CmpOpc = Is64BitCmp ? PPC::CMPD : PPC::CMPW;
12925 SDValue Cmp = SDValue(DAG.getMachineNode(Opcode: CmpOpc, dl: DL, VT: MVT::i32, Op1: LHS, Op2: RHS), 0);
12926
12927 // Move comparison result to CR7 to have known bit positions
12928 SDValue CR7Reg = DAG.getRegister(Reg: PPC::CR7, VT: MVT::i32);
12929 SDValue InGlue; // Null incoming glue
12930 SDValue CopyToReg =
12931 DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: DL, Reg: CR7Reg, N: Cmp, Glue: InGlue);
12932 SDValue Glue = CopyToReg.getValue(R: 1);
12933
12934 // Use MFOCRF to read CR7
12935 unsigned MFOCRFOpc = Use64BitOps ? PPC::MFOCRF8 : PPC::MFOCRF;
12936 EVT MFVT = Use64BitOps ? MVT::i64 : MVT::i32;
12937 SDValue MFOCRF =
12938 SDValue(DAG.getMachineNode(Opcode: MFOCRFOpc, dl: DL, VT: MFVT, Op1: CR7Reg, Op2: Glue), 0);
12939
12940 // CR7 bits in the result (after MFOCRF):
12941 // Bit 28: LT
12942 // Bit 29: GT
12943 // Bit 30: EQ
12944 // Bit 31: SO
12945
12946 // Extract LT and GT bits using RLWINM
12947 unsigned RLWinmOpc = Use64BitOps ? PPC::RLWINM8 : PPC::RLWINM;
12948
12949 // Extract LT bit (Bit 28 -> Rotate Left 29 to move to bit 31, then mask)
12950 SDValue LTOps[] = {MFOCRF, DAG.getTargetConstant(Val: 29, DL, VT: MVT::i32),
12951 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32),
12952 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32)};
12953 SDValue LTBit = SDValue(DAG.getMachineNode(Opcode: RLWinmOpc, dl: DL, VT: MFVT, Ops: LTOps), 0);
12954
12955 // Extract GT bit (Bit 29 -> Rotate Left 30 to move to bit 31, then mask)
12956 SDValue GTOps[] = {MFOCRF, DAG.getTargetConstant(Val: 30, DL, VT: MVT::i32),
12957 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32),
12958 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32)};
12959 SDValue GTBit = SDValue(DAG.getMachineNode(Opcode: RLWinmOpc, dl: DL, VT: MFVT, Ops: GTOps), 0);
12960
12961 // Compute result: GT - LT
12962 // If LT: 0 - 1 = -1
12963 // If GT: 1 - 0 = 1
12964 // If EQ: 0 - 0 = 0
12965 unsigned SubOpc = Use64BitOps ? PPC::SUBF8 : PPC::SUBF;
12966 SDValue Result =
12967 SDValue(DAG.getMachineNode(Opcode: SubOpc, dl: DL, VT: MFVT, Op1: LTBit, Op2: GTBit), 0);
12968
12969 if (ResVT == MFVT)
12970 return Result;
12971
12972 // Adjust result to match expected type (sign-extend or truncate as needed)
12973 return DAG.getSExtOrTrunc(Op: Result, DL, VT: ResVT);
12974}
12975
12976/// LowerOperation - Provide custom lowering hooks for some operations.
12977///
12978SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
12979 switch (Op.getOpcode()) {
12980 default:
12981 llvm_unreachable("Wasn't expecting to be able to lower this!");
12982 case ISD::FPOW: return lowerPow(Op, DAG);
12983 case ISD::FSIN: return lowerSin(Op, DAG);
12984 case ISD::FCOS: return lowerCos(Op, DAG);
12985 case ISD::FLOG: return lowerLog(Op, DAG);
12986 case ISD::FLOG10: return lowerLog10(Op, DAG);
12987 case ISD::FEXP: return lowerExp(Op, DAG);
12988 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
12989 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
12990 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
12991 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
12992 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
12993 case ISD::STRICT_FSETCC:
12994 case ISD::STRICT_FSETCCS:
12995 case ISD::SETCC: return LowerSETCC(Op, DAG);
12996 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
12997 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
12998 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
12999 case ISD::SSUBO:
13000 return LowerSSUBO(Op, DAG);
13001 case ISD::SADDO:
13002 return LowerSADDO(Op, DAG);
13003
13004 case ISD::INLINEASM:
13005 case ISD::INLINEASM_BR: return LowerINLINEASM(Op, DAG);
13006 // Variable argument lowering.
13007 case ISD::VASTART: return LowerVASTART(Op, DAG);
13008 case ISD::VAARG: return LowerVAARG(Op, DAG);
13009 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
13010
13011 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG);
13012 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
13013 case ISD::GET_DYNAMIC_AREA_OFFSET:
13014 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
13015
13016 // Exception handling lowering.
13017 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG);
13018 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
13019 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
13020
13021 case ISD::LOAD: return LowerLOAD(Op, DAG);
13022 case ISD::STORE: return LowerSTORE(Op, DAG);
13023 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
13024 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
13025 case ISD::STRICT_FP_TO_UINT:
13026 case ISD::STRICT_FP_TO_SINT:
13027 case ISD::FP_TO_UINT:
13028 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, dl: SDLoc(Op));
13029 case ISD::STRICT_UINT_TO_FP:
13030 case ISD::STRICT_SINT_TO_FP:
13031 case ISD::UINT_TO_FP:
13032 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
13033 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
13034 case ISD::SET_ROUNDING:
13035 return LowerSET_ROUNDING(Op, DAG);
13036
13037 // Lower 64-bit shifts.
13038 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG);
13039 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG);
13040 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG);
13041
13042 case ISD::FSHL: return LowerFunnelShift(Op, DAG);
13043 case ISD::FSHR: return LowerFunnelShift(Op, DAG);
13044
13045 // Vector-related lowering.
13046 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
13047 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
13048 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
13049 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
13050 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
13051 case ISD::MUL: return LowerMUL(Op, DAG);
13052 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
13053 case ISD::STRICT_FP_ROUND:
13054 case ISD::FP_ROUND:
13055 return LowerFP_ROUND(Op, DAG);
13056 case ISD::ROTL: return LowerROTL(Op, DAG);
13057
13058 // For counter-based loop handling.
13059 case ISD::INTRINSIC_W_CHAIN:
13060 return SDValue();
13061
13062 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
13063
13064 // Frame & Return address.
13065 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
13066 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
13067
13068 case ISD::INTRINSIC_VOID:
13069 return LowerINTRINSIC_VOID(Op, DAG);
13070 case ISD::BSWAP:
13071 return LowerBSWAP(Op, DAG);
13072 case ISD::ATOMIC_CMP_SWAP:
13073 return LowerATOMIC_CMP_SWAP(Op, DAG);
13074 case ISD::ATOMIC_STORE:
13075 return LowerATOMIC_LOAD_STORE(Op, DAG);
13076 case ISD::IS_FPCLASS:
13077 return LowerIS_FPCLASS(Op, DAG);
13078 case ISD::UADDO:
13079 case ISD::USUBO:
13080 return LowerADDSUBO(Op, DAG);
13081 case ISD::UADDO_CARRY:
13082 case ISD::USUBO_CARRY:
13083 return LowerADDSUBO_CARRY(Op, DAG);
13084 case ISD::UCMP:
13085 return LowerUCMP(Op, DAG);
13086 case ISD::ABDU:
13087 return LowerABDU(Op, DAG);
13088 case ISD::SCMP:
13089 return LowerSCMP(Op, DAG);
13090 case ISD::STRICT_LRINT:
13091 case ISD::STRICT_LLRINT:
13092 case ISD::STRICT_LROUND:
13093 case ISD::STRICT_LLROUND:
13094 case ISD::STRICT_FNEARBYINT:
13095 if (Op->getFlags().hasNoFPExcept())
13096 return Op;
13097 return SDValue();
13098 case ISD::VP_LOAD:
13099 return LowerVP_LOAD(Op, DAG);
13100 case ISD::VP_STORE:
13101 return LowerVP_STORE(Op, DAG);
13102 case ISD::PARTIAL_REDUCE_UMLA:
13103 return LowerPartialReduce(Op, DAG);
13104 }
13105}
13106
13107void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
13108 SmallVectorImpl<SDValue>&Results,
13109 SelectionDAG &DAG) const {
13110 SDLoc dl(N);
13111 switch (N->getOpcode()) {
13112 default:
13113 llvm_unreachable("Do not know how to custom type legalize this operation!");
13114 case ISD::ATOMIC_LOAD: {
13115 SDValue Res = LowerATOMIC_LOAD_STORE(Op: SDValue(N, 0), DAG);
13116 Results.push_back(Elt: Res);
13117 Results.push_back(Elt: Res.getValue(R: 1));
13118 break;
13119 }
13120 case ISD::READCYCLECOUNTER: {
13121 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other);
13122 SDValue RTB = DAG.getNode(Opcode: PPCISD::READ_TIME_BASE, DL: dl, VTList: VTs, N: N->getOperand(Num: 0));
13123
13124 Results.push_back(
13125 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: RTB, N2: RTB.getValue(R: 1)));
13126 Results.push_back(Elt: RTB.getValue(R: 2));
13127 break;
13128 }
13129 case ISD::INTRINSIC_W_CHAIN: {
13130 if (N->getConstantOperandVal(Num: 1) != Intrinsic::loop_decrement)
13131 break;
13132
13133 assert(N->getValueType(0) == MVT::i1 &&
13134 "Unexpected result type for CTR decrement intrinsic");
13135 EVT SVT = getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(),
13136 VT: N->getValueType(ResNo: 0));
13137 SDVTList VTs = DAG.getVTList(VT1: SVT, VT2: MVT::Other);
13138 SDValue NewInt = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: VTs, N1: N->getOperand(Num: 0),
13139 N2: N->getOperand(Num: 1));
13140
13141 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewInt));
13142 Results.push_back(Elt: NewInt.getValue(R: 1));
13143 break;
13144 }
13145 case ISD::INTRINSIC_WO_CHAIN: {
13146 switch (N->getConstantOperandVal(Num: 0)) {
13147 case Intrinsic::ppc_pack_longdouble:
13148 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::ppcf128,
13149 N1: N->getOperand(Num: 2), N2: N->getOperand(Num: 1)));
13150 break;
13151 case Intrinsic::ppc_maxfe:
13152 case Intrinsic::ppc_minfe:
13153 case Intrinsic::ppc_fnmsub:
13154 case Intrinsic::ppc_convert_f128_to_ppcf128:
13155 Results.push_back(Elt: LowerINTRINSIC_WO_CHAIN(Op: SDValue(N, 0), DAG));
13156 break;
13157 }
13158 break;
13159 }
13160 case ISD::VAARG: {
13161 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64())
13162 return;
13163
13164 EVT VT = N->getValueType(ResNo: 0);
13165
13166 if (VT == MVT::i64) {
13167 SDValue NewNode = LowerVAARG(Op: SDValue(N, 1), DAG);
13168
13169 Results.push_back(Elt: NewNode);
13170 Results.push_back(Elt: NewNode.getValue(R: 1));
13171 }
13172 return;
13173 }
13174 case ISD::STRICT_FP_TO_SINT:
13175 case ISD::STRICT_FP_TO_UINT:
13176 case ISD::FP_TO_SINT:
13177 case ISD::FP_TO_UINT: {
13178 // LowerFP_TO_INT() can only handle f32 and f64.
13179 if (N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType() ==
13180 MVT::ppcf128)
13181 return;
13182 SDValue LoweredValue = LowerFP_TO_INT(Op: SDValue(N, 0), DAG, dl);
13183 Results.push_back(Elt: LoweredValue);
13184 if (N->isStrictFPOpcode())
13185 Results.push_back(Elt: LoweredValue.getValue(R: 1));
13186 return;
13187 }
13188 case ISD::TRUNCATE: {
13189 if (!N->getValueType(ResNo: 0).isVector())
13190 return;
13191 SDValue Lowered = LowerTRUNCATEVector(Op: SDValue(N, 0), DAG);
13192 if (Lowered)
13193 Results.push_back(Elt: Lowered);
13194 return;
13195 }
13196 case ISD::SCALAR_TO_VECTOR: {
13197 SDValue Lowered = LowerSCALAR_TO_VECTOR(Op: SDValue(N, 0), DAG);
13198 if (Lowered)
13199 Results.push_back(Elt: Lowered);
13200 return;
13201 }
13202 case ISD::FSHL:
13203 case ISD::FSHR:
13204 // Don't handle funnel shifts here.
13205 return;
13206 case ISD::BITCAST:
13207 // Don't handle bitcast here.
13208 return;
13209 case ISD::FP_EXTEND:
13210 SDValue Lowered = LowerFP_EXTEND(Op: SDValue(N, 0), DAG);
13211 if (Lowered)
13212 Results.push_back(Elt: Lowered);
13213 return;
13214 }
13215}
13216
13217//===----------------------------------------------------------------------===//
13218// Other Lowering Code
13219//===----------------------------------------------------------------------===//
13220
13221static CallInst *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
13222 return Builder.CreateIntrinsicWithoutFolding(ID: Id, Args: {});
13223}
13224
13225Value *PPCTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
13226 Value *Addr,
13227 AtomicOrdering Ord) const {
13228 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
13229
13230 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13231 "Only 8/16/32/64-bit atomic loads supported");
13232 Intrinsic::ID IntID;
13233 switch (SZ) {
13234 default:
13235 llvm_unreachable("Unexpected PrimitiveSize");
13236 case 8:
13237 IntID = Intrinsic::ppc_lbarx;
13238 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13239 break;
13240 case 16:
13241 IntID = Intrinsic::ppc_lharx;
13242 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13243 break;
13244 case 32:
13245 IntID = Intrinsic::ppc_lwarx;
13246 break;
13247 case 64:
13248 IntID = Intrinsic::ppc_ldarx;
13249 break;
13250 }
13251 Value *Call =
13252 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
13253
13254 return Builder.CreateTruncOrBitCast(V: Call, DestTy: ValueTy);
13255}
13256
13257// Perform a store-conditional operation to Addr. Return the status of the
13258// store. This should be 0 if the store succeeded, non-zero otherwise.
13259Value *PPCTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
13260 Value *Val, Value *Addr,
13261 AtomicOrdering Ord) const {
13262 Type *Ty = Val->getType();
13263 unsigned SZ = Ty->getPrimitiveSizeInBits();
13264
13265 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13266 "Only 8/16/32/64-bit atomic loads supported");
13267 Intrinsic::ID IntID;
13268 switch (SZ) {
13269 default:
13270 llvm_unreachable("Unexpected PrimitiveSize");
13271 case 8:
13272 IntID = Intrinsic::ppc_stbcx;
13273 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13274 break;
13275 case 16:
13276 IntID = Intrinsic::ppc_sthcx;
13277 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13278 break;
13279 case 32:
13280 IntID = Intrinsic::ppc_stwcx;
13281 break;
13282 case 64:
13283 IntID = Intrinsic::ppc_stdcx;
13284 break;
13285 }
13286
13287 if (SZ == 8 || SZ == 16)
13288 Val = Builder.CreateZExt(V: Val, DestTy: Builder.getInt32Ty());
13289
13290 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
13291 /*FMFSource=*/nullptr, Name: "stcx");
13292 return Builder.CreateXor(LHS: Call, RHS: Builder.getInt32(C: 1));
13293}
13294
13295// The mappings for emitLeading/TrailingFence is taken from
13296// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
13297Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
13298 Instruction *Inst,
13299 AtomicOrdering Ord) const {
13300 if (Ord == AtomicOrdering::SequentiallyConsistent)
13301 return callIntrinsic(Builder, Id: Intrinsic::ppc_sync);
13302 if (isReleaseOrStronger(AO: Ord))
13303 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13304 return nullptr;
13305}
13306
13307Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
13308 Instruction *Inst,
13309 AtomicOrdering Ord) const {
13310 if (Inst->hasAtomicLoad() && isAcquireOrStronger(AO: Ord)) {
13311 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and
13312 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
13313 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
13314 if (isa<LoadInst>(Val: Inst))
13315 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::ppc_cfence,
13316 OverloadTypes: {Inst->getType()}, Args: {Inst});
13317 // FIXME: Can use isync for rmw operation.
13318 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13319 }
13320 return nullptr;
13321}
13322
13323MachineBasicBlock *PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI,
13324 MachineBasicBlock *BB,
13325 unsigned BinOpcode,
13326 unsigned CmpOpcode,
13327 unsigned CmpPred) const {
13328 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13329 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13330 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13331 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13332 unsigned AtomicSize = MI.getOperand(i: 3).getImm();
13333
13334 auto LoadMnemonic = PPC::LDARX;
13335 auto StoreMnemonic = PPC::STDCX;
13336 switch (AtomicSize) {
13337 default:
13338 llvm_unreachable("Unexpected size of atomic entity");
13339 case 1:
13340 LoadMnemonic = PPC::LBARX;
13341 StoreMnemonic = PPC::STBCX;
13342 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13343 break;
13344 case 2:
13345 LoadMnemonic = PPC::LHARX;
13346 StoreMnemonic = PPC::STHCX;
13347 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13348 break;
13349 case 4:
13350 LoadMnemonic = PPC::LWARX;
13351 StoreMnemonic = PPC::STWCX;
13352 break;
13353 case 8:
13354 LoadMnemonic = PPC::LDARX;
13355 StoreMnemonic = PPC::STDCX;
13356 break;
13357 }
13358
13359 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13360 MachineFunction *F = BB->getParent();
13361 MachineFunction::iterator It = ++BB->getIterator();
13362
13363 if (CmpOpcode == PPC::CMPW && (AtomicSize == 1 || AtomicSize == 2))
13364 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, /*IsByte=*/AtomicSize == 1, TII);
13365
13366 Register dest = MI.getOperand(i: 0).getReg();
13367 Register ptrA = MI.getOperand(i: 1).getReg();
13368 Register ptrB = MI.getOperand(i: 2).getReg();
13369 Register incr = MI.getOperand(i: 4).getReg();
13370 DebugLoc dl = MI.getDebugLoc();
13371
13372 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13373 MachineBasicBlock *loop2MBB =
13374 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13375 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13376 F->insert(MBBI: It, MBB: loopMBB);
13377 if (CmpOpcode)
13378 F->insert(MBBI: It, MBB: loop2MBB);
13379 F->insert(MBBI: It, MBB: exitMBB);
13380 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13381 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13382 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13383
13384 MachineRegisterInfo &RegInfo = F->getRegInfo();
13385 Register TmpReg = (!BinOpcode) ? incr :
13386 RegInfo.createVirtualRegister( RegClass: AtomicSize == 8 ? &PPC::G8RCRegClass
13387 : &PPC::GPRCRegClass);
13388
13389 // thisMBB:
13390 // ...
13391 // fallthrough --> loopMBB
13392 BB->addSuccessor(Succ: loopMBB);
13393
13394 // loopMBB:
13395 // l[wd]arx dest, ptr
13396 // add r0, dest, incr
13397 // st[wd]cx. r0, ptr
13398 // bne- loopMBB
13399 // fallthrough --> exitMBB
13400
13401 // For max/min...
13402 // loopMBB:
13403 // l[wd]arx dest, ptr
13404 // cmpl?[wd] dest, incr
13405 // bgt exitMBB
13406 // loop2MBB:
13407 // st[wd]cx. dest, ptr
13408 // bne- loopMBB
13409 // fallthrough --> exitMBB
13410
13411 BB = loopMBB;
13412 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest)
13413 .addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13414 if (BinOpcode)
13415 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg).addReg(RegNo: incr).addReg(RegNo: dest);
13416 if (CmpOpcode) {
13417 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13418 // Signed comparisons of byte or halfword values must be sign-extended.
13419 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) {
13420 Register ExtReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13421 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH),
13422 DestReg: ExtReg).addReg(RegNo: dest);
13423 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ExtReg).addReg(RegNo: incr);
13424 } else
13425 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: dest).addReg(RegNo: incr);
13426
13427 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13428 .addImm(Val: CmpPred)
13429 .addReg(RegNo: CrReg)
13430 .addMBB(MBB: exitMBB);
13431 BB->addSuccessor(Succ: loop2MBB);
13432 BB->addSuccessor(Succ: exitMBB);
13433 BB = loop2MBB;
13434 }
13435 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
13436 .addReg(RegNo: TmpReg).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13437 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13438 .addImm(Val: PPC::PRED_NE_MINUS)
13439 .addReg(RegNo: PPC::CR0)
13440 .addMBB(MBB: loopMBB);
13441 BB->addSuccessor(Succ: loopMBB);
13442 BB->addSuccessor(Succ: exitMBB);
13443
13444 // exitMBB:
13445 // ...
13446 BB = exitMBB;
13447 return BB;
13448}
13449
13450static bool isSignExtended(MachineInstr &MI, const PPCInstrInfo *TII) {
13451 switch(MI.getOpcode()) {
13452 default:
13453 return false;
13454 case PPC::COPY:
13455 return TII->isSignExtended(Reg: MI.getOperand(i: 1).getReg(),
13456 MRI: &MI.getMF()->getRegInfo());
13457 case PPC::LHA:
13458 case PPC::LHA8:
13459 case PPC::LHAU:
13460 case PPC::LHAU8:
13461 case PPC::LHAUX:
13462 case PPC::LHAUX8:
13463 case PPC::LHAX:
13464 case PPC::LHAX8:
13465 case PPC::LWA:
13466 case PPC::LWAUX:
13467 case PPC::LWAX:
13468 case PPC::LWAX_32:
13469 case PPC::LWA_32:
13470 case PPC::PLHA:
13471 case PPC::PLHA8:
13472 case PPC::PLHA8pc:
13473 case PPC::PLHApc:
13474 case PPC::PLWA:
13475 case PPC::PLWA8:
13476 case PPC::PLWA8pc:
13477 case PPC::PLWApc:
13478 case PPC::EXTSB:
13479 case PPC::EXTSB8:
13480 case PPC::EXTSB8_32_64:
13481 case PPC::EXTSB8_rec:
13482 case PPC::EXTSB_rec:
13483 case PPC::EXTSH:
13484 case PPC::EXTSH8:
13485 case PPC::EXTSH8_32_64:
13486 case PPC::EXTSH8_rec:
13487 case PPC::EXTSH_rec:
13488 case PPC::EXTSW:
13489 case PPC::EXTSWSLI:
13490 case PPC::EXTSWSLI_32_64:
13491 case PPC::EXTSWSLI_32_64_rec:
13492 case PPC::EXTSWSLI_rec:
13493 case PPC::EXTSW_32:
13494 case PPC::EXTSW_32_64:
13495 case PPC::EXTSW_32_64_rec:
13496 case PPC::EXTSW_rec:
13497 case PPC::SRAW:
13498 case PPC::SRAWI:
13499 case PPC::SRAWI_rec:
13500 case PPC::SRAW_rec:
13501 return true;
13502 }
13503 return false;
13504}
13505
13506// Sign extend operand OpIdx if the value is not known to be sign extended.
13507// Assumes the operand is a register. The flag IsByte controls which intruction
13508// is used for the sign extension.
13509static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
13510 unsigned OpIdx, bool IsByte,
13511 const PPCInstrInfo *TII) {
13512 MachineFunction *F = MI.getMF();
13513 MachineRegisterInfo &RegInfo = F->getRegInfo();
13514 Register Reg = MI.getOperand(i: OpIdx).getReg();
13515 bool IsSignExtended =
13516 Reg.isVirtual() && isSignExtended(MI&: *RegInfo.getVRegDef(Reg), TII);
13517
13518 if (!IsSignExtended) {
13519 Register ValueReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13520 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(),
13521 MCID: TII->get(Opcode: IsByte ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueReg)
13522 .addReg(RegNo: Reg);
13523 MI.getOperand(i: OpIdx).setReg(ValueReg);
13524 }
13525}
13526
13527MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
13528 MachineInstr &MI, MachineBasicBlock *BB, unsigned BinOpcode,
13529 unsigned CmpOpcode, unsigned CmpPred) const {
13530 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13531 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13532 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13533 assert(!Subtarget.hasPartwordAtomics() &&
13534 "Assumes that part-word atomics are not available");
13535 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13536
13537 // If this is a signed comparison and the value being compared is not known
13538 // to be sign extended, sign extend it here.
13539 DebugLoc dl = MI.getDebugLoc();
13540 MachineFunction *F = BB->getParent();
13541 MachineRegisterInfo &RegInfo = F->getRegInfo();
13542 const bool is8bit = MI.getOperand(i: 3).getImm() == 1;
13543 if (CmpOpcode == PPC::CMPW)
13544 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, IsByte: is8bit, TII);
13545 Register incr = MI.getOperand(i: 4).getReg();
13546
13547 // In 64 bit mode we have to use 64 bits for addresses, even though the
13548 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address
13549 // registers without caring whether they're 32 or 64, but here we're
13550 // doing actual arithmetic on the addresses.
13551 bool is64bit = Subtarget.isPPC64();
13552 bool isLittleEndian = Subtarget.isLittleEndian();
13553 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
13554
13555 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13556 MachineFunction::iterator It = ++BB->getIterator();
13557
13558 Register dest = MI.getOperand(i: 0).getReg();
13559 Register ptrA = MI.getOperand(i: 1).getReg();
13560 Register ptrB = MI.getOperand(i: 2).getReg();
13561
13562 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13563 MachineBasicBlock *loop2MBB =
13564 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13565 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13566 F->insert(MBBI: It, MBB: loopMBB);
13567 if (CmpOpcode)
13568 F->insert(MBBI: It, MBB: loop2MBB);
13569 F->insert(MBBI: It, MBB: exitMBB);
13570 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13571 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13572 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13573
13574 const TargetRegisterClass *RC =
13575 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13576 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
13577
13578 Register PtrReg = RegInfo.createVirtualRegister(RegClass: RC);
13579 Register Shift1Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13580 Register ShiftReg =
13581 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13582 Register Incr2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13583 Register MaskReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13584 Register Mask2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13585 Register Mask3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13586 Register Tmp2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13587 Register Tmp3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13588 Register Tmp4Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13589 Register TmpDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13590 Register SrwDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13591 Register Ptr1Reg;
13592 Register TmpReg =
13593 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13594
13595 // thisMBB:
13596 // ...
13597 // fallthrough --> loopMBB
13598 BB->addSuccessor(Succ: loopMBB);
13599
13600 // The 4-byte load must be aligned, while a char or short may be
13601 // anywhere in the word. Hence all this nasty bookkeeping code.
13602 // add ptr1, ptrA, ptrB [copy if ptrA==0]
13603 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
13604 // xori shift, shift1, 24 [16]
13605 // rlwinm ptr, ptr1, 0, 0, 29
13606 // slw incr2, incr, shift
13607 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
13608 // slw mask, mask2, shift
13609 // loopMBB:
13610 // lwarx tmpDest, ptr
13611 // add tmp, tmpDest, incr2
13612 // andc tmp2, tmpDest, mask
13613 // and tmp3, tmp, mask
13614 // or tmp4, tmp3, tmp2
13615 // stwcx. tmp4, ptr
13616 // bne- loopMBB
13617 // fallthrough --> exitMBB
13618 // srw SrwDest, tmpDest, shift
13619 // rlwinm SrwDest, SrwDest, 0, 24 [16], 31
13620 if (ptrA != ZeroReg) {
13621 Ptr1Reg = RegInfo.createVirtualRegister(RegClass: RC);
13622 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
13623 .addReg(RegNo: ptrA)
13624 .addReg(RegNo: ptrB);
13625 } else {
13626 Ptr1Reg = ptrB;
13627 }
13628 // We need use 32-bit subregister to avoid mismatch register class in 64-bit
13629 // mode.
13630 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
13631 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
13632 .addImm(Val: 3)
13633 .addImm(Val: 27)
13634 .addImm(Val: is8bit ? 28 : 27);
13635 if (!isLittleEndian)
13636 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
13637 .addReg(RegNo: Shift1Reg)
13638 .addImm(Val: is8bit ? 24 : 16);
13639 if (is64bit)
13640 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
13641 .addReg(RegNo: Ptr1Reg)
13642 .addImm(Val: 0)
13643 .addImm(Val: 61);
13644 else
13645 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
13646 .addReg(RegNo: Ptr1Reg)
13647 .addImm(Val: 0)
13648 .addImm(Val: 0)
13649 .addImm(Val: 29);
13650 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: Incr2Reg).addReg(RegNo: incr).addReg(RegNo: ShiftReg);
13651 if (is8bit)
13652 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
13653 else {
13654 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
13655 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
13656 .addReg(RegNo: Mask3Reg)
13657 .addImm(Val: 65535);
13658 }
13659 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
13660 .addReg(RegNo: Mask2Reg)
13661 .addReg(RegNo: ShiftReg);
13662
13663 BB = loopMBB;
13664 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
13665 .addReg(RegNo: ZeroReg)
13666 .addReg(RegNo: PtrReg);
13667 if (BinOpcode)
13668 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg)
13669 .addReg(RegNo: Incr2Reg)
13670 .addReg(RegNo: TmpDestReg);
13671 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
13672 .addReg(RegNo: TmpDestReg)
13673 .addReg(RegNo: MaskReg);
13674 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: Tmp3Reg).addReg(RegNo: TmpReg).addReg(RegNo: MaskReg);
13675 if (CmpOpcode) {
13676 // For unsigned comparisons, we can directly compare the shifted values.
13677 // For signed comparisons we shift and sign extend.
13678 Register SReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13679 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13680 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: SReg)
13681 .addReg(RegNo: TmpDestReg)
13682 .addReg(RegNo: MaskReg);
13683 unsigned ValueReg = SReg;
13684 unsigned CmpReg = Incr2Reg;
13685 if (CmpOpcode == PPC::CMPW) {
13686 ValueReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13687 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: ValueReg)
13688 .addReg(RegNo: SReg)
13689 .addReg(RegNo: ShiftReg);
13690 Register ValueSReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13691 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is8bit ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueSReg)
13692 .addReg(RegNo: ValueReg);
13693 ValueReg = ValueSReg;
13694 CmpReg = incr;
13695 }
13696 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ValueReg).addReg(RegNo: CmpReg);
13697 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13698 .addImm(Val: CmpPred)
13699 .addReg(RegNo: CrReg)
13700 .addMBB(MBB: exitMBB);
13701 BB->addSuccessor(Succ: loop2MBB);
13702 BB->addSuccessor(Succ: exitMBB);
13703 BB = loop2MBB;
13704 }
13705 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg).addReg(RegNo: Tmp3Reg).addReg(RegNo: Tmp2Reg);
13706 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
13707 .addReg(RegNo: Tmp4Reg)
13708 .addReg(RegNo: ZeroReg)
13709 .addReg(RegNo: PtrReg);
13710 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13711 .addImm(Val: PPC::PRED_NE_MINUS)
13712 .addReg(RegNo: PPC::CR0)
13713 .addMBB(MBB: loopMBB);
13714 BB->addSuccessor(Succ: loopMBB);
13715 BB->addSuccessor(Succ: exitMBB);
13716
13717 // exitMBB:
13718 // ...
13719 BB = exitMBB;
13720 // Since the shift amount is not a constant, we need to clear
13721 // the upper bits with a separate RLWINM.
13722 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: dest)
13723 .addReg(RegNo: SrwDestReg)
13724 .addImm(Val: 0)
13725 .addImm(Val: is8bit ? 24 : 16)
13726 .addImm(Val: 31);
13727 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: SrwDestReg)
13728 .addReg(RegNo: TmpDestReg)
13729 .addReg(RegNo: ShiftReg);
13730 return BB;
13731}
13732
13733llvm::MachineBasicBlock *
13734PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
13735 MachineBasicBlock *MBB) const {
13736 DebugLoc DL = MI.getDebugLoc();
13737 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13738 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo();
13739
13740 MachineFunction *MF = MBB->getParent();
13741 MachineRegisterInfo &MRI = MF->getRegInfo();
13742
13743 const BasicBlock *BB = MBB->getBasicBlock();
13744 MachineFunction::iterator I = ++MBB->getIterator();
13745
13746 Register DstReg = MI.getOperand(i: 0).getReg();
13747 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
13748 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
13749 Register mainDstReg = MRI.createVirtualRegister(RegClass: RC);
13750 Register restoreDstReg = MRI.createVirtualRegister(RegClass: RC);
13751
13752 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13753 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13754 "Invalid Pointer Size!");
13755 // For v = setjmp(buf), we generate
13756 //
13757 // thisMBB:
13758 // SjLjSetup mainMBB
13759 // bl mainMBB
13760 // v_restore = 1
13761 // b sinkMBB
13762 //
13763 // mainMBB:
13764 // buf[LabelOffset] = LR
13765 // v_main = 0
13766 //
13767 // sinkMBB:
13768 // v = phi(main, restore)
13769 //
13770
13771 MachineBasicBlock *thisMBB = MBB;
13772 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
13773 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
13774 MF->insert(MBBI: I, MBB: mainMBB);
13775 MF->insert(MBBI: I, MBB: sinkMBB);
13776
13777 MachineInstrBuilder MIB;
13778
13779 // Transfer the remainder of BB and its successor edges to sinkMBB.
13780 sinkMBB->splice(Where: sinkMBB->begin(), Other: MBB,
13781 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
13782 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
13783
13784 // Note that the structure of the jmp_buf used here is not compatible
13785 // with that used by libc, and is not designed to be. Specifically, it
13786 // stores only those 'reserved' registers that LLVM does not otherwise
13787 // understand how to spill. Also, by convention, by the time this
13788 // intrinsic is called, Clang has already stored the frame address in the
13789 // first slot of the buffer and stack address in the third. Following the
13790 // X86 target code, we'll store the jump address in the second slot. We also
13791 // need to save the TOC pointer (R2) to handle jumps between shared
13792 // libraries, and that will be stored in the fourth slot. The thread
13793 // identifier (R13) is not affected.
13794
13795 // thisMBB:
13796 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13797 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13798 const int64_t BPOffset = 4 * PVT.getStoreSize();
13799
13800 // Prepare IP either in reg.
13801 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
13802 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
13803 Register BufReg = MI.getOperand(i: 1).getReg();
13804
13805 if (Subtarget.is64BitELFABI()) {
13806 setUsesTOCBasePtr(*MBB->getParent());
13807 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13808 .addReg(RegNo: PPC::X2)
13809 .addImm(Val: TOCOffset)
13810 .addReg(RegNo: BufReg)
13811 .cloneMemRefs(OtherMI: MI);
13812 }
13813
13814 // Naked functions never have a base pointer, and so we use r1. For all
13815 // other functions, this decision must be delayed until during PEI.
13816 unsigned BaseReg;
13817 if (MF->getFunction().hasFnAttribute(Kind: Attribute::Naked))
13818 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
13819 else
13820 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP;
13821
13822 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL,
13823 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::STD : PPC::STW))
13824 .addReg(RegNo: BaseReg)
13825 .addImm(Val: BPOffset)
13826 .addReg(RegNo: BufReg)
13827 .cloneMemRefs(OtherMI: MI);
13828
13829 // Setup
13830 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::BCLalways)).addMBB(MBB: mainMBB);
13831 MIB.addRegMask(Mask: TRI->getNoPreservedMask());
13832
13833 BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: restoreDstReg).addImm(Val: 1);
13834
13835 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::EH_SjLj_Setup))
13836 .addMBB(MBB: mainMBB);
13837 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: sinkMBB);
13838
13839 thisMBB->addSuccessor(Succ: mainMBB, Prob: BranchProbability::getZero());
13840 thisMBB->addSuccessor(Succ: sinkMBB, Prob: BranchProbability::getOne());
13841
13842 // mainMBB:
13843 // mainDstReg = 0
13844 MIB =
13845 BuildMI(BB: mainMBB, MIMD: DL,
13846 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), DestReg: LabelReg);
13847
13848 // Store IP
13849 if (Subtarget.isPPC64()) {
13850 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13851 .addReg(RegNo: LabelReg)
13852 .addImm(Val: LabelOffset)
13853 .addReg(RegNo: BufReg);
13854 } else {
13855 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STW))
13856 .addReg(RegNo: LabelReg)
13857 .addImm(Val: LabelOffset)
13858 .addReg(RegNo: BufReg);
13859 }
13860 MIB.cloneMemRefs(OtherMI: MI);
13861
13862 BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: mainDstReg).addImm(Val: 0);
13863 mainMBB->addSuccessor(Succ: sinkMBB);
13864
13865 // sinkMBB:
13866 BuildMI(BB&: *sinkMBB, I: sinkMBB->begin(), MIMD: DL,
13867 MCID: TII->get(Opcode: PPC::PHI), DestReg: DstReg)
13868 .addReg(RegNo: mainDstReg).addMBB(MBB: mainMBB)
13869 .addReg(RegNo: restoreDstReg).addMBB(MBB: thisMBB);
13870
13871 MI.eraseFromParent();
13872 return sinkMBB;
13873}
13874
13875MachineBasicBlock *
13876PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
13877 MachineBasicBlock *MBB) const {
13878 DebugLoc DL = MI.getDebugLoc();
13879 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13880
13881 MachineFunction *MF = MBB->getParent();
13882 MachineRegisterInfo &MRI = MF->getRegInfo();
13883
13884 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13885 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13886 "Invalid Pointer Size!");
13887
13888 const TargetRegisterClass *RC =
13889 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13890 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
13891 // Since FP is only updated here but NOT referenced, it's treated as GPR.
13892 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31;
13893 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1;
13894 unsigned BP =
13895 (PVT == MVT::i64)
13896 ? PPC::X30
13897 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29
13898 : PPC::R30);
13899
13900 MachineInstrBuilder MIB;
13901
13902 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13903 const int64_t SPOffset = 2 * PVT.getStoreSize();
13904 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13905 const int64_t BPOffset = 4 * PVT.getStoreSize();
13906
13907 Register BufReg = MI.getOperand(i: 0).getReg();
13908
13909 // Reload FP (the jumped-to function may not have had a
13910 // frame pointer, and if so, then its r31 will be restored
13911 // as necessary).
13912 if (PVT == MVT::i64) {
13913 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: FP)
13914 .addImm(Val: 0)
13915 .addReg(RegNo: BufReg);
13916 } else {
13917 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: FP)
13918 .addImm(Val: 0)
13919 .addReg(RegNo: BufReg);
13920 }
13921 MIB.cloneMemRefs(OtherMI: MI);
13922
13923 // Reload IP
13924 if (PVT == MVT::i64) {
13925 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: Tmp)
13926 .addImm(Val: LabelOffset)
13927 .addReg(RegNo: BufReg);
13928 } else {
13929 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: Tmp)
13930 .addImm(Val: LabelOffset)
13931 .addReg(RegNo: BufReg);
13932 }
13933 MIB.cloneMemRefs(OtherMI: MI);
13934
13935 // Reload SP
13936 if (PVT == MVT::i64) {
13937 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: SP)
13938 .addImm(Val: SPOffset)
13939 .addReg(RegNo: BufReg);
13940 } else {
13941 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: SP)
13942 .addImm(Val: SPOffset)
13943 .addReg(RegNo: BufReg);
13944 }
13945 MIB.cloneMemRefs(OtherMI: MI);
13946
13947 // Reload BP
13948 if (PVT == MVT::i64) {
13949 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: BP)
13950 .addImm(Val: BPOffset)
13951 .addReg(RegNo: BufReg);
13952 } else {
13953 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: BP)
13954 .addImm(Val: BPOffset)
13955 .addReg(RegNo: BufReg);
13956 }
13957 MIB.cloneMemRefs(OtherMI: MI);
13958
13959 // Reload TOC
13960 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) {
13961 setUsesTOCBasePtr(*MBB->getParent());
13962 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: PPC::X2)
13963 .addImm(Val: TOCOffset)
13964 .addReg(RegNo: BufReg)
13965 .cloneMemRefs(OtherMI: MI);
13966 }
13967
13968 // Jump
13969 BuildMI(BB&: *MBB, I&: MI, MIMD: DL,
13970 MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(RegNo: Tmp);
13971 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR));
13972
13973 MI.eraseFromParent();
13974 return MBB;
13975}
13976
13977bool PPCTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
13978 // If the function specifically requests inline stack probes, emit them.
13979 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
13980 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
13981 "inline-asm";
13982 return false;
13983}
13984
13985unsigned PPCTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
13986 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
13987 unsigned StackAlign = TFI->getStackAlignment();
13988 assert(StackAlign >= 1 && isPowerOf2_32(StackAlign) &&
13989 "Unexpected stack alignment");
13990 // The default stack probe size is 4096 if the function has no
13991 // stack-probe-size attribute.
13992 const Function &Fn = MF.getFunction();
13993 unsigned StackProbeSize =
13994 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
13995 // Round down to the stack alignment.
13996 StackProbeSize &= ~(StackAlign - 1);
13997 return StackProbeSize ? StackProbeSize : StackAlign;
13998}
13999
14000// Lower dynamic stack allocation with probing. `emitProbedAlloca` is splitted
14001// into three phases. In the first phase, it uses pseudo instruction
14002// PREPARE_PROBED_ALLOCA to get the future result of actual FramePointer and
14003// FinalStackPtr. In the second phase, it generates a loop for probing blocks.
14004// At last, it uses pseudo instruction DYNAREAOFFSET to get the future result of
14005// MaxCallFrameSize so that it can calculate correct data area pointer.
14006MachineBasicBlock *
14007PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
14008 MachineBasicBlock *MBB) const {
14009 const bool isPPC64 = Subtarget.isPPC64();
14010 MachineFunction *MF = MBB->getParent();
14011 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
14012 DebugLoc DL = MI.getDebugLoc();
14013 const unsigned ProbeSize = getStackProbeSize(MF: *MF);
14014 const BasicBlock *ProbedBB = MBB->getBasicBlock();
14015 MachineRegisterInfo &MRI = MF->getRegInfo();
14016 // The CFG of probing stack looks as
14017 // +-----+
14018 // | MBB |
14019 // +--+--+
14020 // |
14021 // +----v----+
14022 // +--->+ TestMBB +---+
14023 // | +----+----+ |
14024 // | | |
14025 // | +-----v----+ |
14026 // +---+ BlockMBB | |
14027 // +----------+ |
14028 // |
14029 // +---------+ |
14030 // | TailMBB +<--+
14031 // +---------+
14032 // In MBB, calculate previous frame pointer and final stack pointer.
14033 // In TestMBB, test if sp is equal to final stack pointer, if so, jump to
14034 // TailMBB. In BlockMBB, update the sp atomically and jump back to TestMBB.
14035 // TailMBB is spliced via \p MI.
14036 MachineBasicBlock *TestMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14037 MachineBasicBlock *TailMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14038 MachineBasicBlock *BlockMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14039
14040 MachineFunction::iterator MBBIter = ++MBB->getIterator();
14041 MF->insert(MBBI: MBBIter, MBB: TestMBB);
14042 MF->insert(MBBI: MBBIter, MBB: BlockMBB);
14043 MF->insert(MBBI: MBBIter, MBB: TailMBB);
14044
14045 const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
14046 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
14047
14048 Register DstReg = MI.getOperand(i: 0).getReg();
14049 Register NegSizeReg = MI.getOperand(i: 1).getReg();
14050 Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
14051 Register FinalStackPtr = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14052 Register FramePointer = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14053 Register ActualNegSizeReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14054
14055 // Since value of NegSizeReg might be realigned in prologepilog, insert a
14056 // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and
14057 // NegSize.
14058 unsigned ProbeOpc;
14059 if (!MRI.hasOneNonDBGUse(RegNo: NegSizeReg))
14060 ProbeOpc =
14061 isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32;
14062 else
14063 // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg
14064 // and NegSizeReg will be allocated in the same phyreg to avoid
14065 // redundant copy when NegSizeReg has only one use which is current MI and
14066 // will be replaced by PREPARE_PROBED_ALLOCA then.
14067 ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64
14068 : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32;
14069 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: ProbeOpc), DestReg: FramePointer)
14070 .addDef(RegNo: ActualNegSizeReg)
14071 .addReg(RegNo: NegSizeReg)
14072 .add(MO: MI.getOperand(i: 2))
14073 .add(MO: MI.getOperand(i: 3));
14074
14075 // Calculate final stack pointer, which equals to SP + ActualNegSize.
14076 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4),
14077 DestReg: FinalStackPtr)
14078 .addReg(RegNo: SPReg)
14079 .addReg(RegNo: ActualNegSizeReg);
14080
14081 // Materialize a scratch register for update.
14082 int64_t NegProbeSize = -(int64_t)ProbeSize;
14083 assert(isInt<32>(NegProbeSize) && "Unhandled probe size!");
14084 Register ScratchReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14085 if (!isInt<16>(x: NegProbeSize)) {
14086 Register TempReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14087 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LIS8 : PPC::LIS), DestReg: TempReg)
14088 .addImm(Val: NegProbeSize >> 16);
14089 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ORI8 : PPC::ORI),
14090 DestReg: ScratchReg)
14091 .addReg(RegNo: TempReg)
14092 .addImm(Val: NegProbeSize & 0xFFFF);
14093 } else
14094 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LI8 : PPC::LI), DestReg: ScratchReg)
14095 .addImm(Val: NegProbeSize);
14096
14097 {
14098 // Probing leading residual part.
14099 Register Div = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14100 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::DIVD : PPC::DIVW), DestReg: Div)
14101 .addReg(RegNo: ActualNegSizeReg)
14102 .addReg(RegNo: ScratchReg);
14103 Register Mul = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14104 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::MULLD : PPC::MULLW), DestReg: Mul)
14105 .addReg(RegNo: Div)
14106 .addReg(RegNo: ScratchReg);
14107 Register NegMod = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14108 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::SUBF8 : PPC::SUBF), DestReg: NegMod)
14109 .addReg(RegNo: Mul)
14110 .addReg(RegNo: ActualNegSizeReg);
14111 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14112 .addReg(RegNo: FramePointer)
14113 .addReg(RegNo: SPReg)
14114 .addReg(RegNo: NegMod);
14115 }
14116
14117 {
14118 // Remaining part should be multiple of ProbeSize.
14119 Register CmpResult = MRI.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14120 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::CMPD : PPC::CMPW), DestReg: CmpResult)
14121 .addReg(RegNo: SPReg)
14122 .addReg(RegNo: FinalStackPtr);
14123 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::BCC))
14124 .addImm(Val: PPC::PRED_EQ)
14125 .addReg(RegNo: CmpResult)
14126 .addMBB(MBB: TailMBB);
14127 TestMBB->addSuccessor(Succ: BlockMBB);
14128 TestMBB->addSuccessor(Succ: TailMBB);
14129 }
14130
14131 {
14132 // Touch the block.
14133 // |P...|P...|P...
14134 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14135 .addReg(RegNo: FramePointer)
14136 .addReg(RegNo: SPReg)
14137 .addReg(RegNo: ScratchReg);
14138 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: TestMBB);
14139 BlockMBB->addSuccessor(Succ: TestMBB);
14140 }
14141
14142 // Calculation of MaxCallFrameSize is deferred to prologepilog, use
14143 // DYNAREAOFFSET pseudo instruction to get the future result.
14144 Register MaxCallFrameSizeReg =
14145 MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14146 BuildMI(BB: TailMBB, MIMD: DL,
14147 MCID: TII->get(Opcode: isPPC64 ? PPC::DYNAREAOFFSET8 : PPC::DYNAREAOFFSET),
14148 DestReg: MaxCallFrameSizeReg)
14149 .add(MO: MI.getOperand(i: 2))
14150 .add(MO: MI.getOperand(i: 3));
14151 BuildMI(BB: TailMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4), DestReg: DstReg)
14152 .addReg(RegNo: SPReg)
14153 .addReg(RegNo: MaxCallFrameSizeReg);
14154
14155 // Splice instructions after MI to TailMBB.
14156 TailMBB->splice(Where: TailMBB->end(), Other: MBB,
14157 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
14158 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
14159 MBB->addSuccessor(Succ: TestMBB);
14160
14161 // Delete the pseudo instruction.
14162 MI.eraseFromParent();
14163
14164 ++NumDynamicAllocaProbed;
14165 return TailMBB;
14166}
14167
14168/// Check if the opcode is a SELECT or SELECT_CC variant.
14169/// @param Opcode The opcode to check
14170/// @param CheckOnlyCC If true, only return true for SELECT_CC variants;
14171/// if false, return true for both SELECT and SELECT_CC
14172static bool IsSelect(unsigned Opcode, bool CheckOnlyCC = false) {
14173 switch (Opcode) {
14174 // SELECT_CC variants - always return true
14175 case PPC::SELECT_CC_I4:
14176 case PPC::SELECT_CC_I8:
14177 case PPC::SELECT_CC_F4:
14178 case PPC::SELECT_CC_F8:
14179 case PPC::SELECT_CC_F16:
14180 case PPC::SELECT_CC_VRRC:
14181 case PPC::SELECT_CC_VSFRC:
14182 case PPC::SELECT_CC_VSSRC:
14183 case PPC::SELECT_CC_VSRC:
14184 case PPC::SELECT_CC_SPE4:
14185 case PPC::SELECT_CC_SPE:
14186 return true;
14187 // SELECT variants - only return true if CheckOnlyCC is false
14188 case PPC::SELECT_I4:
14189 case PPC::SELECT_I8:
14190 case PPC::SELECT_F4:
14191 case PPC::SELECT_F8:
14192 case PPC::SELECT_F16:
14193 case PPC::SELECT_SPE:
14194 case PPC::SELECT_SPE4:
14195 case PPC::SELECT_VRRC:
14196 case PPC::SELECT_VSFRC:
14197 case PPC::SELECT_VSSRC:
14198 case PPC::SELECT_VSRC:
14199 return !CheckOnlyCC; // true if checking all SELECTs, false if only CC
14200 default:
14201 return false;
14202 }
14203}
14204static bool IsSelectCC(unsigned Opcode) { return IsSelect(Opcode, CheckOnlyCC: true); }
14205
14206/// Emit SELECT instruction, using ISEL if available, otherwise use
14207/// branch-based control flow.
14208///
14209/// For targets with ISEL support (SELECT_CC_I4/I8, SELECT_I4/I8), this
14210/// generates a single ISEL instruction. Otherwise, it creates a
14211/// branch-based control flow pattern with PHI nodes.
14212static MachineBasicBlock *emitSelect(MachineInstr &MI, MachineBasicBlock *BB,
14213 const TargetInstrInfo *TII,
14214 const PPCSubtarget &Subtarget) {
14215 assert(IsSelect(MI.getOpcode()) && "Instruction must be a SELECT variant");
14216
14217 // Check if we can use ISEL for this SELECT
14218 if (Subtarget.hasISEL() &&
14219 (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14220 MI.getOpcode() == PPC::SELECT_CC_I8 ||
14221 MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8)) {
14222 SmallVector<MachineOperand, 2> Cond;
14223 if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14224 MI.getOpcode() == PPC::SELECT_CC_I8)
14225 Cond.push_back(Elt: MI.getOperand(i: 4));
14226 else
14227 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
14228 Cond.push_back(Elt: MI.getOperand(i: 1));
14229
14230 DebugLoc dl = MI.getDebugLoc();
14231 TII->insertSelect(MBB&: *BB, I: MI, DL: dl, DstReg: MI.getOperand(i: 0).getReg(), Cond,
14232 TrueReg: MI.getOperand(i: 2).getReg(), FalseReg: MI.getOperand(i: 3).getReg());
14233 MI.eraseFromParent();
14234 return BB;
14235 }
14236
14237 // Fall back to branch-based SELECT implementation
14238 MachineFunction *F = BB->getParent();
14239 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14240 MachineFunction::iterator It = ++BB->getIterator();
14241 DebugLoc dl = MI.getDebugLoc();
14242
14243 MachineBasicBlock *thisMBB = BB;
14244 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14245 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14246 F->insert(MBBI: It, MBB: copy0MBB);
14247 F->insert(MBBI: It, MBB: sinkMBB);
14248
14249 if (isPhysRegUsedAfter(Reg: PPC::CARRY, MBI: MI.getIterator())) {
14250 copy0MBB->addLiveIn(PhysReg: PPC::CARRY);
14251 sinkMBB->addLiveIn(PhysReg: PPC::CARRY);
14252 }
14253
14254 // Set the call frame size on entry to the new basic blocks.
14255 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
14256 copy0MBB->setCallFrameSize(CallFrameSize);
14257 sinkMBB->setCallFrameSize(CallFrameSize);
14258
14259 // Transfer the remainder of BB and its successor edges to sinkMBB.
14260 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14261 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14262 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14263
14264 // Add successors
14265 BB->addSuccessor(Succ: copy0MBB);
14266 BB->addSuccessor(Succ: sinkMBB);
14267
14268 // Build branch instruction
14269 if (IsSelectCC(Opcode: MI.getOpcode()))
14270 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14271 .addImm(Val: MI.getOperand(i: 4).getImm())
14272 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14273 .addMBB(MBB: sinkMBB);
14274 else
14275 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BC))
14276 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14277 .addMBB(MBB: sinkMBB);
14278
14279 // copy0MBB: fallthrough to sinkMBB
14280 BB = copy0MBB;
14281 BB->addSuccessor(Succ: sinkMBB);
14282
14283 // sinkMBB: PHI instruction
14284 BB = sinkMBB;
14285 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::PHI), DestReg: MI.getOperand(i: 0).getReg())
14286 .addReg(RegNo: MI.getOperand(i: 3).getReg())
14287 .addMBB(MBB: copy0MBB)
14288 .addReg(RegNo: MI.getOperand(i: 2).getReg())
14289 .addMBB(MBB: thisMBB);
14290 MI.eraseFromParent();
14291 return BB;
14292}
14293
14294/// Helper function to create basic blocks for atomic compare-and-swap.
14295/// Creates three basic blocks (loop1MBB, loop2MBB, exitMBB) and sets up
14296/// the control flow structure common to both hardware and software
14297/// implementations of atomic compare-and-swap operations.
14298static void createAtomicLoopBlocks(MachineFunction *F, MachineBasicBlock *BB,
14299 MachineBasicBlock *&loop1MBB,
14300 MachineBasicBlock *&loop2MBB,
14301 MachineBasicBlock *&exitMBB,
14302 MachineInstr &MI,
14303 MachineFunction::iterator It) {
14304 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14305 loop1MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14306 loop2MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14307 exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14308 F->insert(MBBI: It, MBB: loop1MBB);
14309 F->insert(MBBI: It, MBB: loop2MBB);
14310 F->insert(MBBI: It, MBB: exitMBB);
14311 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
14312 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14313 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14314 BB->addSuccessor(Succ: loop1MBB);
14315}
14316
14317/// Emit hardware-supported atomic compare-and-swap for I32/I64 and I8/I16
14318/// with partword atomic support.
14319///
14320/// This uses native PowerPC atomic instructions (LBARX/LHARX/LWARX/LDARX for
14321/// load-and-reserve, STBCX/STHCX/STWCX/STDCX for store-conditional) to
14322/// implement atomic compare-and-swap at byte, halfword, word, or doubleword
14323/// granularity.
14324///
14325/// Control flow:
14326/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14327/// | |
14328/// +------------+
14329///
14330/// loop1MBB:
14331/// - Load-and-reserve from memory
14332/// - Compare loaded value with expected old value
14333/// - Branch to exitMBB if not equal (CAS failed)
14334/// loop2MBB:
14335/// - Store-conditional new value to memory
14336/// - Branch back to loop1MBB if store failed (retry)
14337/// - Fall through to exitMBB on success
14338static MachineBasicBlock *
14339emitAtomicCmpSwapHardware(MachineInstr &MI, MachineBasicBlock *BB,
14340 const TargetInstrInfo *TII,
14341 const PPCSubtarget &Subtarget) {
14342 MachineFunction *F = BB->getParent();
14343 MachineFunction::iterator It = ++BB->getIterator();
14344
14345 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64;
14346
14347 unsigned LoadMnemonic = PPC::LDARX;
14348 unsigned StoreMnemonic = PPC::STDCX;
14349 switch (MI.getOpcode()) {
14350 default:
14351 llvm_unreachable("Compare and swap of unknown size");
14352 case PPC::ATOMIC_CMP_SWAP_I8:
14353 LoadMnemonic = PPC::LBARX;
14354 StoreMnemonic = PPC::STBCX;
14355 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14356 break;
14357 case PPC::ATOMIC_CMP_SWAP_I16:
14358 LoadMnemonic = PPC::LHARX;
14359 StoreMnemonic = PPC::STHCX;
14360 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14361 break;
14362 case PPC::ATOMIC_CMP_SWAP_I32:
14363 LoadMnemonic = PPC::LWARX;
14364 StoreMnemonic = PPC::STWCX;
14365 break;
14366 case PPC::ATOMIC_CMP_SWAP_I64:
14367 LoadMnemonic = PPC::LDARX;
14368 StoreMnemonic = PPC::STDCX;
14369 break;
14370 }
14371
14372 MachineRegisterInfo &RegInfo = F->getRegInfo();
14373 Register dest = MI.getOperand(i: 0).getReg();
14374 Register ptrA = MI.getOperand(i: 1).getReg();
14375 Register ptrB = MI.getOperand(i: 2).getReg();
14376 Register oldval = MI.getOperand(i: 3).getReg();
14377 Register newval = MI.getOperand(i: 4).getReg();
14378 DebugLoc dl = MI.getDebugLoc();
14379
14380 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14381 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14382
14383 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14384
14385 // loop1MBB:
14386 // l[bhwd]arx dest, ptr
14387 // cmp[wd] dest, oldval
14388 // bne- exitBB
14389 BB = loop1MBB;
14390 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
14391 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::CMPD : PPC::CMPW), DestReg: CrReg)
14392 .addReg(RegNo: dest)
14393 .addReg(RegNo: oldval);
14394 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14395 .addImm(Val: PPC::PRED_NE_MINUS)
14396 .addReg(RegNo: CrReg)
14397 .addMBB(MBB: exitMBB);
14398 BB->addSuccessor(Succ: loop2MBB);
14399 BB->addSuccessor(Succ: exitMBB);
14400
14401 // loop2MBB:
14402 // st[bhwd]cx. newval, ptr
14403 // bne- loopMBB
14404 // b exitBB
14405 BB = loop2MBB;
14406 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
14407 .addReg(RegNo: newval)
14408 .addReg(RegNo: ptrA)
14409 .addReg(RegNo: ptrB);
14410 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14411 .addImm(Val: PPC::PRED_NE_MINUS)
14412 .addReg(RegNo: PPC::CR0)
14413 .addMBB(MBB: loop1MBB);
14414 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14415 BB->addSuccessor(Succ: loop1MBB);
14416 BB->addSuccessor(Succ: exitMBB);
14417
14418 return exitMBB;
14419}
14420
14421/// Emit software-emulated atomic compare-and-swap for I8/I16 without
14422/// hardware partword atomic support.
14423///
14424/// This emulates byte/halfword atomic operations using word (32-bit) atomic
14425/// instructions. Since PowerPC atomic instructions work at word granularity,
14426/// we must:
14427/// 1. Align the pointer to a word boundary
14428/// 2. Calculate the bit shift for the target byte/halfword within the word
14429/// 3. Create masks to isolate the target byte/halfword
14430/// 4. Shift old/new values into the correct bit position
14431/// 5. Use LWARX/STWCX on the full word
14432/// 6. Mask and merge to preserve other bytes in the word
14433/// 7. Extract and shift the result back
14434///
14435/// Control flow:
14436/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14437/// | |
14438/// +------------+
14439///
14440/// loop1MBB:
14441/// - LWARX: Load-and-reserve full word
14442/// - Mask to extract target byte/halfword
14443/// - Compare with expected old value
14444/// - Branch to exitMBB if not equal (CAS failed)
14445/// loop2MBB:
14446/// - Merge new value with other bytes in the word
14447/// - STWCX: Store-conditional full word
14448/// - Branch back to loop1MBB if store failed (retry)
14449/// - Fall through to exitMBB on success
14450/// exitMBB:
14451/// - Extract and return the loaded value
14452static MachineBasicBlock *
14453emitAtomicCmpSwapSoftware(MachineInstr &MI, MachineBasicBlock *BB,
14454 const TargetInstrInfo *TII,
14455 const PPCSubtarget &Subtarget) {
14456 MachineFunction *F = BB->getParent();
14457 MachineFunction::iterator It = ++BB->getIterator();
14458
14459 bool is64bit = Subtarget.isPPC64();
14460 bool isLittleEndian = Subtarget.isLittleEndian();
14461 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8;
14462
14463 Register dest = MI.getOperand(i: 0).getReg();
14464 Register ptrA = MI.getOperand(i: 1).getReg();
14465 Register ptrB = MI.getOperand(i: 2).getReg();
14466 Register oldval = MI.getOperand(i: 3).getReg();
14467 Register newval = MI.getOperand(i: 4).getReg();
14468 DebugLoc dl = MI.getDebugLoc();
14469
14470 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14471 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14472
14473 MachineRegisterInfo &RegInfo = F->getRegInfo();
14474 const TargetRegisterClass *RC =
14475 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
14476 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
14477
14478 // Lambda to create virtual registers
14479 auto createVReg = [&](const TargetRegisterClass *RC) {
14480 return RegInfo.createVirtualRegister(RegClass: RC);
14481 };
14482
14483 Register PtrReg = createVReg(RC);
14484 Register Shift1Reg = createVReg(GPRC);
14485 Register ShiftReg = isLittleEndian ? Shift1Reg : createVReg(GPRC);
14486 Register NewVal2Reg = createVReg(GPRC);
14487 Register NewVal3Reg = createVReg(GPRC);
14488 Register OldVal2Reg = createVReg(GPRC);
14489 Register OldVal3Reg = createVReg(GPRC);
14490 Register MaskReg = createVReg(GPRC);
14491 Register Mask2Reg = createVReg(GPRC);
14492 Register Mask3Reg = createVReg(GPRC);
14493 Register Tmp2Reg = createVReg(GPRC);
14494 Register Tmp4Reg = createVReg(GPRC);
14495 Register TmpDestReg = createVReg(GPRC);
14496 Register TmpReg = createVReg(GPRC);
14497 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
14498 Register CrReg = createVReg(&PPC::CRRCRegClass);
14499
14500 // Compute aligned pointer and shift amount
14501 Register Ptr1Reg;
14502 if (ptrA != ZeroReg) {
14503 Ptr1Reg = createVReg(RC);
14504 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
14505 .addReg(RegNo: ptrA)
14506 .addReg(RegNo: ptrB);
14507 } else {
14508 Ptr1Reg = ptrB;
14509 }
14510
14511 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
14512 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
14513 .addImm(Val: 3)
14514 .addImm(Val: 27)
14515 .addImm(Val: is8bit ? 28 : 27);
14516 if (!isLittleEndian)
14517 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
14518 .addReg(RegNo: Shift1Reg)
14519 .addImm(Val: is8bit ? 24 : 16);
14520 if (is64bit)
14521 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
14522 .addReg(RegNo: Ptr1Reg)
14523 .addImm(Val: 0)
14524 .addImm(Val: 61);
14525 else
14526 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
14527 .addReg(RegNo: Ptr1Reg)
14528 .addImm(Val: 0)
14529 .addImm(Val: 0)
14530 .addImm(Val: 29);
14531
14532 // Prepare masked values
14533 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: NewVal2Reg)
14534 .addReg(RegNo: newval)
14535 .addReg(RegNo: ShiftReg);
14536 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: OldVal2Reg)
14537 .addReg(RegNo: oldval)
14538 .addReg(RegNo: ShiftReg);
14539 if (is8bit)
14540 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
14541 else {
14542 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
14543 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
14544 .addReg(RegNo: Mask3Reg)
14545 .addImm(Val: 65535);
14546 }
14547 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
14548 .addReg(RegNo: Mask2Reg)
14549 .addReg(RegNo: ShiftReg);
14550 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: NewVal3Reg)
14551 .addReg(RegNo: NewVal2Reg)
14552 .addReg(RegNo: MaskReg);
14553 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: OldVal3Reg)
14554 .addReg(RegNo: OldVal2Reg)
14555 .addReg(RegNo: MaskReg);
14556
14557 // loop1MBB:
14558 // lwarx tmpDest, ptr
14559 // and tmp, tmpDest, mask
14560 // cmpw tmp, oldval3
14561 // bne- exitBB
14562 BB = loop1MBB;
14563 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
14564 .addReg(RegNo: ZeroReg)
14565 .addReg(RegNo: PtrReg);
14566 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: TmpReg)
14567 .addReg(RegNo: TmpDestReg)
14568 .addReg(RegNo: MaskReg);
14569 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CrReg).addReg(RegNo: TmpReg).addReg(RegNo: OldVal3Reg);
14570 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14571 .addImm(Val: PPC::PRED_NE)
14572 .addReg(RegNo: CrReg)
14573 .addMBB(MBB: exitMBB);
14574 BB->addSuccessor(Succ: loop2MBB);
14575 BB->addSuccessor(Succ: exitMBB);
14576
14577 // loop2MBB:
14578 // andc tmp2, tmpDest, mask
14579 // or tmp4, tmp2, newval3
14580 // stwcx. tmp4, ptr
14581 // bne- loop1MBB
14582 // b exitBB
14583 BB = loop2MBB;
14584 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
14585 .addReg(RegNo: TmpDestReg)
14586 .addReg(RegNo: MaskReg);
14587 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg)
14588 .addReg(RegNo: Tmp2Reg)
14589 .addReg(RegNo: NewVal3Reg);
14590 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
14591 .addReg(RegNo: Tmp4Reg)
14592 .addReg(RegNo: ZeroReg)
14593 .addReg(RegNo: PtrReg);
14594 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14595 .addImm(Val: PPC::PRED_NE)
14596 .addReg(RegNo: PPC::CR0)
14597 .addMBB(MBB: loop1MBB);
14598 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14599 BB->addSuccessor(Succ: loop1MBB);
14600 BB->addSuccessor(Succ: exitMBB);
14601
14602 // exitMBB:
14603 // srw dest, tmpDest, shift
14604 BB = exitMBB;
14605 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: dest)
14606 .addReg(RegNo: TmpReg)
14607 .addReg(RegNo: ShiftReg);
14608
14609 return BB;
14610}
14611
14612MachineBasicBlock *
14613PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
14614 MachineBasicBlock *BB) const {
14615 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
14616
14617 // To "insert" these instructions we actually have to insert their
14618 // control-flow patterns.
14619 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14620 MachineFunction::iterator It = ++BB->getIterator();
14621
14622 MachineFunction *F = BB->getParent();
14623 MachineRegisterInfo &MRI = F->getRegInfo();
14624
14625 // Handle SELECT with ISEL support first (before generic SELECT handling)
14626 if (IsSelect(Opcode: MI.getOpcode()))
14627 return emitSelect(MI, BB, TII, Subtarget);
14628
14629 switch (MI.getOpcode()) {
14630 case TargetOpcode::STACKMAP:
14631 return emitPatchPoint(MI, MBB: BB);
14632 case TargetOpcode::PATCHPOINT:
14633 // Call lowering should have added an r2 operand to indicate a dependence
14634 // on the TOC base pointer value. It can't however, because there is no
14635 // way to mark the dependence as implicit there, and so the stackmap code
14636 // will confuse it with a regular operand. Instead, add the dependence
14637 // here.
14638 if (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls())
14639 MI.addOperand(Op: MachineOperand::CreateReg(Reg: PPC::X2, isDef: false, isImp: true));
14640 return emitPatchPoint(MI, MBB: BB);
14641
14642 case PPC::EH_SjLj_SetJmp32:
14643 case PPC::EH_SjLj_SetJmp64:
14644 return emitEHSjLjSetJmp(MI, MBB: BB);
14645
14646 case PPC::EH_SjLj_LongJmp32:
14647 case PPC::EH_SjLj_LongJmp64:
14648 return emitEHSjLjLongJmp(MI, MBB: BB);
14649
14650 case PPC::ReadTB: {
14651 // To read the 64-bit time-base register on a 32-bit target, we read the
14652 // two halves. Should the counter have wrapped while it was being read, we
14653 // need to try again.
14654 // ...
14655 // readLoop:
14656 // mfspr Rx,TBU # load from TBU
14657 // mfspr Ry,TB # load from TB
14658 // mfspr Rz,TBU # load from TBU
14659 // cmpw crX,Rx,Rz # check if 'old'='new'
14660 // bne readLoop # branch if they're not equal
14661 // ...
14662
14663 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14664 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14665 DebugLoc dl = MI.getDebugLoc();
14666 F->insert(MBBI: It, MBB: readMBB);
14667 F->insert(MBBI: It, MBB: sinkMBB);
14668
14669 // Transfer the remainder of BB and its successor edges to sinkMBB.
14670 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14671 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14672 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14673
14674 BB->addSuccessor(Succ: readMBB);
14675 BB = readMBB;
14676
14677 MachineRegisterInfo &RegInfo = F->getRegInfo();
14678 Register ReadAgainReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
14679 Register LoReg = MI.getOperand(i: 0).getReg();
14680 Register HiReg = MI.getOperand(i: 1).getReg();
14681
14682 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: HiReg).addImm(Val: 269);
14683 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: LoReg).addImm(Val: 268);
14684 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: ReadAgainReg).addImm(Val: 269);
14685
14686 Register CmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14687
14688 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CmpReg)
14689 .addReg(RegNo: HiReg)
14690 .addReg(RegNo: ReadAgainReg);
14691 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14692 .addImm(Val: PPC::PRED_NE)
14693 .addReg(RegNo: CmpReg)
14694 .addMBB(MBB: readMBB);
14695
14696 BB->addSuccessor(Succ: readMBB);
14697 BB->addSuccessor(Succ: sinkMBB);
14698 break;
14699 }
14700 case PPC::ATOMIC_LOAD_ADD_NOWP:
14701 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14702 break;
14703 case PPC::ATOMIC_LOAD_ADD:
14704 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14705 break;
14706 case PPC::ATOMIC_LOAD_ADD_I64:
14707 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD8);
14708 break;
14709 case PPC::ATOMIC_LOAD_AND_NOWP:
14710 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14711 break;
14712 case PPC::ATOMIC_LOAD_AND:
14713 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14714 break;
14715 case PPC::ATOMIC_LOAD_AND_I64:
14716 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND8);
14717 break;
14718 case PPC::ATOMIC_LOAD_OR_NOWP:
14719 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14720 break;
14721 case PPC::ATOMIC_LOAD_OR:
14722 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14723 break;
14724 case PPC::ATOMIC_LOAD_OR_I64:
14725 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR8);
14726 break;
14727 case PPC::ATOMIC_LOAD_XOR_NOWP:
14728 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14729 break;
14730 case PPC::ATOMIC_LOAD_XOR:
14731 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14732 break;
14733 case PPC::ATOMIC_LOAD_XOR_I64:
14734 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR8);
14735 break;
14736 case PPC::ATOMIC_LOAD_NAND_NOWP:
14737 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14738 break;
14739 case PPC::ATOMIC_LOAD_NAND:
14740 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14741 break;
14742 case PPC::ATOMIC_LOAD_NAND_I64:
14743 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND8);
14744 break;
14745 case PPC::ATOMIC_LOAD_SUB_NOWP:
14746 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14747 break;
14748 case PPC::ATOMIC_LOAD_SUB:
14749 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14750 break;
14751 case PPC::ATOMIC_LOAD_SUB_I64:
14752 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF8);
14753 break;
14754 case PPC::ATOMIC_LOAD_MIN_NOWP:
14755 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14756 break;
14757 case PPC::ATOMIC_LOAD_MIN:
14758 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14759 break;
14760 case PPC::ATOMIC_LOAD_MIN_I64:
14761 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_LT);
14762 break;
14763 case PPC::ATOMIC_LOAD_MAX_NOWP:
14764 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14765 break;
14766 case PPC::ATOMIC_LOAD_MAX:
14767 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14768 break;
14769 case PPC::ATOMIC_LOAD_MAX_I64:
14770 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_GT);
14771 break;
14772 case PPC::ATOMIC_LOAD_UMIN_NOWP:
14773 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14774 break;
14775 case PPC::ATOMIC_LOAD_UMIN:
14776 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14777 break;
14778 case PPC::ATOMIC_LOAD_UMIN_I64:
14779 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_LT);
14780 break;
14781 case PPC::ATOMIC_LOAD_UMAX_NOWP:
14782 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14783 break;
14784 case PPC::ATOMIC_LOAD_UMAX:
14785 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14786 break;
14787 case PPC::ATOMIC_LOAD_UMAX_I64:
14788 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_GT);
14789 break;
14790 case PPC::ATOMIC_SWAP_NOWP:
14791 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0);
14792 break;
14793 case PPC::ATOMIC_SWAP:
14794 case PPC::ATOMIC_SWAP_I64:
14795 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0);
14796 break;
14797 case PPC::ATOMIC_CMP_SWAP_I32:
14798 case PPC::ATOMIC_CMP_SWAP_I64:
14799 case PPC::ATOMIC_CMP_SWAP_I8:
14800 case PPC::ATOMIC_CMP_SWAP_I16: {
14801 // Use hardware-supported atomic operations if available
14802 bool useHardware = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
14803 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 ||
14804 (Subtarget.hasPartwordAtomics() &&
14805 (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 ||
14806 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16));
14807
14808 if (useHardware)
14809 BB = emitAtomicCmpSwapHardware(MI, BB, TII, Subtarget);
14810 else
14811 BB = emitAtomicCmpSwapSoftware(MI, BB, TII, Subtarget);
14812 break;
14813 }
14814 case PPC::FADDrtz: {
14815 // This pseudo performs an FADD with rounding mode temporarily forced
14816 // to round-to-zero. We emit this via custom inserter since the FPSCR
14817 // is not modeled at the SelectionDAG level.
14818 Register Dest = MI.getOperand(i: 0).getReg();
14819 Register Src1 = MI.getOperand(i: 1).getReg();
14820 Register Src2 = MI.getOperand(i: 2).getReg();
14821 DebugLoc dl = MI.getDebugLoc();
14822
14823 MachineRegisterInfo &RegInfo = F->getRegInfo();
14824 Register MFFSReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
14825
14826 // Save FPSCR value.
14827 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: MFFSReg);
14828
14829 // Set rounding mode to round-to-zero.
14830 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB1))
14831 .addImm(Val: 31)
14832 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14833
14834 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB0))
14835 .addImm(Val: 30)
14836 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14837
14838 // Perform addition.
14839 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::FADD), DestReg: Dest)
14840 .addReg(RegNo: Src1)
14841 .addReg(RegNo: Src2);
14842 if (MI.getFlag(Flag: MachineInstr::NoFPExcept))
14843 MIB.setMIFlag(MachineInstr::NoFPExcept);
14844
14845 // Restore FPSCR value.
14846 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSFb)).addImm(Val: 1).addReg(RegNo: MFFSReg);
14847 break;
14848 }
14849 case PPC::ANDI_rec_1_EQ_BIT:
14850 case PPC::ANDI_rec_1_GT_BIT:
14851 case PPC::ANDI_rec_1_EQ_BIT8:
14852 case PPC::ANDI_rec_1_GT_BIT8: {
14853 unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 ||
14854 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8)
14855 ? PPC::ANDI8_rec
14856 : PPC::ANDI_rec;
14857 bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT ||
14858 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8);
14859
14860 MachineRegisterInfo &RegInfo = F->getRegInfo();
14861 Register Dest = RegInfo.createVirtualRegister(
14862 RegClass: Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass);
14863
14864 DebugLoc Dl = MI.getDebugLoc();
14865 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode), DestReg: Dest)
14866 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14867 .addImm(Val: 1);
14868 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14869 DestReg: MI.getOperand(i: 0).getReg())
14870 .addReg(RegNo: IsEQ ? PPC::CR0EQ : PPC::CR0GT);
14871 break;
14872 }
14873 case PPC::TCHECK_RET: {
14874 DebugLoc Dl = MI.getDebugLoc();
14875 MachineRegisterInfo &RegInfo = F->getRegInfo();
14876 Register CRReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14877 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TCHECK), DestReg: CRReg);
14878 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14879 DestReg: MI.getOperand(i: 0).getReg())
14880 .addReg(RegNo: CRReg);
14881 break;
14882 }
14883 case PPC::TBEGIN_RET: {
14884 DebugLoc Dl = MI.getDebugLoc();
14885 unsigned Imm = MI.getOperand(i: 1).getImm();
14886 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TBEGIN)).addImm(Val: Imm);
14887 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14888 DestReg: MI.getOperand(i: 0).getReg())
14889 .addReg(RegNo: PPC::CR0EQ);
14890 break;
14891 }
14892 case PPC::SETRNDi: {
14893 DebugLoc dl = MI.getDebugLoc();
14894 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14895
14896 // Save FPSCR value.
14897 if (MRI.use_empty(RegNo: OldFPSCRReg))
14898 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
14899 else
14900 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14901
14902 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has
14903 // the following settings:
14904 // 00 Round to nearest
14905 // 01 Round to 0
14906 // 10 Round to +inf
14907 // 11 Round to -inf
14908
14909 // When the operand is immediate, using the two least significant bits of
14910 // the immediate to set the bits 62:63 of FPSCR.
14911 unsigned Mode = MI.getOperand(i: 1).getImm();
14912 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0))
14913 .addImm(Val: 31)
14914 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14915
14916 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0))
14917 .addImm(Val: 30)
14918 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14919 break;
14920 }
14921 case PPC::SETRND: {
14922 DebugLoc dl = MI.getDebugLoc();
14923
14924 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg
14925 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg.
14926 // If the target doesn't have DirectMove, we should use stack to do the
14927 // conversion, because the target doesn't have the instructions like mtvsrd
14928 // or mfvsrd to do this conversion directly.
14929 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) {
14930 if (Subtarget.hasDirectMove()) {
14931 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg)
14932 .addReg(RegNo: SrcReg);
14933 } else {
14934 // Use stack to do the register copy.
14935 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD;
14936 MachineRegisterInfo &RegInfo = F->getRegInfo();
14937 const TargetRegisterClass *RC = RegInfo.getRegClass(Reg: SrcReg);
14938 if (RC == &PPC::F8RCRegClass) {
14939 // Copy register from F8RCRegClass to G8RCRegclass.
14940 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) &&
14941 "Unsupported RegClass.");
14942
14943 StoreOp = PPC::STFD;
14944 LoadOp = PPC::LD;
14945 } else {
14946 // Copy register from G8RCRegClass to F8RCRegclass.
14947 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) &&
14948 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) &&
14949 "Unsupported RegClass.");
14950 }
14951
14952 MachineFrameInfo &MFI = F->getFrameInfo();
14953 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
14954
14955 MachineMemOperand *MMOStore = F->getMachineMemOperand(
14956 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14957 F: MachineMemOperand::MOStore, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14958 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14959
14960 // Store the SrcReg into the stack.
14961 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: StoreOp))
14962 .addReg(RegNo: SrcReg)
14963 .addImm(Val: 0)
14964 .addFrameIndex(Idx: FrameIdx)
14965 .addMemOperand(MMO: MMOStore);
14966
14967 MachineMemOperand *MMOLoad = F->getMachineMemOperand(
14968 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14969 F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14970 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14971
14972 // Load from the stack where SrcReg is stored, and save to DestReg,
14973 // so we have done the RegClass conversion from RegClass::SrcReg to
14974 // RegClass::DestReg.
14975 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: LoadOp), DestReg)
14976 .addImm(Val: 0)
14977 .addFrameIndex(Idx: FrameIdx)
14978 .addMemOperand(MMO: MMOLoad);
14979 }
14980 };
14981
14982 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14983
14984 // Save FPSCR value.
14985 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14986
14987 // When the operand is gprc register, use two least significant bits of the
14988 // register and mtfsf instruction to set the bits 62:63 of FPSCR.
14989 //
14990 // copy OldFPSCRTmpReg, OldFPSCRReg
14991 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1)
14992 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62
14993 // copy NewFPSCRReg, NewFPSCRTmpReg
14994 // mtfsf 255, NewFPSCRReg
14995 MachineOperand SrcOp = MI.getOperand(i: 1);
14996 MachineRegisterInfo &RegInfo = F->getRegInfo();
14997 Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14998
14999 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg);
15000
15001 Register ImDefReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
15002 Register ExtSrcReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
15003
15004 // The first operand of INSERT_SUBREG should be a register which has
15005 // subregisters, we only care about its RegClass, so we should use an
15006 // IMPLICIT_DEF register.
15007 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImDefReg);
15008 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::INSERT_SUBREG), DestReg: ExtSrcReg)
15009 .addReg(RegNo: ImDefReg)
15010 .add(MO: SrcOp)
15011 .addImm(Val: PPC::sub_32);
15012
15013 Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
15014 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDIMI), DestReg: NewFPSCRTmpReg)
15015 .addReg(RegNo: OldFPSCRTmpReg)
15016 .addReg(RegNo: ExtSrcReg)
15017 .addImm(Val: 0)
15018 .addImm(Val: 62);
15019
15020 Register NewFPSCRReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
15021 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg);
15022
15023 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63
15024 // bits of FPSCR.
15025 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSF))
15026 .addImm(Val: 255)
15027 .addReg(RegNo: NewFPSCRReg)
15028 .addImm(Val: 0)
15029 .addImm(Val: 0);
15030 break;
15031 }
15032 case PPC::SETFLM: {
15033 DebugLoc Dl = MI.getDebugLoc();
15034
15035 // Result of setflm is previous FPSCR content, so we need to save it first.
15036 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
15037 if (MRI.use_empty(RegNo: OldFPSCRReg))
15038 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
15039 else
15040 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
15041
15042 // Put bits in 32:63 to FPSCR.
15043 Register NewFPSCRReg = MI.getOperand(i: 1).getReg();
15044 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MTFSF))
15045 .addImm(Val: 255)
15046 .addReg(RegNo: NewFPSCRReg)
15047 .addImm(Val: 0)
15048 .addImm(Val: 0);
15049 break;
15050 }
15051 case PPC::PROBED_ALLOCA_32:
15052 case PPC::PROBED_ALLOCA_64:
15053 return emitProbedAlloca(MI, MBB: BB);
15054
15055 case PPC::SPLIT_QUADWORD: {
15056 DebugLoc DL = MI.getDebugLoc();
15057 Register Src = MI.getOperand(i: 2).getReg();
15058 Register Lo = MI.getOperand(i: 0).getReg();
15059 Register Hi = MI.getOperand(i: 1).getReg();
15060 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
15061 .addDef(RegNo: Lo)
15062 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x1);
15063 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
15064 .addDef(RegNo: Hi)
15065 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x0);
15066 break;
15067 }
15068 case PPC::LQX_PSEUDO:
15069 case PPC::STQX_PSEUDO: {
15070 DebugLoc DL = MI.getDebugLoc();
15071 // Ptr is used as the ptr_rc_no_r0 part
15072 // of LQ/STQ's memory operand and adding result of RA and RB,
15073 // so it has to be g8rc_and_g8rc_nox0.
15074 Register Ptr =
15075 F->getRegInfo().createVirtualRegister(RegClass: &PPC::G8RC_and_G8RC_NOX0RegClass);
15076 Register Val = MI.getOperand(i: 0).getReg();
15077 Register RA = MI.getOperand(i: 1).getReg();
15078 Register RB = MI.getOperand(i: 2).getReg();
15079 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::ADD8), DestReg: Ptr).addReg(RegNo: RA).addReg(RegNo: RB);
15080 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
15081 MCID: MI.getOpcode() == PPC::LQX_PSEUDO ? TII->get(Opcode: PPC::LQ)
15082 : TII->get(Opcode: PPC::STQ))
15083 .addReg(RegNo: Val, Flags: getDefRegState(B: MI.getOpcode() == PPC::LQX_PSEUDO))
15084 .addImm(Val: 0)
15085 .addReg(RegNo: Ptr);
15086 break;
15087 }
15088 default:
15089 llvm_unreachable("Unexpected instr type to insert");
15090 }
15091
15092 MI.eraseFromParent(); // The pseudo instruction is gone now.
15093 return BB;
15094}
15095
15096//===----------------------------------------------------------------------===//
15097// Target Optimization Hooks
15098//===----------------------------------------------------------------------===//
15099
15100static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
15101 // For the estimates, convergence is quadratic, so we essentially double the
15102 // number of digits correct after every iteration. For both FRE and FRSQRTE,
15103 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
15104 // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
15105 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
15106 if (VT.getScalarType() == MVT::f64)
15107 RefinementSteps++;
15108 return RefinementSteps;
15109}
15110
15111SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
15112 const DenormalMode &Mode,
15113 SDNodeFlags Flags) const {
15114 // We only have VSX Vector Test for software Square Root.
15115 EVT VT = Op.getValueType();
15116 if (!isTypeLegal(VT: MVT::i1) ||
15117 (VT != MVT::f64 &&
15118 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())))
15119 return TargetLowering::getSqrtInputTest(Operand: Op, DAG, Mode, Flags);
15120
15121 SDLoc DL(Op);
15122 // The output register of FTSQRT is CR field.
15123 SDValue FTSQRT = DAG.getNode(Opcode: PPCISD::FTSQRT, DL, VT: MVT::i32, Operand: Op, Flags);
15124 // ftsqrt BF,FRB
15125 // Let e_b be the unbiased exponent of the double-precision
15126 // floating-point operand in register FRB.
15127 // fe_flag is set to 1 if either of the following conditions occurs.
15128 // - The double-precision floating-point operand in register FRB is a zero,
15129 // a NaN, or an infinity, or a negative value.
15130 // - e_b is less than or equal to -970.
15131 // Otherwise fe_flag is set to 0.
15132 // Both VSX and non-VSX versions would set EQ bit in the CR if the number is
15133 // not eligible for iteration. (zero/negative/infinity/nan or unbiased
15134 // exponent is less than -970)
15135 SDValue SRIdxVal = DAG.getTargetConstant(Val: PPC::sub_eq, DL, VT: MVT::i32);
15136 return SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, VT: MVT::i1,
15137 Op1: FTSQRT, Op2: SRIdxVal),
15138 0);
15139}
15140
15141SDValue
15142PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op,
15143 SelectionDAG &DAG) const {
15144 // We only have VSX Vector Square Root.
15145 EVT VT = Op.getValueType();
15146 if (VT != MVT::f64 &&
15147 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))
15148 return TargetLowering::getSqrtResultForDenormInput(Operand: Op, DAG);
15149
15150 return DAG.getNode(Opcode: PPCISD::FSQRT, DL: SDLoc(Op), VT, Operand: Op);
15151}
15152
15153SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
15154 int Enabled, int &RefinementSteps,
15155 bool &UseOneConstNR,
15156 bool Reciprocal) const {
15157 EVT VT = Operand.getValueType();
15158 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
15159 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
15160 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15161 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15162 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15163 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15164
15165 // The Newton-Raphson computation with a single constant does not provide
15166 // enough accuracy on some CPUs.
15167 UseOneConstNR = !Subtarget.needsTwoConstNR();
15168 return DAG.getNode(Opcode: PPCISD::FRSQRTE, DL: SDLoc(Operand), VT, Operand);
15169 }
15170 return SDValue();
15171}
15172
15173SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
15174 int Enabled,
15175 int &RefinementSteps) const {
15176 EVT VT = Operand.getValueType();
15177 if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
15178 (VT == MVT::f64 && Subtarget.hasFRE()) ||
15179 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15180 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15181 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15182 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15183 return DAG.getNode(Opcode: PPCISD::FRE, DL: SDLoc(Operand), VT, Operand);
15184 }
15185 return SDValue();
15186}
15187
15188unsigned PPCTargetLowering::combineRepeatedFPDivisors() const {
15189 // Note: This functionality is used only when arcp is enabled, and
15190 // on cores with reciprocal estimates (which are used when arcp is
15191 // enabled for division), this functionality is redundant with the default
15192 // combiner logic (once the division -> reciprocal/multiply transformation
15193 // has taken place). As a result, this matters more for older cores than for
15194 // newer ones.
15195
15196 // Combine multiple FDIVs with the same divisor into multiple FMULs by the
15197 // reciprocal if there are two or more FDIVs (for embedded cores with only
15198 // one FP pipeline) for three or more FDIVs (for generic OOO cores).
15199 switch (Subtarget.getCPUDirective()) {
15200 default:
15201 return 3;
15202 case PPC::DIR_440:
15203 case PPC::DIR_A2:
15204 case PPC::DIR_E500:
15205 case PPC::DIR_E500mc:
15206 case PPC::DIR_E5500:
15207 return 2;
15208 }
15209}
15210
15211// isConsecutiveLSLoc needs to work even if all adds have not yet been
15212// collapsed, and so we need to look through chains of them.
15213static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base,
15214 int64_t& Offset, SelectionDAG &DAG) {
15215 if (DAG.isBaseWithConstantOffset(Op: Loc)) {
15216 Base = Loc.getOperand(i: 0);
15217 Offset += cast<ConstantSDNode>(Val: Loc.getOperand(i: 1))->getSExtValue();
15218
15219 // The base might itself be a base plus an offset, and if so, accumulate
15220 // that as well.
15221 getBaseWithConstantOffset(Loc: Loc.getOperand(i: 0), Base, Offset, DAG);
15222 }
15223}
15224
15225static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base,
15226 unsigned Bytes, int Dist,
15227 SelectionDAG &DAG) {
15228 if (VT.getSizeInBits() / 8 != Bytes)
15229 return false;
15230
15231 SDValue BaseLoc = Base->getBasePtr();
15232 if (Loc.getOpcode() == ISD::FrameIndex) {
15233 if (BaseLoc.getOpcode() != ISD::FrameIndex)
15234 return false;
15235 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
15236 int FI = cast<FrameIndexSDNode>(Val&: Loc)->getIndex();
15237 int BFI = cast<FrameIndexSDNode>(Val&: BaseLoc)->getIndex();
15238 int FS = MFI.getObjectSize(ObjectIdx: FI);
15239 int BFS = MFI.getObjectSize(ObjectIdx: BFI);
15240 if (FS != BFS || FS != (int)Bytes) return false;
15241 return MFI.getObjectOffset(ObjectIdx: FI) == (MFI.getObjectOffset(ObjectIdx: BFI) + Dist*Bytes);
15242 }
15243
15244 SDValue Base1 = Loc, Base2 = BaseLoc;
15245 int64_t Offset1 = 0, Offset2 = 0;
15246 getBaseWithConstantOffset(Loc, Base&: Base1, Offset&: Offset1, DAG);
15247 getBaseWithConstantOffset(Loc: BaseLoc, Base&: Base2, Offset&: Offset2, DAG);
15248 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes))
15249 return true;
15250
15251 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15252 const GlobalValue *GV1 = nullptr;
15253 const GlobalValue *GV2 = nullptr;
15254 Offset1 = 0;
15255 Offset2 = 0;
15256 bool isGA1 = TLI.isGAPlusOffset(N: Loc.getNode(), GA&: GV1, Offset&: Offset1);
15257 bool isGA2 = TLI.isGAPlusOffset(N: BaseLoc.getNode(), GA&: GV2, Offset&: Offset2);
15258 if (isGA1 && isGA2 && GV1 == GV2)
15259 return Offset1 == (Offset2 + Dist*Bytes);
15260 return false;
15261}
15262
15263// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does
15264// not enforce equality of the chain operands.
15265static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
15266 unsigned Bytes, int Dist,
15267 SelectionDAG &DAG) {
15268 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Val: N)) {
15269 EVT VT = LS->getMemoryVT();
15270 SDValue Loc = LS->getBasePtr();
15271 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG);
15272 }
15273
15274 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
15275 EVT VT;
15276 switch (N->getConstantOperandVal(Num: 1)) {
15277 default: return false;
15278 case Intrinsic::ppc_altivec_lvx:
15279 case Intrinsic::ppc_altivec_lvxl:
15280 case Intrinsic::ppc_vsx_lxvw4x:
15281 case Intrinsic::ppc_vsx_lxvw4x_be:
15282 VT = MVT::v4i32;
15283 break;
15284 case Intrinsic::ppc_vsx_lxvd2x:
15285 case Intrinsic::ppc_vsx_lxvd2x_be:
15286 VT = MVT::v2f64;
15287 break;
15288 case Intrinsic::ppc_altivec_lvebx:
15289 VT = MVT::i8;
15290 break;
15291 case Intrinsic::ppc_altivec_lvehx:
15292 VT = MVT::i16;
15293 break;
15294 case Intrinsic::ppc_altivec_lvewx:
15295 VT = MVT::i32;
15296 break;
15297 }
15298
15299 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 2), VT, Base, Bytes, Dist, DAG);
15300 }
15301
15302 if (N->getOpcode() == ISD::INTRINSIC_VOID) {
15303 EVT VT;
15304 switch (N->getConstantOperandVal(Num: 1)) {
15305 default: return false;
15306 case Intrinsic::ppc_altivec_stvx:
15307 case Intrinsic::ppc_altivec_stvxl:
15308 case Intrinsic::ppc_vsx_stxvw4x:
15309 VT = MVT::v4i32;
15310 break;
15311 case Intrinsic::ppc_vsx_stxvd2x:
15312 VT = MVT::v2f64;
15313 break;
15314 case Intrinsic::ppc_vsx_stxvw4x_be:
15315 VT = MVT::v4i32;
15316 break;
15317 case Intrinsic::ppc_vsx_stxvd2x_be:
15318 VT = MVT::v2f64;
15319 break;
15320 case Intrinsic::ppc_altivec_stvebx:
15321 VT = MVT::i8;
15322 break;
15323 case Intrinsic::ppc_altivec_stvehx:
15324 VT = MVT::i16;
15325 break;
15326 case Intrinsic::ppc_altivec_stvewx:
15327 VT = MVT::i32;
15328 break;
15329 }
15330
15331 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 3), VT, Base, Bytes, Dist, DAG);
15332 }
15333
15334 return false;
15335}
15336
15337// Return true is there is a nearyby consecutive load to the one provided
15338// (regardless of alignment). We search up and down the chain, looking though
15339// token factors and other loads (but nothing else). As a result, a true result
15340// indicates that it is safe to create a new consecutive load adjacent to the
15341// load provided.
15342static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) {
15343 SDValue Chain = LD->getChain();
15344 EVT VT = LD->getMemoryVT();
15345
15346 SmallPtrSet<SDNode *, 16> LoadRoots;
15347 SmallVector<SDNode *, 8> Queue(1, Chain.getNode());
15348 SmallPtrSet<SDNode *, 16> Visited;
15349
15350 // First, search up the chain, branching to follow all token-factor operands.
15351 // If we find a consecutive load, then we're done, otherwise, record all
15352 // nodes just above the top-level loads and token factors.
15353 while (!Queue.empty()) {
15354 SDNode *ChainNext = Queue.pop_back_val();
15355 if (!Visited.insert(Ptr: ChainNext).second)
15356 continue;
15357
15358 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: ChainNext)) {
15359 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15360 return true;
15361
15362 if (!Visited.count(Ptr: ChainLD->getChain().getNode()))
15363 Queue.push_back(Elt: ChainLD->getChain().getNode());
15364 } else if (ChainNext->getOpcode() == ISD::TokenFactor) {
15365 for (const SDUse &O : ChainNext->ops())
15366 if (!Visited.count(Ptr: O.getNode()))
15367 Queue.push_back(Elt: O.getNode());
15368 } else
15369 LoadRoots.insert(Ptr: ChainNext);
15370 }
15371
15372 // Second, search down the chain, starting from the top-level nodes recorded
15373 // in the first phase. These top-level nodes are the nodes just above all
15374 // loads and token factors. Starting with their uses, recursively look though
15375 // all loads (just the chain uses) and token factors to find a consecutive
15376 // load.
15377 Visited.clear();
15378 Queue.clear();
15379
15380 for (SDNode *I : LoadRoots) {
15381 Queue.push_back(Elt: I);
15382
15383 while (!Queue.empty()) {
15384 SDNode *LoadRoot = Queue.pop_back_val();
15385 if (!Visited.insert(Ptr: LoadRoot).second)
15386 continue;
15387
15388 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: LoadRoot))
15389 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15390 return true;
15391
15392 for (SDNode *U : LoadRoot->users())
15393 if (((isa<MemSDNode>(Val: U) &&
15394 cast<MemSDNode>(Val: U)->getChain().getNode() == LoadRoot) ||
15395 U->getOpcode() == ISD::TokenFactor) &&
15396 !Visited.count(Ptr: U))
15397 Queue.push_back(Elt: U);
15398 }
15399 }
15400
15401 return false;
15402}
15403
15404/// This function is called when we have proved that a SETCC node can be replaced
15405/// by subtraction (and other supporting instructions) so that the result of
15406/// comparison is kept in a GPR instead of CR. This function is purely for
15407/// codegen purposes and has some flags to guide the codegen process.
15408static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement,
15409 bool Swap, SDLoc &DL, SelectionDAG &DAG) {
15410 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15411
15412 // Zero extend the operands to the largest legal integer. Originally, they
15413 // must be of a strictly smaller size.
15414 auto Op0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, N1: N->getOperand(Num: 0),
15415 N2: DAG.getConstant(Val: Size, DL, VT: MVT::i32));
15416 auto Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, N1: N->getOperand(Num: 1),
15417 N2: DAG.getConstant(Val: Size, DL, VT: MVT::i32));
15418
15419 // Swap if needed. Depends on the condition code.
15420 if (Swap)
15421 std::swap(a&: Op0, b&: Op1);
15422
15423 // Subtract extended integers.
15424 auto SubNode = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: Op0, N2: Op1);
15425
15426 // Move the sign bit to the least significant position and zero out the rest.
15427 // Now the least significant bit carries the result of original comparison.
15428 auto Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: SubNode,
15429 N2: DAG.getConstant(Val: Size - 1, DL, VT: MVT::i32));
15430 auto Final = Shifted;
15431
15432 // Complement the result if needed. Based on the condition code.
15433 if (Complement)
15434 Final = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, N1: Shifted,
15435 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i64));
15436
15437 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Final);
15438}
15439
15440SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N,
15441 DAGCombinerInfo &DCI) const {
15442 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15443
15444 SelectionDAG &DAG = DCI.DAG;
15445 SDLoc DL(N);
15446
15447 // Size of integers being compared has a critical role in the following
15448 // analysis, so we prefer to do this when all types are legal.
15449 if (!DCI.isAfterLegalizeDAG())
15450 return SDValue();
15451
15452 // If all users of SETCC extend its value to a legal integer type
15453 // then we replace SETCC with a subtraction
15454 for (const SDNode *U : N->users())
15455 if (U->getOpcode() != ISD::ZERO_EXTEND)
15456 return SDValue();
15457
15458 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
15459 auto OpSize = N->getOperand(Num: 0).getValueSizeInBits();
15460
15461 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits();
15462
15463 if (OpSize < Size) {
15464 switch (CC) {
15465 default: break;
15466 case ISD::SETULT:
15467 return generateEquivalentSub(N, Size, Complement: false, Swap: false, DL, DAG);
15468 case ISD::SETULE:
15469 return generateEquivalentSub(N, Size, Complement: true, Swap: true, DL, DAG);
15470 case ISD::SETUGT:
15471 return generateEquivalentSub(N, Size, Complement: false, Swap: true, DL, DAG);
15472 case ISD::SETUGE:
15473 return generateEquivalentSub(N, Size, Complement: true, Swap: false, DL, DAG);
15474 }
15475 }
15476
15477 return SDValue();
15478}
15479
15480SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
15481 DAGCombinerInfo &DCI) const {
15482 SelectionDAG &DAG = DCI.DAG;
15483 SDLoc dl(N);
15484
15485 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits");
15486 // If we're tracking CR bits, we need to be careful that we don't have:
15487 // trunc(binary-ops(zext(x), zext(y)))
15488 // or
15489 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...)
15490 // such that we're unnecessarily moving things into GPRs when it would be
15491 // better to keep them in CR bits.
15492
15493 // Note that trunc here can be an actual i1 trunc, or can be the effective
15494 // truncation that comes from a setcc or select_cc.
15495 if (N->getOpcode() == ISD::TRUNCATE &&
15496 N->getValueType(ResNo: 0) != MVT::i1)
15497 return SDValue();
15498
15499 if (N->getOperand(Num: 0).getValueType() != MVT::i32 &&
15500 N->getOperand(Num: 0).getValueType() != MVT::i64)
15501 return SDValue();
15502
15503 if (N->getOpcode() == ISD::SETCC ||
15504 N->getOpcode() == ISD::SELECT_CC) {
15505 // If we're looking at a comparison, then we need to make sure that the
15506 // high bits (all except for the first) don't matter the result.
15507 ISD::CondCode CC =
15508 cast<CondCodeSDNode>(Val: N->getOperand(
15509 Num: N->getOpcode() == ISD::SETCC ? 2 : 4))->get();
15510 unsigned OpBits = N->getOperand(Num: 0).getValueSizeInBits();
15511
15512 if (ISD::isSignedIntSetCC(Code: CC)) {
15513 if (DAG.ComputeNumSignBits(Op: N->getOperand(Num: 0)) != OpBits ||
15514 DAG.ComputeNumSignBits(Op: N->getOperand(Num: 1)) != OpBits)
15515 return SDValue();
15516 } else if (ISD::isUnsignedIntSetCC(Code: CC)) {
15517 if (!DAG.MaskedValueIsZero(Op: N->getOperand(Num: 0),
15518 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)) ||
15519 !DAG.MaskedValueIsZero(Op: N->getOperand(Num: 1),
15520 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)))
15521 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI)
15522 : SDValue());
15523 } else {
15524 // This is neither a signed nor an unsigned comparison, just make sure
15525 // that the high bits are equal.
15526 KnownBits Op1Known = DAG.computeKnownBits(Op: N->getOperand(Num: 0));
15527 KnownBits Op2Known = DAG.computeKnownBits(Op: N->getOperand(Num: 1));
15528
15529 // We don't really care about what is known about the first bit (if
15530 // anything), so pretend that it is known zero for both to ensure they can
15531 // be compared as constants.
15532 Op1Known.Zero.setBit(0); Op1Known.One.clearBit(BitPosition: 0);
15533 Op2Known.Zero.setBit(0); Op2Known.One.clearBit(BitPosition: 0);
15534
15535 if (!Op1Known.isConstant() || !Op2Known.isConstant() ||
15536 Op1Known.getConstant() != Op2Known.getConstant())
15537 return SDValue();
15538 }
15539 }
15540
15541 // We now know that the higher-order bits are irrelevant, we just need to
15542 // make sure that all of the intermediate operations are bit operations, and
15543 // all inputs are extensions.
15544 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15545 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15546 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15547 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15548 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC &&
15549 N->getOperand(Num: 0).getOpcode() != ISD::TRUNCATE &&
15550 N->getOperand(Num: 0).getOpcode() != ISD::SIGN_EXTEND &&
15551 N->getOperand(Num: 0).getOpcode() != ISD::ZERO_EXTEND &&
15552 N->getOperand(Num: 0).getOpcode() != ISD::ANY_EXTEND)
15553 return SDValue();
15554
15555 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) &&
15556 N->getOperand(Num: 1).getOpcode() != ISD::AND &&
15557 N->getOperand(Num: 1).getOpcode() != ISD::OR &&
15558 N->getOperand(Num: 1).getOpcode() != ISD::XOR &&
15559 N->getOperand(Num: 1).getOpcode() != ISD::SELECT &&
15560 N->getOperand(Num: 1).getOpcode() != ISD::SELECT_CC &&
15561 N->getOperand(Num: 1).getOpcode() != ISD::TRUNCATE &&
15562 N->getOperand(Num: 1).getOpcode() != ISD::SIGN_EXTEND &&
15563 N->getOperand(Num: 1).getOpcode() != ISD::ZERO_EXTEND &&
15564 N->getOperand(Num: 1).getOpcode() != ISD::ANY_EXTEND)
15565 return SDValue();
15566
15567 SmallVector<SDValue, 4> Inputs;
15568 SmallVector<SDValue, 8> BinOps, PromOps;
15569 SmallPtrSet<SDNode *, 16> Visited;
15570
15571 for (unsigned i = 0; i < 2; ++i) {
15572 if (((N->getOperand(Num: i).getOpcode() == ISD::SIGN_EXTEND ||
15573 N->getOperand(Num: i).getOpcode() == ISD::ZERO_EXTEND ||
15574 N->getOperand(Num: i).getOpcode() == ISD::ANY_EXTEND) &&
15575 N->getOperand(Num: i).getOperand(i: 0).getValueType() == MVT::i1) ||
15576 isa<ConstantSDNode>(Val: N->getOperand(Num: i)))
15577 Inputs.push_back(Elt: N->getOperand(Num: i));
15578 else
15579 BinOps.push_back(Elt: N->getOperand(Num: i));
15580
15581 if (N->getOpcode() == ISD::TRUNCATE)
15582 break;
15583 }
15584
15585 // Visit all inputs, collect all binary operations (and, or, xor and
15586 // select) that are all fed by extensions.
15587 while (!BinOps.empty()) {
15588 SDValue BinOp = BinOps.pop_back_val();
15589
15590 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15591 continue;
15592
15593 PromOps.push_back(Elt: BinOp);
15594
15595 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15596 // The condition of the select is not promoted.
15597 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15598 continue;
15599 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15600 continue;
15601
15602 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15603 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15604 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
15605 BinOp.getOperand(i).getOperand(i: 0).getValueType() == MVT::i1) ||
15606 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15607 Inputs.push_back(Elt: BinOp.getOperand(i));
15608 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15609 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15610 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15611 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15612 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC ||
15613 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15614 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15615 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15616 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) {
15617 BinOps.push_back(Elt: BinOp.getOperand(i));
15618 } else {
15619 // We have an input that is not an extension or another binary
15620 // operation; we'll abort this transformation.
15621 return SDValue();
15622 }
15623 }
15624 }
15625
15626 // Make sure that this is a self-contained cluster of operations (which
15627 // is not quite the same thing as saying that everything has only one
15628 // use).
15629 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15630 if (isa<ConstantSDNode>(Val: Inputs[i]))
15631 continue;
15632
15633 for (const SDNode *User : Inputs[i].getNode()->users()) {
15634 if (User != N && !Visited.count(Ptr: User))
15635 return SDValue();
15636
15637 // Make sure that we're not going to promote the non-output-value
15638 // operand(s) or SELECT or SELECT_CC.
15639 // FIXME: Although we could sometimes handle this, and it does occur in
15640 // practice that one of the condition inputs to the select is also one of
15641 // the outputs, we currently can't deal with this.
15642 if (User->getOpcode() == ISD::SELECT) {
15643 if (User->getOperand(Num: 0) == Inputs[i])
15644 return SDValue();
15645 } else if (User->getOpcode() == ISD::SELECT_CC) {
15646 if (User->getOperand(Num: 0) == Inputs[i] ||
15647 User->getOperand(Num: 1) == Inputs[i])
15648 return SDValue();
15649 }
15650 }
15651 }
15652
15653 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15654 for (const SDNode *User : PromOps[i].getNode()->users()) {
15655 if (User != N && !Visited.count(Ptr: User))
15656 return SDValue();
15657
15658 // Make sure that we're not going to promote the non-output-value
15659 // operand(s) or SELECT or SELECT_CC.
15660 // FIXME: Although we could sometimes handle this, and it does occur in
15661 // practice that one of the condition inputs to the select is also one of
15662 // the outputs, we currently can't deal with this.
15663 if (User->getOpcode() == ISD::SELECT) {
15664 if (User->getOperand(Num: 0) == PromOps[i])
15665 return SDValue();
15666 } else if (User->getOpcode() == ISD::SELECT_CC) {
15667 if (User->getOperand(Num: 0) == PromOps[i] ||
15668 User->getOperand(Num: 1) == PromOps[i])
15669 return SDValue();
15670 }
15671 }
15672 }
15673
15674 // Replace all inputs with the extension operand.
15675 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15676 // Constants may have users outside the cluster of to-be-promoted nodes,
15677 // and so we need to replace those as we do the promotions.
15678 if (isa<ConstantSDNode>(Val: Inputs[i]))
15679 continue;
15680 else
15681 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: Inputs[i].getOperand(i: 0));
15682 }
15683
15684 std::list<HandleSDNode> PromOpHandles;
15685 for (auto &PromOp : PromOps)
15686 PromOpHandles.emplace_back(args&: PromOp);
15687
15688 // Replace all operations (these are all the same, but have a different
15689 // (i1) return type). DAG.getNode will validate that the types of
15690 // a binary operator match, so go through the list in reverse so that
15691 // we've likely promoted both operands first. Any intermediate truncations or
15692 // extensions disappear.
15693 while (!PromOpHandles.empty()) {
15694 SDValue PromOp = PromOpHandles.back().getValue();
15695 PromOpHandles.pop_back();
15696
15697 if (PromOp.getOpcode() == ISD::TRUNCATE ||
15698 PromOp.getOpcode() == ISD::SIGN_EXTEND ||
15699 PromOp.getOpcode() == ISD::ZERO_EXTEND ||
15700 PromOp.getOpcode() == ISD::ANY_EXTEND) {
15701 if (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: 0)) &&
15702 PromOp.getOperand(i: 0).getValueType() != MVT::i1) {
15703 // The operand is not yet ready (see comment below).
15704 PromOpHandles.emplace_front(args&: PromOp);
15705 continue;
15706 }
15707
15708 SDValue RepValue = PromOp.getOperand(i: 0);
15709 if (isa<ConstantSDNode>(Val: RepValue))
15710 RepValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: RepValue);
15711
15712 DAG.ReplaceAllUsesOfValueWith(From: PromOp, To: RepValue);
15713 continue;
15714 }
15715
15716 unsigned C;
15717 switch (PromOp.getOpcode()) {
15718 default: C = 0; break;
15719 case ISD::SELECT: C = 1; break;
15720 case ISD::SELECT_CC: C = 2; break;
15721 }
15722
15723 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15724 PromOp.getOperand(i: C).getValueType() != MVT::i1) ||
15725 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15726 PromOp.getOperand(i: C+1).getValueType() != MVT::i1)) {
15727 // The to-be-promoted operands of this node have not yet been
15728 // promoted (this should be rare because we're going through the
15729 // list backward, but if one of the operands has several users in
15730 // this cluster of to-be-promoted nodes, it is possible).
15731 PromOpHandles.emplace_front(args&: PromOp);
15732 continue;
15733 }
15734
15735 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15736
15737 // If there are any constant inputs, make sure they're replaced now.
15738 for (unsigned i = 0; i < 2; ++i)
15739 if (isa<ConstantSDNode>(Val: Ops[C+i]))
15740 Ops[C+i] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: Ops[C+i]);
15741
15742 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
15743 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: MVT::i1, Ops));
15744 }
15745
15746 // Now we're left with the initial truncation itself.
15747 if (N->getOpcode() == ISD::TRUNCATE)
15748 return N->getOperand(Num: 0);
15749
15750 // Otherwise, this is a comparison. The operands to be compared have just
15751 // changed type (to i1), but everything else is the same.
15752 return SDValue(N, 0);
15753}
15754
15755SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
15756 DAGCombinerInfo &DCI) const {
15757 SelectionDAG &DAG = DCI.DAG;
15758 SDLoc dl(N);
15759
15760 // If we're tracking CR bits, we need to be careful that we don't have:
15761 // zext(binary-ops(trunc(x), trunc(y)))
15762 // or
15763 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...)
15764 // such that we're unnecessarily moving things into CR bits that can more
15765 // efficiently stay in GPRs. Note that if we're not certain that the high
15766 // bits are set as required by the final extension, we still may need to do
15767 // some masking to get the proper behavior.
15768
15769 // This same functionality is important on PPC64 when dealing with
15770 // 32-to-64-bit extensions; these occur often when 32-bit values are used as
15771 // the return values of functions. Because it is so similar, it is handled
15772 // here as well.
15773
15774 if (N->getValueType(ResNo: 0) != MVT::i32 &&
15775 N->getValueType(ResNo: 0) != MVT::i64)
15776 return SDValue();
15777
15778 if (!((N->getOperand(Num: 0).getValueType() == MVT::i1 && Subtarget.useCRBits()) ||
15779 (N->getOperand(Num: 0).getValueType() == MVT::i32 && Subtarget.isPPC64())))
15780 return SDValue();
15781
15782 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15783 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15784 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15785 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15786 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC)
15787 return SDValue();
15788
15789 SmallVector<SDValue, 4> Inputs;
15790 SmallVector<SDValue, 8> BinOps(1, N->getOperand(Num: 0)), PromOps;
15791 SmallPtrSet<SDNode *, 16> Visited;
15792
15793 // Visit all inputs, collect all binary operations (and, or, xor and
15794 // select) that are all fed by truncations.
15795 while (!BinOps.empty()) {
15796 SDValue BinOp = BinOps.pop_back_val();
15797
15798 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15799 continue;
15800
15801 PromOps.push_back(Elt: BinOp);
15802
15803 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15804 // The condition of the select is not promoted.
15805 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15806 continue;
15807 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15808 continue;
15809
15810 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15811 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15812 Inputs.push_back(Elt: BinOp.getOperand(i));
15813 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15814 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15815 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15816 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15817 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) {
15818 BinOps.push_back(Elt: BinOp.getOperand(i));
15819 } else {
15820 // We have an input that is not a truncation or another binary
15821 // operation; we'll abort this transformation.
15822 return SDValue();
15823 }
15824 }
15825 }
15826
15827 // The operands of a select that must be truncated when the select is
15828 // promoted because the operand is actually part of the to-be-promoted set.
15829 DenseMap<SDNode *, EVT> SelectTruncOp[2];
15830
15831 // Make sure that this is a self-contained cluster of operations (which
15832 // is not quite the same thing as saying that everything has only one
15833 // use).
15834 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15835 if (isa<ConstantSDNode>(Val: Inputs[i]))
15836 continue;
15837
15838 for (SDNode *User : Inputs[i].getNode()->users()) {
15839 if (User != N && !Visited.count(Ptr: User))
15840 return SDValue();
15841
15842 // If we're going to promote the non-output-value operand(s) or SELECT or
15843 // SELECT_CC, record them for truncation.
15844 if (User->getOpcode() == ISD::SELECT) {
15845 if (User->getOperand(Num: 0) == Inputs[i])
15846 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15847 y: User->getOperand(Num: 0).getValueType()));
15848 } else if (User->getOpcode() == ISD::SELECT_CC) {
15849 if (User->getOperand(Num: 0) == Inputs[i])
15850 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15851 y: User->getOperand(Num: 0).getValueType()));
15852 if (User->getOperand(Num: 1) == Inputs[i])
15853 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15854 y: User->getOperand(Num: 1).getValueType()));
15855 }
15856 }
15857 }
15858
15859 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15860 for (SDNode *User : PromOps[i].getNode()->users()) {
15861 if (User != N && !Visited.count(Ptr: User))
15862 return SDValue();
15863
15864 // If we're going to promote the non-output-value operand(s) or SELECT or
15865 // SELECT_CC, record them for truncation.
15866 if (User->getOpcode() == ISD::SELECT) {
15867 if (User->getOperand(Num: 0) == PromOps[i])
15868 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15869 y: User->getOperand(Num: 0).getValueType()));
15870 } else if (User->getOpcode() == ISD::SELECT_CC) {
15871 if (User->getOperand(Num: 0) == PromOps[i])
15872 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15873 y: User->getOperand(Num: 0).getValueType()));
15874 if (User->getOperand(Num: 1) == PromOps[i])
15875 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15876 y: User->getOperand(Num: 1).getValueType()));
15877 }
15878 }
15879 }
15880
15881 unsigned PromBits = N->getOperand(Num: 0).getValueSizeInBits();
15882 bool ReallyNeedsExt = false;
15883 if (N->getOpcode() != ISD::ANY_EXTEND) {
15884 // If all of the inputs are not already sign/zero extended, then
15885 // we'll still need to do that at the end.
15886 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15887 if (isa<ConstantSDNode>(Val: Inputs[i]))
15888 continue;
15889
15890 unsigned OpBits =
15891 Inputs[i].getOperand(i: 0).getValueSizeInBits();
15892 assert(PromBits < OpBits && "Truncation not to a smaller bit count?");
15893
15894 if ((N->getOpcode() == ISD::ZERO_EXTEND &&
15895 !DAG.MaskedValueIsZero(Op: Inputs[i].getOperand(i: 0),
15896 Mask: APInt::getHighBitsSet(numBits: OpBits,
15897 hiBitsSet: OpBits-PromBits))) ||
15898 (N->getOpcode() == ISD::SIGN_EXTEND &&
15899 DAG.ComputeNumSignBits(Op: Inputs[i].getOperand(i: 0)) <
15900 (OpBits-(PromBits-1)))) {
15901 ReallyNeedsExt = true;
15902 break;
15903 }
15904 }
15905 }
15906
15907 // Convert PromOps to handles before doing any RAUW operations, as these
15908 // may CSE with existing nodes, deleting the originals.
15909 std::list<HandleSDNode> PromOpHandles;
15910 for (auto &PromOp : PromOps)
15911 PromOpHandles.emplace_back(args&: PromOp);
15912
15913 // Replace all inputs, either with the truncation operand, or a
15914 // truncation or extension to the final output type.
15915 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15916 // Constant inputs need to be replaced with the to-be-promoted nodes that
15917 // use them because they might have users outside of the cluster of
15918 // promoted nodes.
15919 if (isa<ConstantSDNode>(Val: Inputs[i]))
15920 continue;
15921
15922 SDValue InSrc = Inputs[i].getOperand(i: 0);
15923 if (Inputs[i].getValueType() == N->getValueType(ResNo: 0))
15924 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: InSrc);
15925 else if (N->getOpcode() == ISD::SIGN_EXTEND)
15926 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15927 To: DAG.getSExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15928 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15929 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15930 To: DAG.getZExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15931 else
15932 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15933 To: DAG.getAnyExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15934 }
15935
15936 // Replace all operations (these are all the same, but have a different
15937 // (promoted) return type). DAG.getNode will validate that the types of
15938 // a binary operator match, so go through the list in reverse so that
15939 // we've likely promoted both operands first.
15940 while (!PromOpHandles.empty()) {
15941 SDValue PromOp = PromOpHandles.back().getValue();
15942 PromOpHandles.pop_back();
15943
15944 unsigned C;
15945 switch (PromOp.getOpcode()) {
15946 default: C = 0; break;
15947 case ISD::SELECT: C = 1; break;
15948 case ISD::SELECT_CC: C = 2; break;
15949 }
15950
15951 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15952 PromOp.getOperand(i: C).getValueType() != N->getValueType(ResNo: 0)) ||
15953 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15954 PromOp.getOperand(i: C+1).getValueType() != N->getValueType(ResNo: 0))) {
15955 // The to-be-promoted operands of this node have not yet been
15956 // promoted (this should be rare because we're going through the
15957 // list backward, but if one of the operands has several users in
15958 // this cluster of to-be-promoted nodes, it is possible).
15959 PromOpHandles.emplace_front(args&: PromOp);
15960 continue;
15961 }
15962
15963 // For SELECT and SELECT_CC nodes, we do a similar check for any
15964 // to-be-promoted comparison inputs.
15965 if (PromOp.getOpcode() == ISD::SELECT ||
15966 PromOp.getOpcode() == ISD::SELECT_CC) {
15967 if ((SelectTruncOp[0].count(Val: PromOp.getNode()) &&
15968 PromOp.getOperand(i: 0).getValueType() != N->getValueType(ResNo: 0)) ||
15969 (SelectTruncOp[1].count(Val: PromOp.getNode()) &&
15970 PromOp.getOperand(i: 1).getValueType() != N->getValueType(ResNo: 0))) {
15971 PromOpHandles.emplace_front(args&: PromOp);
15972 continue;
15973 }
15974 }
15975
15976 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15977
15978 // If this node has constant inputs, then they'll need to be promoted here.
15979 for (unsigned i = 0; i < 2; ++i) {
15980 if (!isa<ConstantSDNode>(Val: Ops[C+i]))
15981 continue;
15982 if (Ops[C+i].getValueType() == N->getValueType(ResNo: 0))
15983 continue;
15984
15985 if (N->getOpcode() == ISD::SIGN_EXTEND)
15986 Ops[C+i] = DAG.getSExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15987 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15988 Ops[C+i] = DAG.getZExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15989 else
15990 Ops[C+i] = DAG.getAnyExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15991 }
15992
15993 // If we've promoted the comparison inputs of a SELECT or SELECT_CC,
15994 // truncate them again to the original value type.
15995 if (PromOp.getOpcode() == ISD::SELECT ||
15996 PromOp.getOpcode() == ISD::SELECT_CC) {
15997 auto SI0 = SelectTruncOp[0].find(Val: PromOp.getNode());
15998 if (SI0 != SelectTruncOp[0].end())
15999 Ops[0] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI0->second, Operand: Ops[0]);
16000 auto SI1 = SelectTruncOp[1].find(Val: PromOp.getNode());
16001 if (SI1 != SelectTruncOp[1].end())
16002 Ops[1] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI1->second, Operand: Ops[1]);
16003 }
16004
16005 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
16006 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Ops));
16007 }
16008
16009 // Now we're left with the initial extension itself.
16010 if (!ReallyNeedsExt)
16011 return N->getOperand(Num: 0);
16012
16013 // To zero extend, just mask off everything except for the first bit (in the
16014 // i1 case).
16015 if (N->getOpcode() == ISD::ZERO_EXTEND)
16016 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
16017 N2: DAG.getConstant(Val: APInt::getLowBitsSet(
16018 numBits: N->getValueSizeInBits(ResNo: 0), loBitsSet: PromBits),
16019 DL: dl, VT: N->getValueType(ResNo: 0)));
16020
16021 assert(N->getOpcode() == ISD::SIGN_EXTEND &&
16022 "Invalid extension type");
16023 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: N->getValueType(ResNo: 0), DL: DAG.getDataLayout());
16024 SDValue ShiftCst =
16025 DAG.getConstant(Val: N->getValueSizeInBits(ResNo: 0) - PromBits, DL: dl, VT: ShiftAmountTy);
16026 return DAG.getNode(
16027 Opcode: ISD::SRA, DL: dl, VT: N->getValueType(ResNo: 0),
16028 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: ShiftCst),
16029 N2: ShiftCst);
16030}
16031
16032// The function check a i128 load can convert to 16i8 load for Vcmpequb.
16033static bool canConvertToVcmpequb(SDValue &LHS, SDValue &RHS, bool IsPPC64) {
16034
16035 auto isValidForConvert = [IsPPC64](SDValue &Operand) {
16036 if (!Operand.hasOneUse())
16037 return false;
16038
16039 if (Operand.getValueType() != MVT::i128)
16040 return false;
16041
16042 if (Operand.getOpcode() == ISD::Constant) {
16043 auto *C = cast<ConstantSDNode>(Val&: Operand);
16044 const APInt &Val = C->getAPIntValue();
16045 // On PPC64, comparing an i128 value loaded from memory against a
16046 // constant smaller than 2^16 is usually better left to scalar lowering.
16047 // In that case, the compare can be lowered using xori (since xori has a
16048 // 16-bit immediate field), which is cheaper than materializing a vector
16049 // constant and using vcmpequb.
16050 if (IsPPC64 && Val.ult(RHS: 1ULL << 16))
16051 return false;
16052 return true;
16053 }
16054
16055 auto *LoadNode = dyn_cast<LoadSDNode>(Val&: Operand);
16056 if (!LoadNode)
16057 return false;
16058
16059 // If memory operation is volatile, do not perform any
16060 // optimization or transformation. Volatile operations must be preserved
16061 // as written to ensure correct program behavior, so we return an empty
16062 // SDValue to indicate no action.
16063
16064 if (LoadNode->isVolatile())
16065 return false;
16066
16067 // Only combine loads if both use the unindexed addressing mode.
16068 // PowerPC AltiVec/VMX does not support vector loads or stores with
16069 // pre/post-increment addressing. Indexed modes may imply implicit
16070 // pointer updates, which are not compatible with AltiVec vector
16071 // instructions.
16072 if (LoadNode->getAddressingMode() != ISD::UNINDEXED)
16073 return false;
16074
16075 // Only combine loads if both are non-extending loads
16076 // (ISD::NON_EXTLOAD). Extending loads (such as ISD::ZEXTLOAD or
16077 // ISD::SEXTLOAD) perform zero or sign extension, which may change the
16078 // loaded value's semantics and are not compatible with vector loads.
16079 if (LoadNode->getExtensionType() != ISD::NON_EXTLOAD)
16080 return false;
16081
16082 return true;
16083 };
16084
16085 return (isValidForConvert(LHS) && isValidForConvert(RHS));
16086}
16087
16088SDValue convertTwoLoadsAndCmpToVCMPEQUB(SelectionDAG &DAG, SDNode *N,
16089 const SDLoc &DL) {
16090
16091 assert(N->getOpcode() == ISD::SETCC && "Should be called with a SETCC node");
16092
16093 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16094 assert((CC == ISD::SETNE || CC == ISD::SETEQ) &&
16095 "CC mus be ISD::SETNE or ISD::SETEQ");
16096
16097 auto getV16i8Load = [&](const SDValue &Operand) {
16098 if (Operand.getOpcode() == ISD::Constant)
16099 return DAG.getBitcast(VT: MVT::v16i8, V: Operand);
16100
16101 assert(Operand.getOpcode() == ISD::LOAD && "Must be LoadSDNode here.");
16102
16103 auto *LoadNode = cast<LoadSDNode>(Val: Operand);
16104 // Create a new MachineMemOperand without range metadata.
16105 // Range metadata is only valid for integer scalar types, not vectors.
16106 // The original i128 load may have range metadata, but when we convert
16107 // to v16i8, that metadata is no longer semantically valid.
16108 MachineMemOperand *MMO = LoadNode->getMemOperand();
16109 MachineFunction &MF = DAG.getMachineFunction();
16110 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
16111 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: MMO->getSize(), BaseAlignment: MMO->getAlign(),
16112 Metadata: MMO->getAAInfo(), SSID: MMO->getSyncScopeID(), Ordering: MMO->getSuccessOrdering(),
16113 FailureOrdering: MMO->getFailureOrdering());
16114 SDValue NewLoad = DAG.getLoad(VT: MVT::v16i8, dl: DL, Chain: LoadNode->getChain(),
16115 Ptr: LoadNode->getBasePtr(), MMO: NewMMO);
16116 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadNode, 1), To: NewLoad.getValue(R: 1));
16117 return NewLoad;
16118 };
16119
16120 // Following code transforms the DAG
16121 // t0: ch,glue = EntryToken
16122 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16123 // t3: i128,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16124 // undef:i64
16125 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16126 // t5: i128,ch =
16127 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64 t6: i1 =
16128 // setcc t3, t5, setne:ch
16129 //
16130 // ---->
16131 //
16132 // t0: ch,glue = EntryToken
16133 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16134 // t3: v16i8,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16135 // undef:i64
16136 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16137 // t5: v16i8,ch =
16138 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64
16139 // t6: i32 =
16140 // llvm.ppc.altivec.vcmpequb.p TargetConstant:i32<10505>,
16141 // Constant:i32<2>, t3, t5
16142 // t7: i1 = setcc t6, Constant:i32<0>, seteq:ch
16143
16144 // Or transforms the DAG
16145 // t5: i128,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16146 // t8: i1 =
16147 // setcc Constant:i128<237684487579686500932345921536>, t5, setne:ch
16148 //
16149 // --->
16150 //
16151 // t5: v16i8,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16152 // t6: v16i8 = bitcast Constant:i128<237684487579686500932345921536>
16153 // t7: i32 =
16154 // llvm.ppc.altivec.vcmpequb.p Constant:i32<10962>, Constant:i32<2>, t5, t2
16155
16156 SDValue LHSVec = getV16i8Load(N->getOperand(Num: 0));
16157 SDValue RHSVec = getV16i8Load(N->getOperand(Num: 1));
16158
16159 SDValue IntrID =
16160 DAG.getConstant(Val: Intrinsic::ppc_altivec_vcmpequb_p, DL, VT: MVT::i32);
16161 SDValue CRSel = DAG.getConstant(Val: 2, DL, VT: MVT::i32); // which CR6 predicate field
16162 SDValue PredResult = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
16163 N1: IntrID, N2: CRSel, N3: LHSVec, N4: RHSVec);
16164 // ppc_altivec_vcmpequb_p returns 1 when two vectors are the same,
16165 // so we need to invert the CC opcode.
16166 return DAG.getSetCC(DL, VT: N->getValueType(ResNo: 0), LHS: PredResult,
16167 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32),
16168 Cond: CC == ISD::SETNE ? ISD::SETEQ : ISD::SETNE);
16169}
16170
16171// Detect whether there is a pattern like (setcc (and X, 1), 0, eq).
16172// If it is , return true; otherwise return false.
16173static bool canConvertSETCCToXori(SDNode *N) {
16174 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16175
16176 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16177 if (CC != ISD::SETEQ)
16178 return false;
16179
16180 SDValue LHS = N->getOperand(Num: 0);
16181 SDValue RHS = N->getOperand(Num: 1);
16182
16183 // Check the `SDValue &V` is from `and` with `1`.
16184 auto IsAndWithOne = [](SDValue &V) {
16185 if (V.getOpcode() == ISD::AND) {
16186 for (const SDValue &Op : V->ops())
16187 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op))
16188 if (C->isOne())
16189 return true;
16190 }
16191 return false;
16192 };
16193
16194 // Check whether the SETCC compare with zero.
16195 auto IsCompareWithZero = [](SDValue &V) {
16196 if (auto *C = dyn_cast<ConstantSDNode>(Val&: V))
16197 if (C->isZero())
16198 return true;
16199 return false;
16200 };
16201
16202 return (IsAndWithOne(LHS) && IsCompareWithZero(RHS)) ||
16203 (IsAndWithOne(RHS) && IsCompareWithZero(LHS));
16204}
16205
16206// You must check whether the `SDNode* N` can be converted to Xori using
16207// the function `static bool canConvertSETCCToXori(SDNode *N)`
16208// before calling the function; otherwise, it may produce incorrect results.
16209static SDValue ConvertSETCCToXori(SDNode *N, SelectionDAG &DAG) {
16210
16211 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16212 SDValue LHS = N->getOperand(Num: 0);
16213 SDValue RHS = N->getOperand(Num: 1);
16214 SDLoc DL(N);
16215
16216 [[maybe_unused]] ISD::CondCode CC =
16217 cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16218 assert((CC == ISD::SETEQ) && "CC must be ISD::SETEQ.");
16219 // Rewrite it as XORI (and X, 1), 1.
16220 auto MakeXor1 = [&](SDValue V) {
16221 EVT VT = V.getValueType();
16222 SDValue One = DAG.getConstant(Val: 1, DL, VT);
16223 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: V, N2: One);
16224 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Xor);
16225 };
16226
16227 if (LHS.getOpcode() == ISD::AND && RHS.getOpcode() != ISD::AND)
16228 return MakeXor1(LHS);
16229
16230 if (RHS.getOpcode() == ISD::AND && LHS.getOpcode() != ISD::AND)
16231 return MakeXor1(RHS);
16232
16233 llvm_unreachable("Should not reach here.");
16234}
16235
16236// Match `sext(setcc X, 0, eq)` and turn it into an ADDIC/SUBFE sequence.
16237//
16238// This generates code for:
16239// X == 0 ? -1 : 0
16240//
16241// On pre-ISA 3.1 targets, this is better than the longer CNTLZW/SRWI/NEG
16242// sequence. This is useful for cases like:
16243// uint8_t f(uint8_t x) { return (x == 0) ? -1 : 0; }
16244//
16245// ISA 3.1+ is skipped because those targets can use SETBC.
16246
16247SDValue PPCTargetLowering::combineSignExtendSetCC(SDNode *N,
16248 DAGCombinerInfo &DCI) const {
16249 if (Subtarget.isISA3_1())
16250 return SDValue();
16251
16252 EVT VT = N->getValueType(ResNo: 0);
16253 if (VT != MVT::i32 && (VT != MVT::i64 || !Subtarget.isPPC64()))
16254 return SDValue();
16255
16256 SDValue N0 = N->getOperand(Num: 0);
16257 if (N0.getOpcode() != ISD::SETCC)
16258 return SDValue();
16259
16260 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
16261 SDValue LHS = N0.getOperand(i: 0);
16262 SDValue RHS = N0.getOperand(i: 1);
16263
16264 // Not match: sext (setcc x, 0, eq) or sext (setcc 0, x, eq)
16265 if (CC != ISD::SETEQ || (!isNullConstant(V: LHS) && !isNullConstant(V: RHS)))
16266 return SDValue();
16267
16268 SDLoc dl(N);
16269 SelectionDAG &DAG = DCI.DAG;
16270 SDValue X = isNullConstant(V: LHS) ? RHS : LHS;
16271 EVT XVT = X.getValueType(); // The type of x in the setcc x, 0, eq.
16272
16273 // The type that ADDC/SUBE operate on. Reject larger types and zero-extend
16274 // smaller ones.
16275 MVT OpVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
16276 if (XVT.bitsGT(VT: OpVT))
16277 return SDValue();
16278
16279 if (XVT.bitsLT(VT: OpVT))
16280 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: OpVT, Operand: X);
16281
16282 // Generate: SUBFE(ADDC(X, -1)).
16283 SDValue MinusOne = DAG.getAllOnesConstant(DL: dl, VT: OpVT);
16284 SDValue Addc =
16285 DAG.getNode(Opcode: PPCISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32), N1: X, N2: MinusOne);
16286 SDValue Carry = Addc.getValue(R: 1);
16287 SDValue Sube = DAG.getNode(Opcode: PPCISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32),
16288 N1: Addc, N2: Addc, N3: Carry);
16289
16290 // Truncate back to i32 if we used i64 operations.
16291 if (OpVT == MVT::i64 && VT == MVT::i32)
16292 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Sube);
16293
16294 return Sube;
16295}
16296
16297SDValue PPCTargetLowering::combineSetCC(SDNode *N,
16298 DAGCombinerInfo &DCI) const {
16299 assert(N->getOpcode() == ISD::SETCC &&
16300 "Should be called with a SETCC node");
16301
16302 // Check if the pattern (setcc (and X, 1), 0, eq) is present.
16303 // If it is, rewrite it as XORI (and X, 1), 1.
16304 if (canConvertSETCCToXori(N))
16305 return ConvertSETCCToXori(N, DAG&: DCI.DAG);
16306
16307 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
16308 if (CC == ISD::SETNE || CC == ISD::SETEQ) {
16309 SDValue LHS = N->getOperand(Num: 0);
16310 SDValue RHS = N->getOperand(Num: 1);
16311
16312 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS.
16313 if (LHS.getOpcode() == ISD::SUB && isNullConstant(V: LHS.getOperand(i: 0)) &&
16314 LHS.hasOneUse())
16315 std::swap(a&: LHS, b&: RHS);
16316
16317 // x == 0-y --> x+y == 0
16318 // x != 0-y --> x+y != 0
16319 if (RHS.getOpcode() == ISD::SUB && isNullConstant(V: RHS.getOperand(i: 0)) &&
16320 RHS.hasOneUse()) {
16321 SDLoc DL(N);
16322 SelectionDAG &DAG = DCI.DAG;
16323 EVT VT = N->getValueType(ResNo: 0);
16324 EVT OpVT = LHS.getValueType();
16325 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: OpVT, N1: LHS, N2: RHS.getOperand(i: 1));
16326 return DAG.getSetCC(DL, VT, LHS: Add, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond: CC);
16327 }
16328
16329 // Optimization: Fold i128 equality/inequality compares of two loads into a
16330 // vectorized compare using vcmpequb.p when Altivec is available.
16331 //
16332 // Rationale:
16333 // A scalar i128 SETCC (eq/ne) normally lowers to multiple scalar ops.
16334 // On VSX-capable subtargets, we can instead reinterpret the i128 loads
16335 // as v16i8 vectors and use the Altive vcmpequb.p instruction to
16336 // perform a full 128-bit equality check in a single vector compare.
16337 //
16338 // Example Result:
16339 // This transformation replaces memcmp(a, b, 16) with two vector loads
16340 // and one vector compare instruction.
16341
16342 if (Subtarget.hasAltivec() &&
16343 canConvertToVcmpequb(LHS, RHS, IsPPC64: Subtarget.isPPC64()))
16344 return convertTwoLoadsAndCmpToVCMPEQUB(DAG&: DCI.DAG, N, DL: SDLoc(N));
16345 }
16346
16347 return DAGCombineTruncBoolExt(N, DCI);
16348}
16349
16350// Is this an extending load from an f32 to an f64?
16351static bool isFPExtLoad(SDValue Op) {
16352 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Op.getNode()))
16353 return LD->getExtensionType() == ISD::EXTLOAD &&
16354 Op.getValueType() == MVT::f64;
16355 return false;
16356}
16357
16358/// Reduces the number of fp-to-int conversion when building a vector.
16359///
16360/// If this vector is built out of floating to integer conversions,
16361/// transform it to a vector built out of floating point values followed by a
16362/// single floating to integer conversion of the vector.
16363/// Namely (build_vector (fptosi $A), (fptosi $B), ...)
16364/// becomes (fptosi (build_vector ($A, $B, ...)))
16365SDValue PPCTargetLowering::
16366combineElementTruncationToVectorTruncation(SDNode *N,
16367 DAGCombinerInfo &DCI) const {
16368 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16369 "Should be called with a BUILD_VECTOR node");
16370
16371 SelectionDAG &DAG = DCI.DAG;
16372 SDLoc dl(N);
16373
16374 SDValue FirstInput = N->getOperand(Num: 0);
16375 assert(FirstInput.getOpcode() == PPCISD::MFVSR &&
16376 "The input operand must be an fp-to-int conversion.");
16377
16378 // This combine happens after legalization so the fp_to_[su]i nodes are
16379 // already converted to PPCSISD nodes.
16380 unsigned FirstConversion = FirstInput.getOperand(i: 0).getOpcode();
16381 if (FirstConversion == PPCISD::FCTIDZ ||
16382 FirstConversion == PPCISD::FCTIDUZ ||
16383 FirstConversion == PPCISD::FCTIWZ ||
16384 FirstConversion == PPCISD::FCTIWUZ) {
16385 bool IsSplat = true;
16386 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ ||
16387 FirstConversion == PPCISD::FCTIWUZ;
16388 EVT SrcVT = FirstInput.getOperand(i: 0).getValueType();
16389 SmallVector<SDValue, 4> Ops;
16390 EVT TargetVT = N->getValueType(ResNo: 0);
16391 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16392 SDValue NextOp = N->getOperand(Num: i);
16393 if (NextOp.getOpcode() != PPCISD::MFVSR)
16394 return SDValue();
16395 unsigned NextConversion = NextOp.getOperand(i: 0).getOpcode();
16396 if (NextConversion != FirstConversion)
16397 return SDValue();
16398 // If we are converting to 32-bit integers, we need to add an FP_ROUND.
16399 // This is not valid if the input was originally double precision. It is
16400 // also not profitable to do unless this is an extending load in which
16401 // case doing this combine will allow us to combine consecutive loads.
16402 if (Is32Bit && !isFPExtLoad(Op: NextOp.getOperand(i: 0).getOperand(i: 0)))
16403 return SDValue();
16404 if (N->getOperand(Num: i) != FirstInput)
16405 IsSplat = false;
16406 }
16407
16408 // If this is a splat, we leave it as-is since there will be only a single
16409 // fp-to-int conversion followed by a splat of the integer. This is better
16410 // for 32-bit and smaller ints and neutral for 64-bit ints.
16411 if (IsSplat)
16412 return SDValue();
16413
16414 // Now that we know we have the right type of node, get its operands
16415 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16416 SDValue In = N->getOperand(Num: i).getOperand(i: 0);
16417 if (Is32Bit) {
16418 // For 32-bit values, we need to add an FP_ROUND node (if we made it
16419 // here, we know that all inputs are extending loads so this is safe).
16420 if (In.isUndef())
16421 Ops.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
16422 else {
16423 SDValue Trunc =
16424 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: In.getOperand(i: 0),
16425 N2: DAG.getIntPtrConstant(Val: 1, DL: dl, /*isTarget=*/true));
16426 Ops.push_back(Elt: Trunc);
16427 }
16428 } else
16429 Ops.push_back(Elt: In.isUndef() ? DAG.getUNDEF(VT: SrcVT) : In.getOperand(i: 0));
16430 }
16431
16432 unsigned Opcode;
16433 if (FirstConversion == PPCISD::FCTIDZ ||
16434 FirstConversion == PPCISD::FCTIWZ)
16435 Opcode = ISD::FP_TO_SINT;
16436 else
16437 Opcode = ISD::FP_TO_UINT;
16438
16439 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32;
16440 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: dl, Ops);
16441 return DAG.getNode(Opcode, DL: dl, VT: TargetVT, Operand: BV);
16442 }
16443 return SDValue();
16444}
16445
16446// LXVKQ instruction load VSX vector with a special quadword value
16447// based on an immediate value. This helper method returns the details of the
16448// match as a tuple of {LXVKQ unsigned IMM Value, right_shift_amount}
16449// to help generate the LXVKQ instruction and the subsequent shift instruction
16450// required to match the original build vector pattern.
16451
16452// LXVKQPattern: {LXVKQ unsigned IMM Value, right_shift_amount}
16453using LXVKQPattern = std::tuple<uint32_t, uint8_t>;
16454
16455static std::optional<LXVKQPattern> getPatternInfo(const APInt &FullVal) {
16456
16457 // LXVKQ instruction loads the Quadword value:
16458 // 0x8000_0000_0000_0000_0000_0000_0000_0000 when imm = 0b10000
16459 static const APInt BasePattern = APInt(128, 0x8000000000000000ULL) << 64;
16460 static const uint32_t Uim = 16;
16461
16462 // Check for direct LXVKQ match (no shift needed)
16463 if (FullVal == BasePattern)
16464 return std::make_tuple(args: Uim, args: uint8_t{0});
16465
16466 // Check if FullValue is 1 (the result of the base pattern >> 127)
16467 if (FullVal == APInt(128, 1))
16468 return std::make_tuple(args: Uim, args: uint8_t{127});
16469
16470 return std::nullopt;
16471}
16472
16473/// Combine vector loads to a single load (using lxvkq) or splat with shift of a
16474/// constant (xxspltib + vsrq) by recognising patterns in the Build Vector.
16475/// LXVKQ instruction load VSX vector with a special quadword value based on an
16476/// immediate value. if UIM=0b10000 then LXVKQ loads VSR[32×TX+T] with value
16477/// 0x8000_0000_0000_0000_0000_0000_0000_0000.
16478/// This can be used to inline the build vector constants that have the
16479/// following patterns:
16480///
16481/// 0x8000_0000_0000_0000_0000_0000_0000_0000 (MSB set pattern)
16482/// 0x0000_0000_0000_0000_0000_0000_0000_0001 (LSB set pattern)
16483/// MSB pattern can directly loaded using LXVKQ while LSB is loaded using a
16484/// combination of splatting and right shift instructions.
16485
16486SDValue PPCTargetLowering::combineBVLoadsSpecialValue(SDValue Op,
16487 SelectionDAG &DAG) const {
16488
16489 assert((Op.getNode() && Op.getOpcode() == ISD::BUILD_VECTOR) &&
16490 "Expected a BuildVectorSDNode in combineBVLoadsSpecialValue");
16491
16492 // This transformation is only supported if we are loading either a byte,
16493 // halfword, word, or doubleword.
16494 EVT VT = Op.getValueType();
16495 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
16496 VT == MVT::v2i64))
16497 return SDValue();
16498
16499 LLVM_DEBUG(llvm::dbgs() << "\ncombineBVLoadsSpecialValue: Build vector ("
16500 << VT.getEVTString() << "): ";
16501 Op->dump());
16502
16503 unsigned NumElems = VT.getVectorNumElements();
16504 unsigned ElemBits = VT.getScalarSizeInBits();
16505
16506 bool IsLittleEndian = DAG.getDataLayout().isLittleEndian();
16507
16508 // Check for Non-constant operand in the build vector.
16509 for (const SDValue &Operand : Op.getNode()->op_values()) {
16510 if (!isa<ConstantSDNode>(Val: Operand))
16511 return SDValue();
16512 }
16513
16514 // Assemble build vector operands as a 128-bit register value
16515 // We need to reconstruct what the 128-bit register pattern would be
16516 // that produces this vector when interpreted with the current endianness
16517 APInt FullVal = APInt::getZero(numBits: 128);
16518
16519 for (unsigned Index = 0; Index < NumElems; ++Index) {
16520 auto *C = cast<ConstantSDNode>(Val: Op.getOperand(i: Index));
16521
16522 // Get element value as raw bits (zero-extended)
16523 uint64_t ElemValue = C->getZExtValue();
16524
16525 // Mask to element size to ensure we only get the relevant bits
16526 if (ElemBits < 64)
16527 ElemValue &= ((1ULL << ElemBits) - 1);
16528
16529 // Calculate bit position for this element in the 128-bit register
16530 unsigned BitPos =
16531 (IsLittleEndian) ? (Index * ElemBits) : (128 - (Index + 1) * ElemBits);
16532
16533 // Create APInt for the element value and shift it to correct position
16534 APInt ElemAPInt(128, ElemValue);
16535 ElemAPInt <<= BitPos;
16536
16537 // Place the element value at the correct bit position
16538 FullVal |= ElemAPInt;
16539 }
16540
16541 if (FullVal.isZero() || FullVal.isAllOnes())
16542 return SDValue();
16543
16544 if (auto UIMOpt = getPatternInfo(FullVal)) {
16545 const auto &[Uim, ShiftAmount] = *UIMOpt;
16546 SDLoc Dl(Op);
16547
16548 // Generate LXVKQ instruction if the shift amount is zero.
16549 if (ShiftAmount == 0) {
16550 SDValue UimVal = DAG.getTargetConstant(Val: Uim, DL: Dl, VT: MVT::i32);
16551 SDValue LxvkqInstr =
16552 SDValue(DAG.getMachineNode(Opcode: PPC::LXVKQ, dl: Dl, VT, Op1: UimVal), 0);
16553 LLVM_DEBUG(llvm::dbgs()
16554 << "combineBVLoadsSpecialValue: Instruction Emitted ";
16555 LxvkqInstr.dump());
16556 return LxvkqInstr;
16557 }
16558
16559 assert(ShiftAmount == 127 && "Unexpected lxvkq shift amount value");
16560
16561 // The right shifted pattern can be constructed using a combination of
16562 // XXSPLTIB and VSRQ instruction. VSRQ uses the shift amount from the lower
16563 // 7 bits of byte 15. This can be specified using XXSPLTIB with immediate
16564 // value 255.
16565 SDValue ShiftAmountVec =
16566 SDValue(DAG.getMachineNode(Opcode: PPC::XXSPLTIB, dl: Dl, VT: MVT::v4i32,
16567 Op1: DAG.getTargetConstant(Val: 255, DL: Dl, VT: MVT::i32)),
16568 0);
16569 // Generate appropriate right shift instruction
16570 SDValue ShiftVec = SDValue(
16571 DAG.getMachineNode(Opcode: PPC::VSRQ, dl: Dl, VT, Op1: ShiftAmountVec, Op2: ShiftAmountVec),
16572 0);
16573 LLVM_DEBUG(llvm::dbgs()
16574 << "\n combineBVLoadsSpecialValue: Instruction Emitted ";
16575 ShiftVec.dump());
16576 return ShiftVec;
16577 }
16578 // No patterns matched for build vectors.
16579 return SDValue();
16580}
16581
16582/// Reduce the number of loads when building a vector.
16583///
16584/// Building a vector out of multiple loads can be converted to a load
16585/// of the vector type if the loads are consecutive. If the loads are
16586/// consecutive but in descending order, a shuffle is added at the end
16587/// to reorder the vector.
16588static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) {
16589 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16590 "Should be called with a BUILD_VECTOR node");
16591
16592 SDLoc dl(N);
16593
16594 // Return early for non byte-sized type, as they can't be consecutive.
16595 if (!N->getValueType(ResNo: 0).getVectorElementType().isByteSized())
16596 return SDValue();
16597
16598 bool InputsAreConsecutiveLoads = true;
16599 bool InputsAreReverseConsecutive = true;
16600 unsigned ElemSize = N->getValueType(ResNo: 0).getScalarType().getStoreSize();
16601 SDValue FirstInput = N->getOperand(Num: 0);
16602 bool IsRoundOfExtLoad = false;
16603 LoadSDNode *FirstLoad = nullptr;
16604
16605 if (FirstInput.getOpcode() == ISD::FP_ROUND &&
16606 FirstInput.getOperand(i: 0).getOpcode() == ISD::LOAD) {
16607 FirstLoad = cast<LoadSDNode>(Val: FirstInput.getOperand(i: 0));
16608 IsRoundOfExtLoad = FirstLoad->getExtensionType() == ISD::EXTLOAD;
16609 }
16610 // Not a build vector of (possibly fp_rounded) loads.
16611 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) ||
16612 N->getNumOperands() == 1)
16613 return SDValue();
16614
16615 if (!IsRoundOfExtLoad)
16616 FirstLoad = cast<LoadSDNode>(Val&: FirstInput);
16617
16618 SmallVector<LoadSDNode *, 4> InputLoads;
16619 InputLoads.push_back(Elt: FirstLoad);
16620 for (int i = 1, e = N->getNumOperands(); i < e; ++i) {
16621 // If any inputs are fp_round(extload), they all must be.
16622 if (IsRoundOfExtLoad && N->getOperand(Num: i).getOpcode() != ISD::FP_ROUND)
16623 return SDValue();
16624
16625 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(Num: i).getOperand(i: 0) :
16626 N->getOperand(Num: i);
16627 if (NextInput.getOpcode() != ISD::LOAD)
16628 return SDValue();
16629
16630 SDValue PreviousInput =
16631 IsRoundOfExtLoad ? N->getOperand(Num: i-1).getOperand(i: 0) : N->getOperand(Num: i-1);
16632 LoadSDNode *LD1 = cast<LoadSDNode>(Val&: PreviousInput);
16633 LoadSDNode *LD2 = cast<LoadSDNode>(Val&: NextInput);
16634
16635 // If any inputs are fp_round(extload), they all must be.
16636 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD)
16637 return SDValue();
16638
16639 // We only care about regular loads. The PPC-specific load intrinsics
16640 // will not lead to a merge opportunity.
16641 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD2, Base: LD1, Bytes: ElemSize, Dist: 1))
16642 InputsAreConsecutiveLoads = false;
16643 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD1, Base: LD2, Bytes: ElemSize, Dist: 1))
16644 InputsAreReverseConsecutive = false;
16645
16646 // Exit early if the loads are neither consecutive nor reverse consecutive.
16647 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive)
16648 return SDValue();
16649 InputLoads.push_back(Elt: LD2);
16650 }
16651
16652 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) &&
16653 "The loads cannot be both consecutive and reverse consecutive.");
16654
16655 SDValue WideLoad;
16656 SDValue ReturnSDVal;
16657 if (InputsAreConsecutiveLoads) {
16658 assert(FirstLoad && "Input needs to be a LoadSDNode.");
16659 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: FirstLoad->getChain(),
16660 Ptr: FirstLoad->getBasePtr(), PtrInfo: FirstLoad->getPointerInfo(),
16661 Alignment: FirstLoad->getAlign());
16662 ReturnSDVal = WideLoad;
16663 } else if (InputsAreReverseConsecutive) {
16664 LoadSDNode *LastLoad = InputLoads.back();
16665 assert(LastLoad && "Input needs to be a LoadSDNode.");
16666 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: LastLoad->getChain(),
16667 Ptr: LastLoad->getBasePtr(), PtrInfo: LastLoad->getPointerInfo(),
16668 Alignment: LastLoad->getAlign());
16669 SmallVector<int, 16> Ops;
16670 for (int i = N->getNumOperands() - 1; i >= 0; i--)
16671 Ops.push_back(Elt: i);
16672
16673 ReturnSDVal = DAG.getVectorShuffle(VT: N->getValueType(ResNo: 0), dl, N1: WideLoad,
16674 N2: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)), Mask: Ops);
16675 } else
16676 return SDValue();
16677
16678 for (auto *LD : InputLoads)
16679 DAG.makeEquivalentMemoryOrdering(OldLoad: LD, NewMemOp: WideLoad);
16680 return ReturnSDVal;
16681}
16682
16683// This function adds the required vector_shuffle needed to get
16684// the elements of the vector extract in the correct position
16685// as specified by the CorrectElems encoding.
16686static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG,
16687 SDValue Input, uint64_t Elems,
16688 uint64_t CorrectElems) {
16689 SDLoc dl(N);
16690
16691 unsigned NumElems = Input.getValueType().getVectorNumElements();
16692 SmallVector<int, 16> ShuffleMask(NumElems, -1);
16693
16694 // Knowing the element indices being extracted from the original
16695 // vector and the order in which they're being inserted, just put
16696 // them at element indices required for the instruction.
16697 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16698 if (DAG.getDataLayout().isLittleEndian())
16699 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF;
16700 else
16701 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4;
16702 CorrectElems = CorrectElems >> 8;
16703 Elems = Elems >> 8;
16704 }
16705
16706 SDValue Shuffle =
16707 DAG.getVectorShuffle(VT: Input.getValueType(), dl, N1: Input,
16708 N2: DAG.getUNDEF(VT: Input.getValueType()), Mask: ShuffleMask);
16709
16710 EVT VT = N->getValueType(ResNo: 0);
16711 SDValue Conv = DAG.getBitcast(VT, V: Shuffle);
16712
16713 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
16714 VT: Input.getValueType().getVectorElementType(),
16715 NumElements: VT.getVectorNumElements());
16716 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT, N1: Conv,
16717 N2: DAG.getValueType(ExtVT));
16718}
16719
16720// Look for build vector patterns where input operands come from sign
16721// extended vector_extract elements of specific indices. If the correct indices
16722// aren't used, add a vector shuffle to fix up the indices and create
16723// SIGN_EXTEND_INREG node which selects the vector sign extend instructions
16724// during instruction selection.
16725static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) {
16726 // This array encodes the indices that the vector sign extend instructions
16727 // extract from when extending from one type to another for both BE and LE.
16728 // The right nibble of each byte corresponds to the LE incides.
16729 // and the left nibble of each byte corresponds to the BE incides.
16730 // For example: 0x3074B8FC byte->word
16731 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC
16732 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF
16733 // For example: 0x000070F8 byte->double word
16734 // For LE: the allowed indices are: 0x0,0x8
16735 // For BE: the allowed indices are: 0x7,0xF
16736 uint64_t TargetElems[] = {
16737 0x3074B8FC, // b->w
16738 0x000070F8, // b->d
16739 0x10325476, // h->w
16740 0x00003074, // h->d
16741 0x00001032, // w->d
16742 };
16743
16744 uint64_t Elems = 0;
16745 int Index;
16746 SDValue Input;
16747
16748 auto isSExtOfVecExtract = [&](SDValue Op) -> bool {
16749 if (!Op)
16750 return false;
16751 if (Op.getOpcode() != ISD::SIGN_EXTEND &&
16752 Op.getOpcode() != ISD::SIGN_EXTEND_INREG)
16753 return false;
16754
16755 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value
16756 // of the right width.
16757 SDValue Extract = Op.getOperand(i: 0);
16758 if (Extract.getOpcode() == ISD::ANY_EXTEND)
16759 Extract = Extract.getOperand(i: 0);
16760 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16761 return false;
16762
16763 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Val: Extract.getOperand(i: 1));
16764 if (!ExtOp)
16765 return false;
16766
16767 Index = ExtOp->getZExtValue();
16768 if (Input && Input != Extract.getOperand(i: 0))
16769 return false;
16770
16771 if (!Input)
16772 Input = Extract.getOperand(i: 0);
16773
16774 Elems = Elems << 8;
16775 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4;
16776 Elems |= Index;
16777
16778 return true;
16779 };
16780
16781 // If the build vector operands aren't sign extended vector extracts,
16782 // of the same input vector, then return.
16783 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16784 if (!isSExtOfVecExtract(N->getOperand(Num: i))) {
16785 return SDValue();
16786 }
16787 }
16788
16789 // If the vector extract indices are not correct, add the appropriate
16790 // vector_shuffle.
16791 int TgtElemArrayIdx;
16792 int InputSize = Input.getValueType().getScalarSizeInBits();
16793 int OutputSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
16794 if (InputSize + OutputSize == 40)
16795 TgtElemArrayIdx = 0;
16796 else if (InputSize + OutputSize == 72)
16797 TgtElemArrayIdx = 1;
16798 else if (InputSize + OutputSize == 48)
16799 TgtElemArrayIdx = 2;
16800 else if (InputSize + OutputSize == 80)
16801 TgtElemArrayIdx = 3;
16802 else if (InputSize + OutputSize == 96)
16803 TgtElemArrayIdx = 4;
16804 else
16805 return SDValue();
16806
16807 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx];
16808 CorrectElems = DAG.getDataLayout().isLittleEndian()
16809 ? CorrectElems & 0x0F0F0F0F0F0F0F0F
16810 : CorrectElems & 0xF0F0F0F0F0F0F0F0;
16811 if (Elems != CorrectElems) {
16812 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems);
16813 }
16814
16815 // Regular lowering will catch cases where a shuffle is not needed.
16816 return SDValue();
16817}
16818
16819// Look for the pattern of a load from a narrow width to i128, feeding
16820// into a BUILD_VECTOR of v1i128. Replace this sequence with a PPCISD node
16821// (LXVRZX). This node represents a zero extending load that will be matched
16822// to the Load VSX Vector Rightmost instructions.
16823static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) {
16824 SDLoc DL(N);
16825
16826 // This combine is only eligible for a BUILD_VECTOR of v1i128.
16827 if (N->getValueType(ResNo: 0) != MVT::v1i128)
16828 return SDValue();
16829
16830 SDValue Operand = N->getOperand(Num: 0);
16831 // Proceed with the transformation if the operand to the BUILD_VECTOR
16832 // is a load instruction.
16833 if (Operand.getOpcode() != ISD::LOAD)
16834 return SDValue();
16835
16836 auto *LD = cast<LoadSDNode>(Val&: Operand);
16837 EVT MemoryType = LD->getMemoryVT();
16838
16839 // This transformation is only valid if the we are loading either a byte,
16840 // halfword, word, or doubleword.
16841 bool ValidLDType = MemoryType == MVT::i8 || MemoryType == MVT::i16 ||
16842 MemoryType == MVT::i32 || MemoryType == MVT::i64;
16843
16844 // Ensure that the load from the narrow width is being zero extended to i128.
16845 if (!ValidLDType ||
16846 (LD->getExtensionType() != ISD::ZEXTLOAD &&
16847 LD->getExtensionType() != ISD::EXTLOAD))
16848 return SDValue();
16849
16850 SDValue LoadOps[] = {
16851 LD->getChain(), LD->getBasePtr(),
16852 DAG.getIntPtrConstant(Val: MemoryType.getScalarSizeInBits(), DL)};
16853
16854 return DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVRZX, dl: DL,
16855 VTList: DAG.getVTList(VT1: MVT::v1i128, VT2: MVT::Other),
16856 Ops: LoadOps, MemVT: MemoryType, MMO: LD->getMemOperand());
16857}
16858
16859SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N,
16860 DAGCombinerInfo &DCI) const {
16861 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16862 "Should be called with a BUILD_VECTOR node");
16863
16864 SelectionDAG &DAG = DCI.DAG;
16865 SDLoc dl(N);
16866
16867 if (!Subtarget.hasVSX())
16868 return SDValue();
16869
16870 // The target independent DAG combiner will leave a build_vector of
16871 // float-to-int conversions intact. We can generate MUCH better code for
16872 // a float-to-int conversion of a vector of floats.
16873 SDValue FirstInput = N->getOperand(Num: 0);
16874 if (FirstInput.getOpcode() == PPCISD::MFVSR) {
16875 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI);
16876 if (Reduced)
16877 return Reduced;
16878 }
16879
16880 // If we're building a vector out of consecutive loads, just load that
16881 // vector type.
16882 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG);
16883 if (Reduced)
16884 return Reduced;
16885
16886 // If we're building a vector out of extended elements from another vector
16887 // we have P9 vector integer extend instructions. The code assumes legal
16888 // input types (i.e. it can't handle things like v4i16) so do not run before
16889 // legalization.
16890 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) {
16891 Reduced = combineBVOfVecSExt(N, DAG);
16892 if (Reduced)
16893 return Reduced;
16894 }
16895
16896 // On Power10, the Load VSX Vector Rightmost instructions can be utilized
16897 // if this is a BUILD_VECTOR of v1i128, and if the operand to the BUILD_VECTOR
16898 // is a load from <valid narrow width> to i128.
16899 if (Subtarget.isISA3_1()) {
16900 SDValue BVOfZLoad = combineBVZEXTLOAD(N, DAG);
16901 if (BVOfZLoad)
16902 return BVOfZLoad;
16903 }
16904
16905 if (N->getValueType(ResNo: 0) != MVT::v2f64)
16906 return SDValue();
16907
16908 // Looking for:
16909 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1))
16910 if (FirstInput.getOpcode() != ISD::SINT_TO_FP &&
16911 FirstInput.getOpcode() != ISD::UINT_TO_FP)
16912 return SDValue();
16913 if (N->getOperand(Num: 1).getOpcode() != ISD::SINT_TO_FP &&
16914 N->getOperand(Num: 1).getOpcode() != ISD::UINT_TO_FP)
16915 return SDValue();
16916 if (FirstInput.getOpcode() != N->getOperand(Num: 1).getOpcode())
16917 return SDValue();
16918
16919 SDValue Ext1 = FirstInput.getOperand(i: 0);
16920 SDValue Ext2 = N->getOperand(Num: 1).getOperand(i: 0);
16921 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
16922 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16923 return SDValue();
16924
16925 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Val: Ext1.getOperand(i: 1));
16926 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Val: Ext2.getOperand(i: 1));
16927 if (!Ext1Op || !Ext2Op)
16928 return SDValue();
16929 if (Ext1.getOperand(i: 0).getValueType() != MVT::v4i32 ||
16930 Ext1.getOperand(i: 0) != Ext2.getOperand(i: 0))
16931 return SDValue();
16932
16933 int FirstElem = Ext1Op->getZExtValue();
16934 int SecondElem = Ext2Op->getZExtValue();
16935 int SubvecIdx;
16936 if (FirstElem == 0 && SecondElem == 1)
16937 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0;
16938 else if (FirstElem == 2 && SecondElem == 3)
16939 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1;
16940 else
16941 return SDValue();
16942
16943 SDValue SrcVec = Ext1.getOperand(i: 0);
16944 auto NodeType = (N->getOperand(Num: 1).getOpcode() == ISD::SINT_TO_FP) ?
16945 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP;
16946 return DAG.getNode(Opcode: NodeType, DL: dl, VT: MVT::v2f64,
16947 N1: SrcVec, N2: DAG.getIntPtrConstant(Val: SubvecIdx, DL: dl));
16948}
16949
16950SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
16951 DAGCombinerInfo &DCI) const {
16952 assert((N->getOpcode() == ISD::SINT_TO_FP ||
16953 N->getOpcode() == ISD::UINT_TO_FP) &&
16954 "Need an int -> FP conversion node here");
16955
16956 if (useSoftFloat() || !Subtarget.has64BitSupport())
16957 return SDValue();
16958
16959 SelectionDAG &DAG = DCI.DAG;
16960 SDLoc dl(N);
16961 SDValue Op(N, 0);
16962
16963 // Don't handle ppc_fp128 here or conversions that are out-of-range capable
16964 // from the hardware.
16965 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
16966 return SDValue();
16967 if (!Op.getOperand(i: 0).getValueType().isSimple())
16968 return SDValue();
16969 if (Op.getOperand(i: 0).getValueType().getSimpleVT() <= MVT(MVT::i1) ||
16970 Op.getOperand(i: 0).getValueType().getSimpleVT() > MVT(MVT::i64))
16971 return SDValue();
16972
16973 SDValue FirstOperand(Op.getOperand(i: 0));
16974 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD &&
16975 (FirstOperand.getValueType() == MVT::i8 ||
16976 FirstOperand.getValueType() == MVT::i16);
16977 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) {
16978 bool Signed = N->getOpcode() == ISD::SINT_TO_FP;
16979 bool DstDouble = Op.getValueType() == MVT::f64;
16980 unsigned ConvOp = Signed ?
16981 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) :
16982 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS);
16983 SDValue WidthConst =
16984 DAG.getIntPtrConstant(Val: FirstOperand.getValueType() == MVT::i8 ? 1 : 2,
16985 DL: dl, isTarget: false);
16986 LoadSDNode *LDN = cast<LoadSDNode>(Val: FirstOperand.getNode());
16987 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst };
16988 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXSIZX, dl,
16989 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
16990 Ops, MemVT: MVT::i8, MMO: LDN->getMemOperand());
16991 DAG.makeEquivalentMemoryOrdering(OldLoad: LDN, NewMemOp: Ld);
16992
16993 // For signed conversion, we need to sign-extend the value in the VSR
16994 if (Signed) {
16995 SDValue ExtOps[] = { Ld, WidthConst };
16996 SDValue Ext = DAG.getNode(Opcode: PPCISD::VEXTS, DL: dl, VT: MVT::f64, Ops: ExtOps);
16997 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ext);
16998 } else
16999 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ld);
17000 }
17001
17002
17003 // For i32 intermediate values, unfortunately, the conversion functions
17004 // leave the upper 32 bits of the value are undefined. Within the set of
17005 // scalar instructions, we have no method for zero- or sign-extending the
17006 // value. Thus, we cannot handle i32 intermediate values here.
17007 if (Op.getOperand(i: 0).getValueType() == MVT::i32)
17008 return SDValue();
17009
17010 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&
17011 "UINT_TO_FP is supported only with FPCVT");
17012
17013 // If we have FCFIDS, then use it when converting to single-precision.
17014 // Otherwise, convert to double-precision and then round.
17015 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
17016 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
17017 : PPCISD::FCFIDS)
17018 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
17019 : PPCISD::FCFID);
17020 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
17021 ? MVT::f32
17022 : MVT::f64;
17023
17024 // If we're converting from a float, to an int, and back to a float again,
17025 // then we don't need the store/load pair at all.
17026 if ((Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_UINT &&
17027 Subtarget.hasFPCVT()) ||
17028 (Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT)) {
17029 SDValue Src = Op.getOperand(i: 0).getOperand(i: 0);
17030 if (Src.getValueType() == MVT::f32) {
17031 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
17032 DCI.AddToWorklist(N: Src.getNode());
17033 } else if (Src.getValueType() != MVT::f64) {
17034 // Make sure that we don't pick up a ppc_fp128 source value.
17035 return SDValue();
17036 }
17037
17038 unsigned FCTOp =
17039 Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
17040 PPCISD::FCTIDUZ;
17041
17042 SDValue Tmp = DAG.getNode(Opcode: FCTOp, DL: dl, VT: MVT::f64, Operand: Src);
17043 SDValue FP = DAG.getNode(Opcode: FCFOp, DL: dl, VT: FCFTy, Operand: Tmp);
17044
17045 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
17046 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
17047 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
17048 DCI.AddToWorklist(N: FP.getNode());
17049 }
17050
17051 return FP;
17052 }
17053
17054 return SDValue();
17055}
17056
17057// expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for
17058// builtins) into loads with swaps.
17059SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
17060 DAGCombinerInfo &DCI) const {
17061 // Delay VSX load for LE combine until after LegalizeOps to prioritize other
17062 // load combines.
17063 if (DCI.isBeforeLegalizeOps())
17064 return SDValue();
17065
17066 SelectionDAG &DAG = DCI.DAG;
17067 SDLoc dl(N);
17068 SDValue Chain;
17069 SDValue Base;
17070 MachineMemOperand *MMO;
17071
17072 switch (N->getOpcode()) {
17073 default:
17074 llvm_unreachable("Unexpected opcode for little endian VSX load");
17075 case ISD::LOAD: {
17076 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
17077 Chain = LD->getChain();
17078 Base = LD->getBasePtr();
17079 MMO = LD->getMemOperand();
17080 // If the MMO suggests this isn't a load of a full vector, leave
17081 // things alone. For a built-in, we have to make the change for
17082 // correctness, so if there is a size problem that will be a bug.
17083 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17084 return SDValue();
17085 break;
17086 }
17087 case ISD::INTRINSIC_W_CHAIN: {
17088 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17089 Chain = Intrin->getChain();
17090 // Similarly to the store case below, Intrin->getBasePtr() doesn't get
17091 // us what we want. Get operand 2 instead.
17092 Base = Intrin->getOperand(Num: 2);
17093 MMO = Intrin->getMemOperand();
17094 break;
17095 }
17096 }
17097
17098 MVT VecTy = N->getValueType(ResNo: 0).getSimpleVT();
17099
17100 SDValue LoadOps[] = { Chain, Base };
17101 SDValue Load = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVD2X, dl,
17102 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other),
17103 Ops: LoadOps, MemVT: MVT::v2f64, MMO);
17104
17105 DCI.AddToWorklist(N: Load.getNode());
17106 Chain = Load.getValue(R: 1);
17107 SDValue Swap = DAG.getNode(
17108 Opcode: PPCISD::XXSWAPD, DL: dl, VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Load);
17109 DCI.AddToWorklist(N: Swap.getNode());
17110
17111 // Add a bitcast if the resulting load type doesn't match v2f64.
17112 if (VecTy != MVT::v2f64) {
17113 SDValue N = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: Swap);
17114 DCI.AddToWorklist(N: N.getNode());
17115 // Package {bitcast value, swap's chain} to match Load's shape.
17116 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VT1: VecTy, VT2: MVT::Other),
17117 N1: N, N2: Swap.getValue(R: 1));
17118 }
17119
17120 return Swap;
17121}
17122
17123// expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for
17124// builtins) into stores with swaps.
17125SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
17126 DAGCombinerInfo &DCI) const {
17127 // Delay VSX store for LE combine until after LegalizeOps to prioritize other
17128 // store combines.
17129 if (DCI.isBeforeLegalizeOps())
17130 return SDValue();
17131
17132 SelectionDAG &DAG = DCI.DAG;
17133 SDLoc dl(N);
17134 SDValue Chain;
17135 SDValue Base;
17136 unsigned SrcOpnd;
17137 MachineMemOperand *MMO;
17138
17139 switch (N->getOpcode()) {
17140 default:
17141 llvm_unreachable("Unexpected opcode for little endian VSX store");
17142 case ISD::STORE: {
17143 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
17144 Chain = ST->getChain();
17145 Base = ST->getBasePtr();
17146 MMO = ST->getMemOperand();
17147 SrcOpnd = 1;
17148 // If the MMO suggests this isn't a store of a full vector, leave
17149 // things alone. For a built-in, we have to make the change for
17150 // correctness, so if there is a size problem that will be a bug.
17151 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17152 return SDValue();
17153 break;
17154 }
17155 case ISD::INTRINSIC_VOID: {
17156 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17157 Chain = Intrin->getChain();
17158 // Intrin->getBasePtr() oddly does not get what we want.
17159 Base = Intrin->getOperand(Num: 3);
17160 MMO = Intrin->getMemOperand();
17161 SrcOpnd = 2;
17162 break;
17163 }
17164 }
17165
17166 SDValue Src = N->getOperand(Num: SrcOpnd);
17167 MVT VecTy = Src.getValueType().getSimpleVT();
17168
17169 // All stores are done as v2f64 and possible bit cast.
17170 if (VecTy != MVT::v2f64) {
17171 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: Src);
17172 DCI.AddToWorklist(N: Src.getNode());
17173 }
17174
17175 SDValue Swap = DAG.getNode(Opcode: PPCISD::XXSWAPD, DL: dl,
17176 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Src);
17177 DCI.AddToWorklist(N: Swap.getNode());
17178 Chain = Swap.getValue(R: 1);
17179 SDValue StoreOps[] = { Chain, Swap, Base };
17180 SDValue Store = DAG.getMemIntrinsicNode(Opcode: PPCISD::STXVD2X, dl,
17181 VTList: DAG.getVTList(VT: MVT::Other),
17182 Ops: StoreOps, MemVT: VecTy, MMO);
17183 DCI.AddToWorklist(N: Store.getNode());
17184 return Store;
17185}
17186
17187// Handle DAG combine for STORE (FP_TO_INT F).
17188SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N,
17189 DAGCombinerInfo &DCI) const {
17190 SelectionDAG &DAG = DCI.DAG;
17191 SDLoc dl(N);
17192 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
17193 (void)Opcode;
17194 bool Strict = N->getOperand(Num: 1)->isStrictFPOpcode();
17195
17196 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
17197 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT)
17198 && "Not a FP_TO_INT Instruction!");
17199
17200 SDValue Val = N->getOperand(Num: 1).getOperand(i: Strict ? 1 : 0);
17201 EVT Op1VT = N->getOperand(Num: 1).getValueType();
17202 EVT ResVT = Val.getValueType();
17203
17204 if (!Subtarget.hasVSX() || !Subtarget.hasFPCVT() || !isTypeLegal(VT: ResVT))
17205 return SDValue();
17206
17207 // Only perform combine for conversion to i64/i32 or power9 i16/i8.
17208 bool ValidTypeForStoreFltAsInt =
17209 (Op1VT == MVT::i32 || (Op1VT == MVT::i64 && Subtarget.isPPC64()) ||
17210 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8)));
17211
17212 // TODO: Lower conversion from f128 on all VSX targets
17213 if (ResVT == MVT::ppcf128 || (ResVT == MVT::f128 && !Subtarget.hasP9Vector()))
17214 return SDValue();
17215
17216 if ((Op1VT != MVT::i64 && !Subtarget.hasP8Vector()) ||
17217 cast<StoreSDNode>(Val: N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt)
17218 return SDValue();
17219
17220 Val = convertFPToInt(Op: N->getOperand(Num: 1), DAG, Subtarget);
17221
17222 // Set number of bytes being converted.
17223 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8;
17224 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2),
17225 DAG.getIntPtrConstant(Val: ByteSize, DL: dl, isTarget: false),
17226 DAG.getValueType(Op1VT)};
17227
17228 Val = DAG.getMemIntrinsicNode(Opcode: PPCISD::ST_VSR_SCAL_INT, dl,
17229 VTList: DAG.getVTList(VT: MVT::Other), Ops,
17230 MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
17231 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
17232
17233 return Val;
17234}
17235
17236static bool isAlternatingShuffMask(const ArrayRef<int> &Mask, int NumElts) {
17237 // Check that the source of the element keeps flipping
17238 // (i.e. Mask[i] < NumElts -> Mask[i+i] >= NumElts).
17239 bool PrevElemFromFirstVec = Mask[0] < NumElts;
17240 for (int i = 1, e = Mask.size(); i < e; i++) {
17241 if (PrevElemFromFirstVec && Mask[i] < NumElts)
17242 return false;
17243 if (!PrevElemFromFirstVec && Mask[i] >= NumElts)
17244 return false;
17245 PrevElemFromFirstVec = !PrevElemFromFirstVec;
17246 }
17247 return true;
17248}
17249
17250static bool isSplatBV(SDValue Op) {
17251 if (Op.getOpcode() != ISD::BUILD_VECTOR)
17252 return false;
17253 SDValue FirstOp;
17254
17255 // Find first non-undef input.
17256 for (int i = 0, e = Op.getNumOperands(); i < e; i++) {
17257 FirstOp = Op.getOperand(i);
17258 if (!FirstOp.isUndef())
17259 break;
17260 }
17261
17262 // All inputs are undef or the same as the first non-undef input.
17263 for (int i = 1, e = Op.getNumOperands(); i < e; i++)
17264 if (Op.getOperand(i) != FirstOp && !Op.getOperand(i).isUndef())
17265 return false;
17266 return true;
17267}
17268
17269static SDValue isScalarToVec(SDValue Op) {
17270 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17271 return Op;
17272 if (Op.getOpcode() != ISD::BITCAST)
17273 return SDValue();
17274 Op = Op.getOperand(i: 0);
17275 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17276 return Op;
17277 return SDValue();
17278}
17279
17280// Fix up the shuffle mask to account for the fact that the result of
17281// scalar_to_vector is not in lane zero. This just takes all values in
17282// the ranges specified by the min/max indices and adds the number of
17283// elements required to ensure each element comes from the respective
17284// position in the valid lane.
17285// On little endian, that's just the corresponding element in the other
17286// half of the vector. On big endian, it is in the same half but right
17287// justified rather than left justified in that half.
17288static void fixupShuffleMaskForPermutedSToV(
17289 SmallVectorImpl<int> &ShuffV, int LHSFirstElt, int LHSLastElt,
17290 int RHSFirstElt, int RHSLastElt, int HalfVec, unsigned LHSNumValidElts,
17291 unsigned RHSNumValidElts, const PPCSubtarget &Subtarget) {
17292 int LHSEltFixup =
17293 Subtarget.isLittleEndian() ? HalfVec : HalfVec - LHSNumValidElts;
17294 int RHSEltFixup =
17295 Subtarget.isLittleEndian() ? HalfVec : HalfVec - RHSNumValidElts;
17296 for (int I = 0, E = ShuffV.size(); I < E; ++I) {
17297 int Idx = ShuffV[I];
17298 if (Idx >= LHSFirstElt && Idx <= LHSLastElt)
17299 ShuffV[I] += LHSEltFixup;
17300 else if (Idx >= RHSFirstElt && Idx <= RHSLastElt)
17301 ShuffV[I] += RHSEltFixup;
17302 }
17303}
17304
17305// Replace a SCALAR_TO_VECTOR with a SCALAR_TO_VECTOR_PERMUTED except if
17306// the original is:
17307// (<n x Ty> (scalar_to_vector (Ty (extract_elt <n x Ty> %a, C))))
17308// In such a case, just change the shuffle mask to extract the element
17309// from the permuted index.
17310static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG,
17311 const PPCSubtarget &Subtarget) {
17312 SDLoc dl(OrigSToV);
17313 EVT VT = OrigSToV.getValueType();
17314 assert(OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR &&
17315 "Expecting a SCALAR_TO_VECTOR here");
17316 SDValue Input = OrigSToV.getOperand(i: 0);
17317
17318 if (Input.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17319 ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Val: Input.getOperand(i: 1));
17320 SDValue OrigVector = Input.getOperand(i: 0);
17321
17322 // Can't handle non-const element indices or different vector types
17323 // for the input to the extract and the output of the scalar_to_vector.
17324 if (Idx && VT == OrigVector.getValueType()) {
17325 unsigned NumElts = VT.getVectorNumElements();
17326 assert(
17327 NumElts > 1 &&
17328 "Cannot produce a permuted scalar_to_vector for one element vector");
17329 SmallVector<int, 16> NewMask(NumElts, -1);
17330 unsigned ResultInElt = NumElts / 2;
17331 ResultInElt -= Subtarget.isLittleEndian() ? 0 : 1;
17332 NewMask[ResultInElt] = Idx->getZExtValue();
17333 return DAG.getVectorShuffle(VT, dl, N1: OrigVector, N2: OrigVector, Mask: NewMask);
17334 }
17335 }
17336 return DAG.getNode(Opcode: PPCISD::SCALAR_TO_VECTOR_PERMUTED, DL: dl, VT,
17337 Operand: OrigSToV.getOperand(i: 0));
17338}
17339
17340static bool isShuffleMaskInRange(const SmallVectorImpl<int> &ShuffV,
17341 int HalfVec, int LHSLastElementDefined,
17342 int RHSLastElementDefined) {
17343 for (int Index : ShuffV) {
17344 if (Index < 0) // Skip explicitly undefined mask indices.
17345 continue;
17346 // Handle first input vector of the vector_shuffle.
17347 if ((LHSLastElementDefined >= 0) && (Index < HalfVec) &&
17348 (Index > LHSLastElementDefined))
17349 return false;
17350 // Handle second input vector of the vector_shuffle.
17351 if ((RHSLastElementDefined >= 0) &&
17352 (Index > HalfVec + RHSLastElementDefined))
17353 return false;
17354 }
17355 return true;
17356}
17357
17358static SDValue generateSToVPermutedForVecShuffle(
17359 int ScalarSize, uint64_t ShuffleEltWidth, unsigned &NumValidElts,
17360 int FirstElt, int &LastElt, SDValue VecShuffOperand, SDValue SToVNode,
17361 SelectionDAG &DAG, const PPCSubtarget &Subtarget) {
17362 EVT VecShuffOperandType = VecShuffOperand.getValueType();
17363 // Set up the values for the shuffle vector fixup.
17364 NumValidElts = ScalarSize / VecShuffOperandType.getScalarSizeInBits();
17365 // The last element depends on if the input comes from the LHS or RHS.
17366 //
17367 // For example:
17368 // (shuff (s_to_v i32), (bitcast (s_to_v i64), v4i32), ...)
17369 //
17370 // For the LHS: The last element that comes from the LHS is actually 0, not 3
17371 // because elements 1 and higher of a scalar_to_vector are undefined.
17372 // For the RHS: The last element that comes from the RHS is actually 5, not 7
17373 // because elements 1 and higher of a scalar_to_vector are undefined.
17374 // It is also not 4 because the original scalar_to_vector is wider and
17375 // actually contains two i32 elements.
17376 LastElt = (uint64_t)ScalarSize > ShuffleEltWidth
17377 ? ScalarSize / ShuffleEltWidth - 1 + FirstElt
17378 : FirstElt;
17379 SDValue SToVPermuted = getSToVPermuted(OrigSToV: SToVNode, DAG, Subtarget);
17380 if (SToVPermuted.getValueType() != VecShuffOperandType)
17381 SToVPermuted = DAG.getBitcast(VT: VecShuffOperandType, V: SToVPermuted);
17382 return SToVPermuted;
17383}
17384
17385// On little endian subtargets, combine shuffles such as:
17386// vector_shuffle<16,1,17,3,18,5,19,7,20,9,21,11,22,13,23,15>, <zero>, %b
17387// into:
17388// vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7>, <zero>, %b
17389// because the latter can be matched to a single instruction merge.
17390// Furthermore, SCALAR_TO_VECTOR on little endian always involves a permute
17391// to put the value into element zero. Adjust the shuffle mask so that the
17392// vector can remain in permuted form (to prevent a swap prior to a shuffle).
17393// On big endian targets, this is still useful for SCALAR_TO_VECTOR
17394// nodes with elements smaller than doubleword because all the ways
17395// of getting scalar data into a vector register put the value in the
17396// rightmost element of the left half of the vector.
17397SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN,
17398 SelectionDAG &DAG) const {
17399 SDValue LHS = SVN->getOperand(Num: 0);
17400 SDValue RHS = SVN->getOperand(Num: 1);
17401 auto Mask = SVN->getMask();
17402 int NumElts = LHS.getValueType().getVectorNumElements();
17403 SDValue Res(SVN, 0);
17404 SDLoc dl(SVN);
17405 bool IsLittleEndian = Subtarget.isLittleEndian();
17406
17407 // On big endian targets this is only useful for subtargets with direct moves.
17408 // On little endian targets it would be useful for all subtargets with VSX.
17409 // However adding special handling for LE subtargets without direct moves
17410 // would be wasted effort since the minimum arch for LE is ISA 2.07 (Power8)
17411 // which includes direct moves.
17412 if (!Subtarget.hasDirectMove())
17413 return Res;
17414
17415 // If this is not a shuffle of a shuffle and the first element comes from
17416 // the second vector, canonicalize to the commuted form. This will make it
17417 // more likely to match one of the single instruction patterns.
17418 if (Mask[0] >= NumElts && LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
17419 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) {
17420 std::swap(a&: LHS, b&: RHS);
17421 Res = DAG.getCommutedVectorShuffle(SV: *SVN);
17422
17423 if (!isa<ShuffleVectorSDNode>(Val: Res))
17424 return Res;
17425
17426 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17427 }
17428
17429 // Adjust the shuffle mask if either input vector comes from a
17430 // SCALAR_TO_VECTOR and keep the respective input vector in permuted
17431 // form (to prevent the need for a swap).
17432 SmallVector<int, 16> ShuffV(Mask);
17433 SDValue SToVLHS = isScalarToVec(Op: LHS);
17434 SDValue SToVRHS = isScalarToVec(Op: RHS);
17435 if (SToVLHS || SToVRHS) {
17436 EVT VT = SVN->getValueType(ResNo: 0);
17437 uint64_t ShuffleEltWidth = VT.getVectorElementType().getSizeInBits();
17438 int ShuffleNumElts = ShuffV.size();
17439 int HalfVec = ShuffleNumElts / 2;
17440 // The width of the "valid lane" (i.e. the lane that contains the value that
17441 // is vectorized) needs to be expressed in terms of the number of elements
17442 // of the shuffle. It is thereby the ratio of the values before and after
17443 // any bitcast, which will be set later on if the LHS or RHS are
17444 // SCALAR_TO_VECTOR nodes.
17445 unsigned LHSNumValidElts = HalfVec;
17446 unsigned RHSNumValidElts = HalfVec;
17447
17448 // Initially assume that neither input is permuted. These will be adjusted
17449 // accordingly if either input is. Note, that -1 means that all elements
17450 // are undefined.
17451 int LHSFirstElt = 0;
17452 int RHSFirstElt = ShuffleNumElts;
17453 int LHSLastElt = -1;
17454 int RHSLastElt = -1;
17455
17456 // Get the permuted scalar to vector nodes for the source(s) that come from
17457 // ISD::SCALAR_TO_VECTOR.
17458 // On big endian systems, this only makes sense for element sizes smaller
17459 // than 64 bits since for 64-bit elements, all instructions already put
17460 // the value into element zero. Since scalar size of LHS and RHS may differ
17461 // after isScalarToVec, this should be checked using their own sizes.
17462 int LHSScalarSize = 0;
17463 int RHSScalarSize = 0;
17464 if (SToVLHS) {
17465 LHSScalarSize = SToVLHS.getValueType().getScalarSizeInBits();
17466 if (!IsLittleEndian && LHSScalarSize >= 64)
17467 return Res;
17468 }
17469 if (SToVRHS) {
17470 RHSScalarSize = SToVRHS.getValueType().getScalarSizeInBits();
17471 if (!IsLittleEndian && RHSScalarSize >= 64)
17472 return Res;
17473 }
17474 if (LHSScalarSize != 0)
17475 LHS = generateSToVPermutedForVecShuffle(
17476 ScalarSize: LHSScalarSize, ShuffleEltWidth, NumValidElts&: LHSNumValidElts, FirstElt: LHSFirstElt,
17477 LastElt&: LHSLastElt, VecShuffOperand: LHS, SToVNode: SToVLHS, DAG, Subtarget);
17478 if (RHSScalarSize != 0)
17479 RHS = generateSToVPermutedForVecShuffle(
17480 ScalarSize: RHSScalarSize, ShuffleEltWidth, NumValidElts&: RHSNumValidElts, FirstElt: RHSFirstElt,
17481 LastElt&: RHSLastElt, VecShuffOperand: RHS, SToVNode: SToVRHS, DAG, Subtarget);
17482
17483 if (!isShuffleMaskInRange(ShuffV, HalfVec, LHSLastElementDefined: LHSLastElt, RHSLastElementDefined: RHSLastElt))
17484 return Res;
17485
17486 // Fix up the shuffle mask to reflect where the desired element actually is.
17487 // The minimum and maximum indices that correspond to element zero for both
17488 // the LHS and RHS are computed and will control which shuffle mask entries
17489 // are to be changed. For example, if the RHS is permuted, any shuffle mask
17490 // entries in the range [RHSFirstElt,RHSLastElt] will be adjusted.
17491 fixupShuffleMaskForPermutedSToV(
17492 ShuffV, LHSFirstElt, LHSLastElt, RHSFirstElt, RHSLastElt, HalfVec,
17493 LHSNumValidElts, RHSNumValidElts, Subtarget);
17494 Res = DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17495
17496 // We may have simplified away the shuffle. We won't be able to do anything
17497 // further with it here.
17498 if (!isa<ShuffleVectorSDNode>(Val: Res))
17499 return Res;
17500 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17501 }
17502
17503 SDValue TheSplat = IsLittleEndian ? RHS : LHS;
17504 // The common case after we commuted the shuffle is that the RHS is a splat
17505 // and we have elements coming in from the splat at indices that are not
17506 // conducive to using a merge.
17507 // Example:
17508 // vector_shuffle<0,17,1,19,2,21,3,23,4,25,5,27,6,29,7,31> t1, <zero>
17509 if (!isSplatBV(Op: TheSplat))
17510 return Res;
17511
17512 // We are looking for a mask such that all even elements are from
17513 // one vector and all odd elements from the other.
17514 if (!isAlternatingShuffMask(Mask, NumElts))
17515 return Res;
17516
17517 // Adjust the mask so we are pulling in the same index from the splat
17518 // as the index from the interesting vector in consecutive elements.
17519 if (IsLittleEndian) {
17520 // Example (even elements from first vector):
17521 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero>
17522 if (Mask[0] < NumElts)
17523 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17524 if (ShuffV[i] < 0)
17525 continue;
17526 // If element from non-splat is undef, pick first element from splat.
17527 ShuffV[i] = (ShuffV[i - 1] >= 0 ? ShuffV[i - 1] : 0) + NumElts;
17528 }
17529 // Example (odd elements from first vector):
17530 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero>
17531 else
17532 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17533 if (ShuffV[i] < 0)
17534 continue;
17535 // If element from non-splat is undef, pick first element from splat.
17536 ShuffV[i] = (ShuffV[i + 1] >= 0 ? ShuffV[i + 1] : 0) + NumElts;
17537 }
17538 } else {
17539 // Example (even elements from first vector):
17540 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> <zero>, t1
17541 if (Mask[0] < NumElts)
17542 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17543 if (ShuffV[i] < 0)
17544 continue;
17545 // If element from non-splat is undef, pick first element from splat.
17546 ShuffV[i] = ShuffV[i + 1] >= 0 ? ShuffV[i + 1] - NumElts : 0;
17547 }
17548 // Example (odd elements from first vector):
17549 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> <zero>, t1
17550 else
17551 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17552 if (ShuffV[i] < 0)
17553 continue;
17554 // If element from non-splat is undef, pick first element from splat.
17555 ShuffV[i] = ShuffV[i - 1] >= 0 ? ShuffV[i - 1] - NumElts : 0;
17556 }
17557 }
17558
17559 // If the RHS has undefs, we need to remove them since we may have created
17560 // a shuffle that adds those instead of the splat value.
17561 SDValue SplatVal =
17562 cast<BuildVectorSDNode>(Val: TheSplat.getNode())->getSplatValue();
17563 TheSplat = DAG.getSplatBuildVector(VT: TheSplat.getValueType(), DL: dl, Op: SplatVal);
17564
17565 if (IsLittleEndian)
17566 RHS = TheSplat;
17567 else
17568 LHS = TheSplat;
17569 return DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17570}
17571
17572SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN,
17573 LSBaseSDNode *LSBase,
17574 DAGCombinerInfo &DCI) const {
17575 assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) &&
17576 "Not a reverse memop pattern!");
17577
17578 auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool {
17579 auto Mask = SVN->getMask();
17580 int i = 0;
17581 auto I = Mask.rbegin();
17582 auto E = Mask.rend();
17583
17584 for (; I != E; ++I) {
17585 if (*I != i)
17586 return false;
17587 i++;
17588 }
17589 return true;
17590 };
17591
17592 SelectionDAG &DAG = DCI.DAG;
17593 EVT VT = SVN->getValueType(ResNo: 0);
17594
17595 if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX())
17596 return SDValue();
17597
17598 // Before P9, we have PPCVSXSwapRemoval pass to hack the element order.
17599 // See comment in PPCVSXSwapRemoval.cpp.
17600 // It is conflict with PPCVSXSwapRemoval opt. So we don't do it.
17601 if (!Subtarget.hasP9Vector())
17602 return SDValue();
17603
17604 if(!IsElementReverse(SVN))
17605 return SDValue();
17606
17607 if (LSBase->getOpcode() == ISD::LOAD) {
17608 // If the load return value 0 has more than one user except the
17609 // shufflevector instruction, it is not profitable to replace the
17610 // shufflevector with a reverse load.
17611 for (SDUse &Use : LSBase->uses())
17612 if (Use.getResNo() == 0 &&
17613 Use.getUser()->getOpcode() != ISD::VECTOR_SHUFFLE)
17614 return SDValue();
17615
17616 SDLoc dl(LSBase);
17617 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()};
17618 return DAG.getMemIntrinsicNode(
17619 Opcode: PPCISD::LOAD_VEC_BE, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops: LoadOps,
17620 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17621 }
17622
17623 if (LSBase->getOpcode() == ISD::STORE) {
17624 // If there are other uses of the shuffle, the swap cannot be avoided.
17625 // Forcing the use of an X-Form (since swapped stores only have
17626 // X-Forms) without removing the swap is unprofitable.
17627 if (!SVN->hasOneUse())
17628 return SDValue();
17629
17630 SDLoc dl(LSBase);
17631 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(Num: 0),
17632 LSBase->getBasePtr()};
17633 return DAG.getMemIntrinsicNode(
17634 Opcode: PPCISD::STORE_VEC_BE, dl, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps,
17635 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17636 }
17637
17638 llvm_unreachable("Expected a load or store node here");
17639}
17640
17641static bool isStoreConditional(SDValue Intrin, unsigned &StoreWidth) {
17642 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 1);
17643 if (IntrinsicID == Intrinsic::ppc_stdcx)
17644 StoreWidth = 8;
17645 else if (IntrinsicID == Intrinsic::ppc_stwcx)
17646 StoreWidth = 4;
17647 else if (IntrinsicID == Intrinsic::ppc_sthcx)
17648 StoreWidth = 2;
17649 else if (IntrinsicID == Intrinsic::ppc_stbcx)
17650 StoreWidth = 1;
17651 else
17652 return false;
17653 return true;
17654}
17655
17656static SDValue DAGCombineAddc(SDNode *N,
17657 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17658 if (N->getOpcode() == PPCISD::ADDC && N->hasAnyUseOfValue(Value: 1)) {
17659 // (ADDC (ADDE 0, 0, C), -1) -> C
17660 SDValue LHS = N->getOperand(Num: 0);
17661 SDValue RHS = N->getOperand(Num: 1);
17662 if (LHS->getOpcode() == PPCISD::ADDE &&
17663 isNullConstant(V: LHS->getOperand(Num: 0)) &&
17664 isNullConstant(V: LHS->getOperand(Num: 1)) && isAllOnesConstant(V: RHS)) {
17665 return DCI.CombineTo(N, Res0: SDValue(N, 0), Res1: LHS->getOperand(Num: 2));
17666 }
17667 }
17668 return SDValue();
17669}
17670
17671static SDValue DAGCombineSube(SDNode *N,
17672 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17673 if (N->getOpcode() == PPCISD::SUBE) {
17674 SDValue LHS = N->getOperand(Num: 0);
17675 SDValue RHS = N->getOperand(Num: 1);
17676 SDValue Carry = N->getOperand(Num: 2);
17677
17678 // SUBE(ADDC(ADDE(0, 0, C), -1), same, C) -> ADDC(ADDE(0, 0, C), -1)
17679 if (LHS == RHS && LHS.getOpcode() == PPCISD::ADDC) {
17680 SDValue AddcLHS = LHS.getOperand(i: 0);
17681 SDValue AddcRHS = LHS.getOperand(i: 1);
17682 if (AddcLHS.getOpcode() == PPCISD::ADDE &&
17683 isNullConstant(V: AddcLHS.getOperand(i: 0)) &&
17684 isNullConstant(V: AddcLHS.getOperand(i: 1)) && isAllOnesConstant(V: AddcRHS) &&
17685 Carry == AddcLHS.getOperand(i: 2)) {
17686 return LHS;
17687 }
17688 }
17689 }
17690 return SDValue();
17691}
17692
17693/// Optimize the bitfloor(X) pattern for PowerPC.
17694/// Transforms: select_cc X, 0, 0, (srl MinSignedValue, (ctlz X)), seteq
17695/// Into: srl MinSignedValue, (ctlz X)
17696///
17697/// This is safe on PowerPC because the srw instruction returns 0 when the
17698/// shift amount is == bitwidth, which matches the behavior we need for X=0.
17699static SDValue combineSELECT_CCBitFloor(SDNode *N, SelectionDAG &DAG) {
17700 if (N->getOpcode() != ISD::SELECT_CC)
17701 return SDValue();
17702
17703 // SELECT_CC operands: LHS, RHS, TrueVal, FalseVal, CC
17704 SDValue CmpLHS = N->getOperand(Num: 0);
17705 SDValue CmpRHS = N->getOperand(Num: 1);
17706 SDValue TrueVal = N->getOperand(Num: 2);
17707 SDValue FalseVal = N->getOperand(Num: 3);
17708 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 4))->get();
17709
17710 // Check if condition is (X == 0)
17711 if (CC != ISD::SETEQ || !isNullConstant(V: CmpRHS))
17712 return SDValue();
17713
17714 // Check if TrueVal is constant 0
17715 if (!isNullConstant(V: TrueVal))
17716 return SDValue();
17717
17718 // This combine is replacing a select_cc with a PPC srl, not an srl with a
17719 // PPC srl. If the original srl had multiple uses it would just remain in the
17720 // code. This is at most a performance consideration.
17721 if (FalseVal.getOpcode() != ISD::SRL || !FalseVal.hasOneUse())
17722 return SDValue();
17723
17724 SDValue ShiftVal = FalseVal.getOperand(i: 0);
17725 SDValue ShiftAmt = FalseVal.getOperand(i: 1);
17726
17727 // Check if ShiftVal is MinSignedValue
17728 auto *ShiftConst = dyn_cast<ConstantSDNode>(Val&: ShiftVal);
17729 if (!ShiftConst || !ShiftConst->getAPIntValue().isMinSignedValue())
17730 return SDValue();
17731
17732 SDValue CtlzArg;
17733 // Check if ShiftAmt is (ctlz CmpLHS) or (truncate (ctlz ...))
17734 if (ShiftAmt.getOpcode() != ISD::CTLZ) {
17735 // Look through truncate if present (for i64 ctlz truncated to i32 shift
17736 // amount)
17737 if (ShiftAmt.getOpcode() != ISD::TRUNCATE)
17738 return SDValue();
17739
17740 // Verify the truncate target type is appropriate for shift amount (i32, not
17741 // i1 or other)
17742 if (ShiftAmt.getValueType() != MVT::i32)
17743 return SDValue();
17744
17745 SDValue CtlzNode = ShiftAmt.getOperand(i: 0);
17746
17747 if (CtlzNode.getOpcode() != ISD::CTLZ)
17748 return SDValue();
17749
17750 CtlzArg = CtlzNode.getOperand(i: 0);
17751 } else {
17752 CtlzArg = ShiftAmt.getOperand(i: 0);
17753 }
17754
17755 // Check if ctlz operates on the same value as the comparison
17756 if (CtlzArg != CmpLHS)
17757 return SDValue();
17758
17759 // Using PPCISD::SRL to ensure well-defined behavior.
17760 // On PowerPC, PPCISD::SRL guarantees that shift by bitwidth returns 0,
17761 // which is exactly what we need for the bitfloor(0) case.
17762 SDLoc DL(N);
17763 SDValue PPCSrl =
17764 DAG.getNode(Opcode: PPCISD::SRL, DL, VT: FalseVal.getValueType(), N1: ShiftVal, N2: ShiftAmt);
17765 return PPCSrl;
17766}
17767
17768// Optimize zero-extension of setcc when the compared value is known to be 0
17769// or 1.
17770//
17771// Pattern: zext(setcc(Value, 0, seteq/setne)) where Value is 0 or 1
17772// -> zext(xor(Value, 1)) for seteq
17773// -> zext(Value) for setne
17774//
17775// This optimization avoids the i32 -> i1 -> i32/i64 conversion sequence
17776// by keeping the value in its original i32 type throughout.
17777//
17778// Example:
17779// Before: zext(setcc(test_data_class(...), 0, seteq))
17780// // test_data_class returns 0 or 1 in i32
17781// // setcc converts i32 -> i1
17782// // zext converts i1 -> i64
17783// After: zext(xor(test_data_class(...), 1))
17784// // Stays in i32, then extends to i64
17785//
17786// This is beneficial because:
17787// 1. Eliminates the setcc instruction
17788// 2. Avoids i32 -> i1 truncation
17789// 3. Keeps computation in native integer width
17790
17791static SDValue combineZextSetccWithZero(SDNode *N, SelectionDAG &DAG) {
17792 // Check if this is a zero_extend
17793 if (N->getOpcode() != ISD::ZERO_EXTEND)
17794 return SDValue();
17795
17796 SDValue Src = N->getOperand(Num: 0);
17797
17798 // Check if the source is a setcc
17799 if (Src.getOpcode() != ISD::SETCC)
17800 return SDValue();
17801
17802 SDValue LHS = Src.getOperand(i: 0);
17803 SDValue RHS = Src.getOperand(i: 1);
17804 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Src.getOperand(i: 2))->get();
17805
17806 if (!isNullConstant(V: RHS) && !isNullConstant(V: LHS))
17807 return SDValue();
17808
17809 SDValue NonNullConstant = isNullConstant(V: RHS) ? LHS : RHS;
17810
17811 auto isZeroOrOne = [=](SDValue &V) {
17812 if (V.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
17813 V.getConstantOperandVal(i: 0) == Intrinsic::ppc_test_data_class)
17814 return true;
17815 return false;
17816 };
17817
17818 if (!isZeroOrOne(NonNullConstant))
17819 return SDValue();
17820
17821 // Check for pattern: zext(setcc (Value), 0, seteq)) or
17822 // zext(setcc (Value), 0, setne))
17823 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
17824 // Replace with: zext(xor(Value, 1)) for seteq
17825 // or: zext(Value) for setne
17826 // This keeps the value in i32 instead of converting to i1
17827 SDLoc DL(N);
17828 EVT VType = N->getValueType(ResNo: 0);
17829 SDValue NewNonNullConstant = DAG.getZExtOrTrunc(Op: NonNullConstant, DL, VT: VType);
17830
17831 if (CC == ISD::SETNE)
17832 return NewNonNullConstant;
17833
17834 SDValue One = DAG.getConstant(Val: 1, DL, VT: VType);
17835 return DAG.getNode(Opcode: ISD::XOR, DL, VT: VType, N1: NewNonNullConstant, N2: One);
17836 }
17837
17838 return SDValue();
17839}
17840
17841// Combine XOR patterns with SELECT_CC_I4/I8, for Example:
17842// 1. XOR(SELECT_CC_I4(cond, 1, 0, cc), 1) -> SELECT_CC_I4(cond, 0, 1, cc)
17843// 2. XOR(ZEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond, 0,
17844// 1, cc))
17845// 3. XOR(ANYEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond,
17846// 0, 1, cc))
17847// 4. etc
17848static SDValue combineXorSelectCC(SDNode *N, SelectionDAG &DAG) {
17849 assert(N->getOpcode() == ISD::XOR && "Expected XOR node");
17850
17851 EVT XorVT = N->getValueType(ResNo: 0);
17852 if ((XorVT != MVT::i32 && XorVT != MVT::i64))
17853 return SDValue();
17854
17855 SDValue LHS = N->getOperand(Num: 0);
17856 SDValue RHS = N->getOperand(Num: 1);
17857
17858 // Check for XOR with constant 1
17859 ConstantSDNode *XorConst = dyn_cast<ConstantSDNode>(Val&: RHS);
17860 if (!XorConst || !XorConst->isOne()) {
17861 XorConst = dyn_cast<ConstantSDNode>(Val&: LHS);
17862 if (!XorConst || !XorConst->isOne())
17863 return SDValue();
17864 // Swap so LHS is the SELECT_CC_I4 (or extension) and RHS is the constant
17865 std::swap(a&: LHS, b&: RHS);
17866 }
17867
17868 // Check if LHS has only one use
17869 if (!LHS.hasOneUse())
17870 return SDValue();
17871
17872 // Handle extensions: ZEXT, ANYEXT
17873 SDValue SelectNode = LHS;
17874
17875 if (LHS.getOpcode() == ISD::ZERO_EXTEND ||
17876 LHS.getOpcode() == ISD::ANY_EXTEND) {
17877 SelectNode = LHS.getOperand(i: 0);
17878
17879 // Check if the extension input has only one use
17880 if (!SelectNode.hasOneUse())
17881 return SDValue();
17882 }
17883
17884 // Check if SelectNode is a MachineSDNode with SELECT_CC_I4/I8 opcode
17885 if (!SelectNode.isMachineOpcode())
17886 return SDValue();
17887
17888 unsigned MachineOpc = SelectNode.getMachineOpcode();
17889
17890 // Handle both SELECT_CC_I4 and SELECT_CC_I8
17891 if (MachineOpc != PPC::SELECT_CC_I4 && MachineOpc != PPC::SELECT_CC_I8)
17892 return SDValue();
17893
17894 // SELECT_CC_I4 operands: (cond, true_val, false_val, bropc)
17895 if (SelectNode.getNumOperands() != 4)
17896 return SDValue();
17897
17898 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 1));
17899 ConstantSDNode *ConstOp2 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 2));
17900
17901 if (!ConstOp1 || !ConstOp2)
17902 return SDValue();
17903
17904 // Only optimize if operands are {0, 1} or {1, 0}
17905 if (!((ConstOp1->isOne() && ConstOp2->isZero()) ||
17906 (ConstOp1->isZero() && ConstOp2->isOne())))
17907 return SDValue();
17908
17909 // Pattern matched! Create new SELECT_CC with swapped 0/1 operands to
17910 // eliminate XOR. If original was SELECT_CC(cond, 1, 0, pred), create
17911 // SELECT_CC(cond, 0, 1, pred). If original was SELECT_CC(cond, 0, 1, pred),
17912 // create SELECT_CC(cond, 1, 0, pred).
17913 SDLoc DL(N);
17914 MachineOpc = (XorVT == MVT::i32) ? PPC::SELECT_CC_I4 : PPC::SELECT_CC_I8;
17915
17916 bool ConstOp1IsOne = ConstOp1->isOne();
17917 return SDValue(
17918 DAG.getMachineNode(Opcode: MachineOpc, dl: DL, VT: XorVT,
17919 Ops: {SelectNode.getOperand(i: 0),
17920 DAG.getConstant(Val: ConstOp1IsOne ? 0 : 1, DL, VT: XorVT),
17921 DAG.getConstant(Val: ConstOp1IsOne ? 1 : 0, DL, VT: XorVT),
17922 SelectNode.getOperand(i: 3)}),
17923 0);
17924}
17925
17926SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
17927 DAGCombinerInfo &DCI) const {
17928 SelectionDAG &DAG = DCI.DAG;
17929 SDLoc dl(N);
17930 switch (N->getOpcode()) {
17931 default: break;
17932 case ISD::ADD:
17933 return combineADD(N, DCI);
17934 case ISD::AND: {
17935 // We don't want (and (zext (shift...)), C) if C fits in the width of the
17936 // original input as that will prevent us from selecting optimal rotates.
17937 // This only matters if the input to the extend is i32 widened to i64.
17938 SDValue Op1 = N->getOperand(Num: 0);
17939 SDValue Op2 = N->getOperand(Num: 1);
17940 if ((Op1.getOpcode() != ISD::ZERO_EXTEND &&
17941 Op1.getOpcode() != ISD::ANY_EXTEND) ||
17942 !isa<ConstantSDNode>(Val: Op2) || N->getValueType(ResNo: 0) != MVT::i64 ||
17943 Op1.getOperand(i: 0).getValueType() != MVT::i32)
17944 break;
17945 SDValue NarrowOp = Op1.getOperand(i: 0);
17946 if (NarrowOp.getOpcode() != ISD::SHL && NarrowOp.getOpcode() != ISD::SRL &&
17947 NarrowOp.getOpcode() != ISD::ROTL && NarrowOp.getOpcode() != ISD::ROTR)
17948 break;
17949
17950 uint64_t Imm = Op2->getAsZExtVal();
17951 // Make sure that the constant is narrow enough to fit in the narrow type.
17952 if (!isUInt<32>(x: Imm))
17953 break;
17954 SDValue ConstOp = DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32);
17955 SDValue NarrowAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: NarrowOp, N2: ConstOp);
17956 return DAG.getZExtOrTrunc(Op: NarrowAnd, DL: dl, VT: N->getValueType(ResNo: 0));
17957 }
17958 case ISD::XOR: {
17959 // Optimize XOR(ISEL(1,0,CR), 1) -> ISEL(0,1,CR)
17960 if (SDValue V = combineXorSelectCC(N, DAG))
17961 return V;
17962 break;
17963 }
17964 case ISD::SHL:
17965 return combineSHL(N, DCI);
17966 case ISD::SRA:
17967 return combineSRA(N, DCI);
17968 case ISD::SRL:
17969 return combineSRL(N, DCI);
17970 case ISD::MUL:
17971 return combineMUL(N, DCI);
17972 case ISD::FMA:
17973 case PPCISD::FNMSUB:
17974 return combineFMALike(N, DCI);
17975 case PPCISD::SHL:
17976 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 << V -> 0.
17977 return N->getOperand(Num: 0);
17978 break;
17979 case PPCISD::SRL:
17980 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 >>u V -> 0.
17981 return N->getOperand(Num: 0);
17982 break;
17983 case PPCISD::SRA:
17984 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
17985 if (C->isZero() || // 0 >>s V -> 0.
17986 C->isAllOnes()) // -1 >>s V -> -1.
17987 return N->getOperand(Num: 0);
17988 }
17989 break;
17990 case ISD::SIGN_EXTEND: {
17991 // Optimize: sign_extend(scmp) -> scmp with i64 result (on PPC64)
17992 // This eliminates the sign_extend when scmp result is being extended to i64
17993 // Pattern: t6: i64 = sign_extend t5:i32
17994 // t5: i32 = scmp t2, t4
17995 // Result: t5: i64 = scmp t2, t4
17996 SDValue N0 = N->getOperand(Num: 0);
17997 if (Subtarget.isPPC64() && N0.getOpcode() == ISD::SCMP &&
17998 N->getValueType(ResNo: 0) == MVT::i64 && N0.getValueType() == MVT::i32 &&
17999 N0.hasOneUse()) {
18000 // Recreate the SCMP with i64 result type directly
18001 SDValue LHS = N0.getOperand(i: 0);
18002 SDValue RHS = N0.getOperand(i: 1);
18003 return DAG.getNode(Opcode: ISD::SCMP, DL: dl, VT: MVT::i64, N1: LHS, N2: RHS);
18004 }
18005
18006 if (SDValue SECC = combineSignExtendSetCC(N, DCI))
18007 return SECC;
18008 [[fallthrough]];
18009 }
18010 case ISD::ZERO_EXTEND:
18011 if (SDValue RetV = combineZextSetccWithZero(N, DAG&: DCI.DAG))
18012 return RetV;
18013 [[fallthrough]];
18014 case ISD::ANY_EXTEND:
18015 return DAGCombineExtBoolTrunc(N, DCI);
18016 case ISD::TRUNCATE:
18017 return combineTRUNCATE(N, DCI);
18018 case ISD::SETCC:
18019 if (SDValue CSCC = combineSetCC(N, DCI))
18020 return CSCC;
18021 [[fallthrough]];
18022 case ISD::SELECT_CC:
18023 if (SDValue V = combineSELECT_CCBitFloor(N, DAG))
18024 return V;
18025 return DAGCombineTruncBoolExt(N, DCI);
18026 case ISD::SINT_TO_FP:
18027 case ISD::UINT_TO_FP:
18028 return combineFPToIntToFP(N, DCI);
18029 case ISD::VECTOR_SHUFFLE:
18030 if (ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode())) {
18031 LSBaseSDNode* LSBase = cast<LSBaseSDNode>(Val: N->getOperand(Num: 0));
18032 return combineVReverseMemOP(SVN: cast<ShuffleVectorSDNode>(Val: N), LSBase, DCI);
18033 }
18034 return combineVectorShuffle(SVN: cast<ShuffleVectorSDNode>(Val: N), DAG&: DCI.DAG);
18035 case ISD::STORE: {
18036
18037 EVT Op1VT = N->getOperand(Num: 1).getValueType();
18038 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
18039
18040 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
18041 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT) {
18042 SDValue Val = combineStoreFPToInt(N, DCI);
18043 if (Val)
18044 return Val;
18045 }
18046
18047 if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) {
18048 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 1));
18049 SDValue Val= combineVReverseMemOP(SVN, LSBase: cast<LSBaseSDNode>(Val: N), DCI);
18050 if (Val)
18051 return Val;
18052 }
18053
18054 // Turn STORE (BSWAP) -> sthbrx/stwbrx.
18055 if (cast<StoreSDNode>(Val: N)->isUnindexed() && Opcode == ISD::BSWAP &&
18056 N->getOperand(Num: 1).getNode()->hasOneUse() &&
18057 (Op1VT == MVT::i32 || Op1VT == MVT::i16 ||
18058 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) {
18059
18060 // STBRX can only handle simple types and it makes no sense to store less
18061 // two bytes in byte-reversed order.
18062 EVT mVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
18063 if (mVT.isExtended() || mVT.getSizeInBits() < 16)
18064 break;
18065
18066 SDValue BSwapOp = N->getOperand(Num: 1).getOperand(i: 0);
18067 // Do an any-extend to 32-bits if this is a half-word input.
18068 if (BSwapOp.getValueType() == MVT::i16)
18069 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: BSwapOp);
18070
18071 // If the type of BSWAP operand is wider than stored memory width
18072 // it need to be shifted to the right side before STBRX.
18073 if (Op1VT.bitsGT(VT: mVT)) {
18074 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits();
18075 BSwapOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: Op1VT, N1: BSwapOp,
18076 N2: DAG.getConstant(Val: Shift, DL: dl, VT: MVT::i32));
18077 // Need to truncate if this is a bswap of i64 stored as i32/i16.
18078 if (Op1VT == MVT::i64)
18079 BSwapOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: BSwapOp);
18080 }
18081
18082 SDValue Ops[] = {
18083 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2), DAG.getValueType(mVT)
18084 };
18085 return
18086 DAG.getMemIntrinsicNode(Opcode: PPCISD::STBRX, dl, VTList: DAG.getVTList(VT: MVT::Other),
18087 Ops, MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
18088 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
18089 }
18090
18091 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0>
18092 // So it can increase the chance of CSE constant construction.
18093 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() &&
18094 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) && Op1VT == MVT::i32) {
18095 // Need to sign-extended to 64-bits to handle negative values.
18096 EVT MemVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
18097 uint64_t Val64 = SignExtend64(X: N->getConstantOperandVal(Num: 1),
18098 B: MemVT.getSizeInBits());
18099 SDValue Const64 = DAG.getConstant(Val: Val64, DL: dl, VT: MVT::i64);
18100
18101 auto *ST = cast<StoreSDNode>(Val: N);
18102 SDValue NewST = DAG.getStore(Chain: ST->getChain(), dl, Val: Const64,
18103 Ptr: ST->getBasePtr(), Offset: ST->getOffset(), SVT: MemVT,
18104 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
18105 /*IsTruncating=*/true);
18106 // Note we use CombineTo here to prevent DAGCombiner from visiting the
18107 // new store which will change the constant by removing non-demanded bits.
18108 return ST->isUnindexed()
18109 ? DCI.CombineTo(N, Res: NewST, /*AddTo=*/false)
18110 : DCI.CombineTo(N, Res0: NewST, Res1: NewST.getValue(R: 1), /*AddTo=*/false);
18111 }
18112
18113 // For little endian, VSX stores require generating xxswapd/lxvd2x.
18114 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18115 if (Op1VT.isSimple()) {
18116 MVT StoreVT = Op1VT.getSimpleVT();
18117 if (Subtarget.needsSwapsForVSXMemOps() &&
18118 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 ||
18119 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32))
18120 return expandVSXStoreForLE(N, DCI);
18121 }
18122 break;
18123 }
18124 case ISD::LOAD: {
18125 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
18126 EVT VT = LD->getValueType(ResNo: 0);
18127
18128 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18129 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18130 if (VT.isSimple()) {
18131 MVT LoadVT = VT.getSimpleVT();
18132 if (Subtarget.needsSwapsForVSXMemOps() &&
18133 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 ||
18134 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32))
18135 return expandVSXLoadForLE(N, DCI);
18136 }
18137
18138 // We sometimes end up with a 64-bit integer load, from which we extract
18139 // two single-precision floating-point numbers. This happens with
18140 // std::complex<float>, and other similar structures, because of the way we
18141 // canonicalize structure copies. However, if we lack direct moves,
18142 // then the final bitcasts from the extracted integer values to the
18143 // floating-point numbers turn into store/load pairs. Even with direct moves,
18144 // just loading the two floating-point numbers is likely better.
18145 auto ReplaceTwoFloatLoad = [&]() {
18146 if (VT != MVT::i64)
18147 return false;
18148
18149 if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
18150 LD->isVolatile())
18151 return false;
18152
18153 // We're looking for a sequence like this:
18154 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64
18155 // t16: i64 = srl t13, Constant:i32<32>
18156 // t17: i32 = truncate t16
18157 // t18: f32 = bitcast t17
18158 // t19: i32 = truncate t13
18159 // t20: f32 = bitcast t19
18160
18161 if (!LD->hasNUsesOfValue(NUses: 2, Value: 0))
18162 return false;
18163
18164 auto UI = LD->user_begin();
18165 while (UI.getUse().getResNo() != 0) ++UI;
18166 SDNode *Trunc = *UI++;
18167 while (UI.getUse().getResNo() != 0) ++UI;
18168 SDNode *RightShift = *UI;
18169 if (Trunc->getOpcode() != ISD::TRUNCATE)
18170 std::swap(a&: Trunc, b&: RightShift);
18171
18172 if (Trunc->getOpcode() != ISD::TRUNCATE ||
18173 Trunc->getValueType(ResNo: 0) != MVT::i32 ||
18174 !Trunc->hasOneUse())
18175 return false;
18176 if (RightShift->getOpcode() != ISD::SRL ||
18177 !isa<ConstantSDNode>(Val: RightShift->getOperand(Num: 1)) ||
18178 RightShift->getConstantOperandVal(Num: 1) != 32 ||
18179 !RightShift->hasOneUse())
18180 return false;
18181
18182 SDNode *Trunc2 = *RightShift->user_begin();
18183 if (Trunc2->getOpcode() != ISD::TRUNCATE ||
18184 Trunc2->getValueType(ResNo: 0) != MVT::i32 ||
18185 !Trunc2->hasOneUse())
18186 return false;
18187
18188 SDNode *Bitcast = *Trunc->user_begin();
18189 SDNode *Bitcast2 = *Trunc2->user_begin();
18190
18191 if (Bitcast->getOpcode() != ISD::BITCAST ||
18192 Bitcast->getValueType(ResNo: 0) != MVT::f32)
18193 return false;
18194 if (Bitcast2->getOpcode() != ISD::BITCAST ||
18195 Bitcast2->getValueType(ResNo: 0) != MVT::f32)
18196 return false;
18197
18198 if (Subtarget.isLittleEndian())
18199 std::swap(a&: Bitcast, b&: Bitcast2);
18200
18201 // Bitcast has the second float (in memory-layout order) and Bitcast2
18202 // has the first one.
18203
18204 SDValue BasePtr = LD->getBasePtr();
18205 if (LD->isIndexed()) {
18206 assert(LD->getAddressingMode() == ISD::PRE_INC &&
18207 "Non-pre-inc AM on PPC?");
18208 BasePtr =
18209 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18210 N2: LD->getOffset());
18211 }
18212
18213 auto MMOFlags =
18214 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile;
18215 SDValue FloatLoad = DAG.getLoad(VT: MVT::f32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18216 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign(),
18217 MMOFlags, Metadata: LD->getAAInfo());
18218 SDValue AddPtr =
18219 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(),
18220 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18221 SDValue FloatLoad2 = DAG.getLoad(
18222 VT: MVT::f32, dl, Chain: SDValue(FloatLoad.getNode(), 1), Ptr: AddPtr,
18223 PtrInfo: LD->getPointerInfo().getWithOffset(O: 4),
18224 Alignment: commonAlignment(A: LD->getAlign(), Offset: 4), MMOFlags, Metadata: LD->getAAInfo());
18225
18226 if (LD->isIndexed()) {
18227 // Note that DAGCombine should re-form any pre-increment load(s) from
18228 // what is produced here if that makes sense.
18229 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: BasePtr);
18230 }
18231
18232 DCI.CombineTo(N: Bitcast2, Res: FloatLoad);
18233 DCI.CombineTo(N: Bitcast, Res: FloatLoad2);
18234
18235 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, LD->isIndexed() ? 2 : 1),
18236 To: SDValue(FloatLoad2.getNode(), 1));
18237 return true;
18238 };
18239
18240 if (ReplaceTwoFloatLoad())
18241 return SDValue(N, 0);
18242
18243 EVT MemVT = LD->getMemoryVT();
18244 Type *Ty = MemVT.getTypeForEVT(Context&: *DAG.getContext());
18245 Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
18246 if (LD->isUnindexed() && VT.isVector() &&
18247 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
18248 // P8 and later hardware should just use LOAD.
18249 !Subtarget.hasP8Vector() &&
18250 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
18251 VT == MVT::v4f32))) &&
18252 LD->getAlign() < ABIAlignment) {
18253 // This is a type-legal unaligned Altivec load.
18254 SDValue Chain = LD->getChain();
18255 SDValue Ptr = LD->getBasePtr();
18256 bool isLittleEndian = Subtarget.isLittleEndian();
18257
18258 // This implements the loading of unaligned vectors as described in
18259 // the venerable Apple Velocity Engine overview. Specifically:
18260 // https://developer.apple.com/hardwaredrivers/ve/alignment.html
18261 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html
18262 //
18263 // The general idea is to expand a sequence of one or more unaligned
18264 // loads into an alignment-based permutation-control instruction (lvsl
18265 // or lvsr), a series of regular vector loads (which always truncate
18266 // their input address to an aligned address), and a series of
18267 // permutations. The results of these permutations are the requested
18268 // loaded values. The trick is that the last "extra" load is not taken
18269 // from the address you might suspect (sizeof(vector) bytes after the
18270 // last requested load), but rather sizeof(vector) - 1 bytes after the
18271 // last requested vector. The point of this is to avoid a page fault if
18272 // the base address happened to be aligned. This works because if the
18273 // base address is aligned, then adding less than a full vector length
18274 // will cause the last vector in the sequence to be (re)loaded.
18275 // Otherwise, the next vector will be fetched as you might suspect was
18276 // necessary.
18277
18278 // We might be able to reuse the permutation generation from
18279 // a different base address offset from this one by an aligned amount.
18280 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this
18281 // optimization later.
18282 Intrinsic::ID Intr, IntrLD, IntrPerm;
18283 MVT PermCntlTy, PermTy, LDTy;
18284 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18285 : Intrinsic::ppc_altivec_lvsl;
18286 IntrLD = Intrinsic::ppc_altivec_lvx;
18287 IntrPerm = Intrinsic::ppc_altivec_vperm;
18288 PermCntlTy = MVT::v16i8;
18289 PermTy = MVT::v4i32;
18290 LDTy = MVT::v4i32;
18291
18292 SDValue PermCntl = BuildIntrinsicOp(IID: Intr, Op: Ptr, DAG, dl, DestVT: PermCntlTy);
18293
18294 // Create the new MMO for the new base load. It is like the original MMO,
18295 // but represents an area in memory almost twice the vector size centered
18296 // on the original address. If the address is unaligned, we might start
18297 // reading up to (sizeof(vector)-1) bytes below the address of the
18298 // original unaligned load.
18299 MachineFunction &MF = DAG.getMachineFunction();
18300 MachineMemOperand *BaseMMO =
18301 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18302 Offset: -(int64_t)MemVT.getStoreSize()+1,
18303 Size: 2*MemVT.getStoreSize()-1);
18304
18305 // Create the new base load.
18306 SDValue LDXIntID =
18307 DAG.getTargetConstant(Val: IntrLD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18308 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
18309 SDValue BaseLoad =
18310 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18311 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18312 Ops: BaseLoadOps, MemVT: LDTy, MMO: BaseMMO);
18313
18314 // Note that the value of IncOffset (which is provided to the next
18315 // load's pointer info offset value, and thus used to calculate the
18316 // alignment), and the value of IncValue (which is actually used to
18317 // increment the pointer value) are different! This is because we
18318 // require the next load to appear to be aligned, even though it
18319 // is actually offset from the base pointer by a lesser amount.
18320 int IncOffset = VT.getSizeInBits() / 8;
18321 int IncValue = IncOffset;
18322
18323 // Walk (both up and down) the chain looking for another load at the real
18324 // (aligned) offset (the alignment of the other load does not matter in
18325 // this case). If found, then do not use the offset reduction trick, as
18326 // that will prevent the loads from being later combined (as they would
18327 // otherwise be duplicates).
18328 if (!findConsecutiveLoad(LD, DAG))
18329 --IncValue;
18330
18331 SDValue Increment =
18332 DAG.getConstant(Val: IncValue, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18333 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr, N2: Increment);
18334
18335 MachineMemOperand *ExtraMMO =
18336 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18337 Offset: 1, Size: 2*MemVT.getStoreSize()-1);
18338 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr };
18339 SDValue ExtraLoad =
18340 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18341 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18342 Ops: ExtraLoadOps, MemVT: LDTy, MMO: ExtraMMO);
18343
18344 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18345 N1: BaseLoad.getValue(R: 1), N2: ExtraLoad.getValue(R: 1));
18346
18347 // Because vperm has a big-endian bias, we must reverse the order
18348 // of the input vectors and complement the permute control vector
18349 // when generating little endian code. We have already handled the
18350 // latter by using lvsr instead of lvsl, so just reverse BaseLoad
18351 // and ExtraLoad here.
18352 SDValue Perm;
18353 if (isLittleEndian)
18354 Perm = BuildIntrinsicOp(IID: IntrPerm,
18355 Op0: ExtraLoad, Op1: BaseLoad, Op2: PermCntl, DAG, dl);
18356 else
18357 Perm = BuildIntrinsicOp(IID: IntrPerm,
18358 Op0: BaseLoad, Op1: ExtraLoad, Op2: PermCntl, DAG, dl);
18359
18360 if (VT != PermTy)
18361 Perm = Subtarget.hasAltivec()
18362 ? DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Perm)
18363 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Perm,
18364 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i64));
18365 // second argument is 1 because this rounding
18366 // is always exact.
18367
18368 // The output of the permutation is our loaded result, the TokenFactor is
18369 // our new chain.
18370 DCI.CombineTo(N, Res0: Perm, Res1: TF);
18371 return SDValue(N, 0);
18372 }
18373 }
18374 break;
18375 case ISD::INTRINSIC_WO_CHAIN: {
18376 bool isLittleEndian = Subtarget.isLittleEndian();
18377 unsigned IID = N->getConstantOperandVal(Num: 0);
18378 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18379 : Intrinsic::ppc_altivec_lvsl);
18380 if (IID == Intr && N->getOperand(Num: 1)->getOpcode() == ISD::ADD) {
18381 SDValue Add = N->getOperand(Num: 1);
18382
18383 int Bits = 4 /* 16 byte alignment */;
18384
18385 if (DAG.MaskedValueIsZero(Op: Add->getOperand(Num: 1),
18386 Mask: APInt::getAllOnes(numBits: Bits /* alignment */)
18387 .zext(width: Add.getScalarValueSizeInBits()))) {
18388 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18389 for (SDNode *U : BasePtr->users()) {
18390 if (U->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18391 U->getConstantOperandVal(Num: 0) == IID) {
18392 // We've found another LVSL/LVSR, and this address is an aligned
18393 // multiple of that one. The results will be the same, so use the
18394 // one we've just found instead.
18395
18396 return SDValue(U, 0);
18397 }
18398 }
18399 }
18400
18401 if (isa<ConstantSDNode>(Val: Add->getOperand(Num: 1))) {
18402 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18403 for (SDNode *U : BasePtr->users()) {
18404 if (U->getOpcode() == ISD::ADD &&
18405 isa<ConstantSDNode>(Val: U->getOperand(Num: 1)) &&
18406 (Add->getConstantOperandVal(Num: 1) - U->getConstantOperandVal(Num: 1)) %
18407 (1ULL << Bits) ==
18408 0) {
18409 SDNode *OtherAdd = U;
18410 for (SDNode *V : OtherAdd->users()) {
18411 if (V->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18412 V->getConstantOperandVal(Num: 0) == IID) {
18413 return SDValue(V, 0);
18414 }
18415 }
18416 }
18417 }
18418 }
18419 }
18420
18421 // Combine vmaxsw/h/b(a, a's negation) to abs(a)
18422 // Expose the vabsduw/h/b opportunity for down stream
18423 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() &&
18424 (IID == Intrinsic::ppc_altivec_vmaxsw ||
18425 IID == Intrinsic::ppc_altivec_vmaxsh ||
18426 IID == Intrinsic::ppc_altivec_vmaxsb)) {
18427 SDValue V1 = N->getOperand(Num: 1);
18428 SDValue V2 = N->getOperand(Num: 2);
18429 if ((V1.getSimpleValueType() == MVT::v4i32 ||
18430 V1.getSimpleValueType() == MVT::v8i16 ||
18431 V1.getSimpleValueType() == MVT::v16i8) &&
18432 V1.getSimpleValueType() == V2.getSimpleValueType()) {
18433 // (0-a, a)
18434 if (V1.getOpcode() == ISD::SUB &&
18435 ISD::isBuildVectorAllZeros(N: V1.getOperand(i: 0).getNode()) &&
18436 V1.getOperand(i: 1) == V2) {
18437 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V2.getValueType(), Operand: V2);
18438 }
18439 // (a, 0-a)
18440 if (V2.getOpcode() == ISD::SUB &&
18441 ISD::isBuildVectorAllZeros(N: V2.getOperand(i: 0).getNode()) &&
18442 V2.getOperand(i: 1) == V1) {
18443 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18444 }
18445 // (x-y, y-x)
18446 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB &&
18447 V1.getOperand(i: 0) == V2.getOperand(i: 1) &&
18448 V1.getOperand(i: 1) == V2.getOperand(i: 0)) {
18449 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18450 }
18451 }
18452 }
18453 }
18454
18455 break;
18456 case ISD::INTRINSIC_W_CHAIN:
18457 switch (N->getConstantOperandVal(Num: 1)) {
18458 default:
18459 break;
18460 case Intrinsic::ppc_altivec_vsum4sbs:
18461 case Intrinsic::ppc_altivec_vsum4shs:
18462 case Intrinsic::ppc_altivec_vsum4ubs: {
18463 // These sum-across intrinsics only have a chain due to the side effect
18464 // that they may set the SAT bit. If we know the SAT bit will not be set
18465 // for some inputs, we can replace any uses of their chain with the
18466 // input chain.
18467 if (BuildVectorSDNode *BVN =
18468 dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 3))) {
18469 APInt APSplatBits, APSplatUndef;
18470 unsigned SplatBitSize;
18471 bool HasAnyUndefs;
18472 bool BVNIsConstantSplat = BVN->isConstantSplat(
18473 SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 0,
18474 isBigEndian: !Subtarget.isLittleEndian());
18475 // If the constant splat vector is 0, the SAT bit will not be set.
18476 if (BVNIsConstantSplat && APSplatBits == 0)
18477 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
18478 }
18479 return SDValue();
18480 }
18481 case Intrinsic::ppc_vsx_lxvw4x:
18482 case Intrinsic::ppc_vsx_lxvd2x:
18483 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18484 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18485 if (Subtarget.needsSwapsForVSXMemOps())
18486 return expandVSXLoadForLE(N, DCI);
18487 break;
18488 }
18489 break;
18490 case ISD::INTRINSIC_VOID:
18491 // For little endian, VSX stores require generating xxswapd/stxvd2x.
18492 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18493 if (Subtarget.needsSwapsForVSXMemOps()) {
18494 switch (N->getConstantOperandVal(Num: 1)) {
18495 default:
18496 break;
18497 case Intrinsic::ppc_vsx_stxvw4x:
18498 case Intrinsic::ppc_vsx_stxvd2x:
18499 return expandVSXStoreForLE(N, DCI);
18500 }
18501 }
18502 break;
18503 case ISD::BSWAP: {
18504 // Turn BSWAP (LOAD) -> lhbrx/lwbrx.
18505 // For subtargets without LDBRX, we can still do better than the default
18506 // expansion even for 64-bit BSWAP (LOAD).
18507 bool Is64BitBswapOn64BitTgt =
18508 Subtarget.isPPC64() && N->getValueType(ResNo: 0) == MVT::i64;
18509 bool IsSingleUseNormalLd = ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode()) &&
18510 N->getOperand(Num: 0).hasOneUse();
18511 if (IsSingleUseNormalLd &&
18512 (N->getValueType(ResNo: 0) == MVT::i32 || N->getValueType(ResNo: 0) == MVT::i16 ||
18513 (Subtarget.hasLDBRX() && Is64BitBswapOn64BitTgt))) {
18514 SDValue Load = N->getOperand(Num: 0);
18515 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
18516 // Create the byte-swapping load.
18517 SDValue Ops[] = {
18518 LD->getChain(), // Chain
18519 LD->getBasePtr(), // Ptr
18520 DAG.getValueType(N->getValueType(ResNo: 0)) // VT
18521 };
18522 SDValue BSLoad =
18523 DAG.getMemIntrinsicNode(Opcode: PPCISD::LBRX, dl,
18524 VTList: DAG.getVTList(VT1: N->getValueType(ResNo: 0) == MVT::i64 ?
18525 MVT::i64 : MVT::i32, VT2: MVT::Other),
18526 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
18527
18528 // If this is an i16 load, insert the truncate.
18529 SDValue ResVal = BSLoad;
18530 if (N->getValueType(ResNo: 0) == MVT::i16)
18531 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: BSLoad);
18532
18533 // First, combine the bswap away. This makes the value produced by the
18534 // load dead.
18535 DCI.CombineTo(N, Res: ResVal);
18536
18537 // Next, combine the load away, we give it a bogus result value but a real
18538 // chain result. The result value is dead because the bswap is dead.
18539 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
18540
18541 // Return N so it doesn't get rechecked!
18542 return SDValue(N, 0);
18543 }
18544 // Convert this to two 32-bit bswap loads and a BUILD_PAIR. Do this only
18545 // before legalization so that the BUILD_PAIR is handled correctly.
18546 if (!DCI.isBeforeLegalize() || !Is64BitBswapOn64BitTgt ||
18547 !IsSingleUseNormalLd)
18548 return SDValue();
18549 LoadSDNode *LD = cast<LoadSDNode>(Val: N->getOperand(Num: 0));
18550
18551 // Can't split volatile or atomic loads.
18552 if (!LD->isSimple())
18553 return SDValue();
18554 SDValue BasePtr = LD->getBasePtr();
18555 SDValue Lo = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18556 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign());
18557 Lo = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Lo);
18558 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18559 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18560 MachineMemOperand *NewMMO = DAG.getMachineFunction().getMachineMemOperand(
18561 MMO: LD->getMemOperand(), Offset: 4, Size: 4);
18562 SDValue Hi = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr, MMO: NewMMO);
18563 Hi = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Hi);
18564 SDValue Res;
18565 if (Subtarget.isLittleEndian())
18566 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Hi, N2: Lo);
18567 else
18568 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
18569 SDValue TF =
18570 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18571 N1: Hi.getOperand(i: 0).getValue(R: 1), N2: Lo.getOperand(i: 0).getValue(R: 1));
18572 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: TF);
18573 return Res;
18574 }
18575 case PPCISD::VCMP:
18576 // If a VCMP_rec node already exists with exactly the same operands as this
18577 // node, use its result instead of this node (VCMP_rec computes both a CR6
18578 // and a normal output).
18579 //
18580 if (!N->getOperand(Num: 0).hasOneUse() &&
18581 !N->getOperand(Num: 1).hasOneUse() &&
18582 !N->getOperand(Num: 2).hasOneUse()) {
18583
18584 // Scan all of the users of the LHS, looking for VCMP_rec's that match.
18585 SDNode *VCMPrecNode = nullptr;
18586
18587 SDNode *LHSN = N->getOperand(Num: 0).getNode();
18588 for (SDNode *User : LHSN->users())
18589 if (User->getOpcode() == PPCISD::VCMP_rec &&
18590 User->getOperand(Num: 1) == N->getOperand(Num: 1) &&
18591 User->getOperand(Num: 2) == N->getOperand(Num: 2) &&
18592 User->getOperand(Num: 0) == N->getOperand(Num: 0)) {
18593 VCMPrecNode = User;
18594 break;
18595 }
18596
18597 // If there is no VCMP_rec node, or if the flag value has a single use,
18598 // don't transform this.
18599 if (!VCMPrecNode || VCMPrecNode->hasNUsesOfValue(NUses: 0, Value: 1))
18600 break;
18601
18602 // Look at the (necessarily single) use of the flag value. If it has a
18603 // chain, this transformation is more complex. Note that multiple things
18604 // could use the value result, which we should ignore.
18605 SDNode *FlagUser = nullptr;
18606 for (SDNode::use_iterator UI = VCMPrecNode->use_begin();
18607 FlagUser == nullptr; ++UI) {
18608 assert(UI != VCMPrecNode->use_end() && "Didn't find user!");
18609 SDNode *User = UI->getUser();
18610 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
18611 if (User->getOperand(Num: i) == SDValue(VCMPrecNode, 1)) {
18612 FlagUser = User;
18613 break;
18614 }
18615 }
18616 }
18617
18618 // If the user is a MFOCRF instruction, we know this is safe.
18619 // Otherwise we give up for right now.
18620 if (FlagUser->getOpcode() == PPCISD::MFOCRF)
18621 return SDValue(VCMPrecNode, 0);
18622 }
18623 break;
18624 case ISD::BR_CC: {
18625 // If this is a branch on an altivec predicate comparison, lower this so
18626 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This
18627 // lowering is done pre-legalize, because the legalizer lowers the predicate
18628 // compare down to code that is difficult to reassemble.
18629 // This code also handles branches that depend on the result of a store
18630 // conditional.
18631 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N->getOperand(Num: 1))->get();
18632 SDValue LHS = N->getOperand(Num: 2), RHS = N->getOperand(Num: 3);
18633
18634 int CompareOpc;
18635 bool isDot;
18636
18637 if (!isa<ConstantSDNode>(Val: RHS) || (CC != ISD::SETEQ && CC != ISD::SETNE))
18638 break;
18639
18640 // Since we are doing this pre-legalize, the RHS can be a constant of
18641 // arbitrary bitwidth which may cause issues when trying to get the value
18642 // from the underlying APInt.
18643 auto RHSAPInt = RHS->getAsAPIntVal();
18644 if (!RHSAPInt.isIntN(N: 64))
18645 break;
18646
18647 unsigned Val = RHSAPInt.getZExtValue();
18648 auto isImpossibleCompare = [&]() {
18649 // If this is a comparison against something other than 0/1, then we know
18650 // that the condition is never/always true.
18651 if (Val != 0 && Val != 1) {
18652 if (CC == ISD::SETEQ) // Cond never true, remove branch.
18653 return N->getOperand(Num: 0);
18654 // Always !=, turn it into an unconditional branch.
18655 return DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other,
18656 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 4));
18657 }
18658 return SDValue();
18659 };
18660 // Combine branches fed by store conditional instructions (st[bhwd]cx).
18661 unsigned StoreWidth = 0;
18662 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
18663 isStoreConditional(Intrin: LHS, StoreWidth)) {
18664 if (SDValue Impossible = isImpossibleCompare())
18665 return Impossible;
18666 PPC::Predicate CompOpc;
18667 // eq 0 => ne
18668 // ne 0 => eq
18669 // eq 1 => eq
18670 // ne 1 => ne
18671 if (Val == 0)
18672 CompOpc = CC == ISD::SETEQ ? PPC::PRED_NE : PPC::PRED_EQ;
18673 else
18674 CompOpc = CC == ISD::SETEQ ? PPC::PRED_EQ : PPC::PRED_NE;
18675
18676 SDValue Ops[] = {LHS.getOperand(i: 0), LHS.getOperand(i: 2), LHS.getOperand(i: 3),
18677 DAG.getConstant(Val: StoreWidth, DL: dl, VT: MVT::i32)};
18678 auto *MemNode = cast<MemSDNode>(Val&: LHS);
18679 SDValue ConstSt = DAG.getMemIntrinsicNode(
18680 Opcode: PPCISD::STORE_COND, dl,
18681 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other, VT3: MVT::Glue), Ops,
18682 MemVT: MemNode->getMemoryVT(), MMO: MemNode->getMemOperand());
18683
18684 SDValue InChain;
18685 // Unchain the branch from the original store conditional.
18686 if (N->getOperand(Num: 0) == LHS.getValue(R: 1))
18687 InChain = LHS.getOperand(i: 0);
18688 else if (N->getOperand(Num: 0).getOpcode() == ISD::TokenFactor) {
18689 SmallVector<SDValue, 4> InChains;
18690 SDValue InTF = N->getOperand(Num: 0);
18691 for (int i = 0, e = InTF.getNumOperands(); i < e; i++)
18692 if (InTF.getOperand(i) != LHS.getValue(R: 1))
18693 InChains.push_back(Elt: InTF.getOperand(i));
18694 InChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: InChains);
18695 }
18696
18697 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: InChain,
18698 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18699 N3: DAG.getRegister(Reg: PPC::CR0, VT: MVT::i32), N4: N->getOperand(Num: 4),
18700 N5: ConstSt.getValue(R: 2));
18701 }
18702
18703 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18704 getVectorCompareInfo(Intrin: LHS, CompareOpc, isDot, Subtarget)) {
18705 assert(isDot && "Can't compare against a vector result!");
18706
18707 if (SDValue Impossible = isImpossibleCompare())
18708 return Impossible;
18709
18710 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0);
18711 // Create the PPCISD altivec 'dot' comparison node.
18712 SDValue Ops[] = {
18713 LHS.getOperand(i: 2), // LHS of compare
18714 LHS.getOperand(i: 3), // RHS of compare
18715 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
18716 };
18717 EVT VTs[] = { LHS.getOperand(i: 2).getValueType(), MVT::Glue };
18718 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
18719
18720 // Unpack the result based on how the target uses it.
18721 PPC::Predicate CompOpc;
18722 switch (LHS.getConstantOperandVal(i: 1)) {
18723 default: // Can't happen, don't crash on invalid number though.
18724 case 0: // Branch on the value of the EQ bit of CR6.
18725 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE;
18726 break;
18727 case 1: // Branch on the inverted value of the EQ bit of CR6.
18728 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ;
18729 break;
18730 case 2: // Branch on the value of the LT bit of CR6.
18731 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE;
18732 break;
18733 case 3: // Branch on the inverted value of the LT bit of CR6.
18734 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT;
18735 break;
18736 }
18737
18738 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
18739 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18740 N3: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32),
18741 N4: N->getOperand(Num: 4), N5: CompNode.getValue(R: 1));
18742 }
18743 break;
18744 }
18745 case ISD::BUILD_VECTOR:
18746 return DAGCombineBuildVector(N, DCI);
18747 case PPCISD::ADDC:
18748 return DAGCombineAddc(N, DCI);
18749 case PPCISD::SUBE:
18750 return DAGCombineSube(N, DCI);
18751
18752 case ISD::BITCAST:
18753 return DAGCombineBitcast(N, DCI);
18754 }
18755
18756 return SDValue();
18757}
18758
18759SDValue
18760PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
18761 SelectionDAG &DAG,
18762 SmallVectorImpl<SDNode *> &Created) const {
18763 // fold (sdiv X, pow2)
18764 EVT VT = N->getValueType(ResNo: 0);
18765 if (VT == MVT::i64 && !Subtarget.isPPC64())
18766 return SDValue();
18767 if ((VT != MVT::i32 && VT != MVT::i64) ||
18768 !(Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()))
18769 return SDValue();
18770
18771 SDLoc DL(N);
18772 SDValue N0 = N->getOperand(Num: 0);
18773
18774 bool IsNegPow2 = Divisor.isNegatedPowerOf2();
18775 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countr_zero();
18776 SDValue ShiftAmt = DAG.getConstant(Val: Lg2, DL, VT);
18777
18778 SDValue Op = DAG.getNode(Opcode: PPCISD::SRA_ADDZE, DL, VT, N1: N0, N2: ShiftAmt);
18779 Created.push_back(Elt: Op.getNode());
18780
18781 if (IsNegPow2) {
18782 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
18783 Created.push_back(Elt: Op.getNode());
18784 }
18785
18786 return Op;
18787}
18788
18789//===----------------------------------------------------------------------===//
18790// Inline Assembly Support
18791//===----------------------------------------------------------------------===//
18792
18793void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
18794 KnownBits &Known,
18795 const APInt &DemandedElts,
18796 const SelectionDAG &DAG,
18797 unsigned Depth) const {
18798 Known.resetAll();
18799 switch (Op.getOpcode()) {
18800 default: break;
18801 case PPCISD::LBRX: {
18802 // lhbrx is known to have the top bits cleared out.
18803 if (cast<VTSDNode>(Val: Op.getOperand(i: 2))->getVT() == MVT::i16)
18804 Known.Zero = 0xFFFF0000;
18805 break;
18806 }
18807 case PPCISD::ADDE: {
18808 if (Op.getResNo() == 0) {
18809 // (0|1), _ = ADDE 0, 0, CARRY
18810 SDValue LHS = Op.getOperand(i: 0);
18811 SDValue RHS = Op.getOperand(i: 1);
18812 if (isNullConstant(V: LHS) && isNullConstant(V: RHS))
18813 Known.Zero = ~1ULL;
18814 }
18815 break;
18816 }
18817 case ISD::INTRINSIC_WO_CHAIN: {
18818 switch (Op.getConstantOperandVal(i: 0)) {
18819 default: break;
18820 case Intrinsic::ppc_altivec_vcmpbfp_p:
18821 case Intrinsic::ppc_altivec_vcmpeqfp_p:
18822 case Intrinsic::ppc_altivec_vcmpequb_p:
18823 case Intrinsic::ppc_altivec_vcmpequh_p:
18824 case Intrinsic::ppc_altivec_vcmpequw_p:
18825 case Intrinsic::ppc_altivec_vcmpequd_p:
18826 case Intrinsic::ppc_altivec_vcmpequq_p:
18827 case Intrinsic::ppc_altivec_vcmpgefp_p:
18828 case Intrinsic::ppc_altivec_vcmpgtfp_p:
18829 case Intrinsic::ppc_altivec_vcmpgtsb_p:
18830 case Intrinsic::ppc_altivec_vcmpgtsh_p:
18831 case Intrinsic::ppc_altivec_vcmpgtsw_p:
18832 case Intrinsic::ppc_altivec_vcmpgtsd_p:
18833 case Intrinsic::ppc_altivec_vcmpgtsq_p:
18834 case Intrinsic::ppc_altivec_vcmpgtub_p:
18835 case Intrinsic::ppc_altivec_vcmpgtuh_p:
18836 case Intrinsic::ppc_altivec_vcmpgtuw_p:
18837 case Intrinsic::ppc_altivec_vcmpgtud_p:
18838 case Intrinsic::ppc_altivec_vcmpgtuq_p:
18839 Known.Zero = ~1U; // All bits but the low one are known to be zero.
18840 break;
18841 }
18842 break;
18843 }
18844 case ISD::INTRINSIC_W_CHAIN: {
18845 switch (Op.getConstantOperandVal(i: 1)) {
18846 default:
18847 break;
18848 case Intrinsic::ppc_load2r:
18849 // Top bits are cleared for load2r (which is the same as lhbrx).
18850 Known.Zero = 0xFFFF0000;
18851 break;
18852 }
18853 break;
18854 }
18855 }
18856}
18857
18858Align PPCTargetLowering::getPrefLoopAlignment(MachineLoop *ML) const {
18859 switch (Subtarget.getCPUDirective()) {
18860 default: break;
18861 case PPC::DIR_970:
18862 case PPC::DIR_PWR4:
18863 case PPC::DIR_PWR5:
18864 case PPC::DIR_PWR5X:
18865 case PPC::DIR_PWR6:
18866 case PPC::DIR_PWR6X:
18867 case PPC::DIR_PWR7:
18868 case PPC::DIR_PWR8:
18869 case PPC::DIR_PWR9:
18870 case PPC::DIR_PWR10:
18871 case PPC::DIR_PWR11:
18872 case PPC::DIR_PWR_FUTURE: {
18873 if (!ML)
18874 break;
18875
18876 if (!DisableInnermostLoopAlign32) {
18877 // If the nested loop is an innermost loop, prefer to a 32-byte alignment,
18878 // so that we can decrease cache misses and branch-prediction misses.
18879 // Actual alignment of the loop will depend on the hotness check and other
18880 // logic in alignBlocks.
18881 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty())
18882 return Align(32);
18883 }
18884
18885 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
18886
18887 // For small loops (between 5 and 8 instructions), align to a 32-byte
18888 // boundary so that the entire loop fits in one instruction-cache line.
18889 uint64_t LoopSize = 0;
18890 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I)
18891 for (const MachineInstr &J : **I) {
18892 LoopSize += TII->getInstSizeInBytes(MI: J);
18893 if (LoopSize > 32)
18894 break;
18895 }
18896
18897 if (LoopSize > 16 && LoopSize <= 32)
18898 return Align(32);
18899
18900 break;
18901 }
18902 }
18903
18904 return TargetLowering::getPrefLoopAlignment(ML);
18905}
18906
18907/// getConstraintType - Given a constraint, return the type of
18908/// constraint it is for this target.
18909PPCTargetLowering::ConstraintType
18910PPCTargetLowering::getConstraintType(StringRef Constraint) const {
18911 if (Constraint.size() == 1) {
18912 switch (Constraint[0]) {
18913 default: break;
18914 case 'b':
18915 case 'r':
18916 case 'f':
18917 case 'd':
18918 case 'v':
18919 case 'y':
18920 return C_RegisterClass;
18921 case 'Z':
18922 // FIXME: While Z does indicate a memory constraint, it specifically
18923 // indicates an r+r address (used in conjunction with the 'y' modifier
18924 // in the replacement string). Currently, we're forcing the base
18925 // register to be r0 in the asm printer (which is interpreted as zero)
18926 // and forming the complete address in the second register. This is
18927 // suboptimal.
18928 return C_Memory;
18929 }
18930 } else if (Constraint == "wc") { // individual CR bits.
18931 return C_RegisterClass;
18932 } else if (Constraint == "wa" || Constraint == "wd" ||
18933 Constraint == "wf" || Constraint == "ws" ||
18934 Constraint == "wi" || Constraint == "ww") {
18935 return C_RegisterClass; // VSX registers.
18936 }
18937 return TargetLowering::getConstraintType(Constraint);
18938}
18939
18940/// Examine constraint type and operand type and determine a weight value.
18941/// This object must already have been set up with the operand type
18942/// and the current alternative constraint selected.
18943TargetLowering::ConstraintWeight
18944PPCTargetLowering::getSingleConstraintMatchWeight(
18945 AsmOperandInfo &info, const char *constraint) const {
18946 ConstraintWeight weight = CW_Invalid;
18947 Value *CallOperandVal = info.CallOperandVal;
18948 // If we don't have a value, we can't do a match,
18949 // but allow it at the lowest weight.
18950 if (!CallOperandVal)
18951 return CW_Default;
18952 Type *type = CallOperandVal->getType();
18953
18954 // Look at the constraint type.
18955 if (StringRef(constraint) == "wc" && type->isIntegerTy(BitWidth: 1))
18956 return CW_Register; // an individual CR bit.
18957 else if ((StringRef(constraint) == "wa" ||
18958 StringRef(constraint) == "wd" ||
18959 StringRef(constraint) == "wf") &&
18960 type->isVectorTy())
18961 return CW_Register;
18962 else if (StringRef(constraint) == "wi" && type->isIntegerTy(BitWidth: 64))
18963 return CW_Register; // just hold 64-bit integers data.
18964 else if (StringRef(constraint) == "ws" && type->isDoubleTy())
18965 return CW_Register;
18966 else if (StringRef(constraint) == "ww" && type->isFloatTy())
18967 return CW_Register;
18968
18969 switch (*constraint) {
18970 default:
18971 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
18972 break;
18973 case 'b':
18974 if (type->isIntegerTy())
18975 weight = CW_Register;
18976 break;
18977 case 'f':
18978 if (type->isFloatTy())
18979 weight = CW_Register;
18980 break;
18981 case 'd':
18982 if (type->isDoubleTy())
18983 weight = CW_Register;
18984 break;
18985 case 'v':
18986 if (type->isVectorTy())
18987 weight = CW_Register;
18988 break;
18989 case 'y':
18990 weight = CW_Register;
18991 break;
18992 case 'Z':
18993 weight = CW_Memory;
18994 break;
18995 }
18996 return weight;
18997}
18998
18999std::pair<unsigned, const TargetRegisterClass *>
19000PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
19001 StringRef Constraint,
19002 MVT VT) const {
19003 if (Constraint.size() == 1) {
19004 // GCC RS6000 Constraint Letters
19005 switch (Constraint[0]) {
19006 case 'b': // R1-R31
19007 if (VT == MVT::i64 && Subtarget.isPPC64())
19008 return std::make_pair(x: 0U, y: &PPC::G8RC_NOX0RegClass);
19009 return std::make_pair(x: 0U, y: &PPC::GPRC_NOR0RegClass);
19010 case 'r': // R0-R31
19011 if (VT == MVT::i64 && Subtarget.isPPC64())
19012 return std::make_pair(x: 0U, y: &PPC::G8RCRegClass);
19013 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
19014 // 'd' and 'f' constraints are both defined to be "the floating point
19015 // registers", where one is for 32-bit and the other for 64-bit. We don't
19016 // really care overly much here so just give them all the same reg classes.
19017 case 'd':
19018 case 'f':
19019 if (Subtarget.hasSPE()) {
19020 if (VT == MVT::f32 || VT == MVT::i32)
19021 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
19022 if (VT == MVT::f64 || VT == MVT::i64)
19023 return std::make_pair(x: 0U, y: &PPC::SPERCRegClass);
19024 } else {
19025 if (VT == MVT::f32 || VT == MVT::i32)
19026 return std::make_pair(x: 0U, y: &PPC::F4RCRegClass);
19027 if (VT == MVT::f64 || VT == MVT::i64)
19028 return std::make_pair(x: 0U, y: &PPC::F8RCRegClass);
19029 }
19030 break;
19031 case 'v':
19032 if (Subtarget.hasAltivec() && VT.isVector())
19033 return std::make_pair(x: 0U, y: &PPC::VRRCRegClass);
19034 else if (Subtarget.hasVSX())
19035 // Scalars in Altivec registers only make sense with VSX.
19036 return std::make_pair(x: 0U, y: &PPC::VFRCRegClass);
19037 break;
19038 case 'y': // crrc
19039 return std::make_pair(x: 0U, y: &PPC::CRRCRegClass);
19040 }
19041 } else if (Constraint == "wc" && Subtarget.useCRBits()) {
19042 // An individual CR bit.
19043 return std::make_pair(x: 0U, y: &PPC::CRBITRCRegClass);
19044 } else if ((Constraint == "wa" || Constraint == "wd" ||
19045 Constraint == "wf" || Constraint == "wi") &&
19046 Subtarget.hasVSX()) {
19047 // A VSX register for either a scalar (FP) or vector. There is no
19048 // support for single precision scalars on subtargets prior to Power8.
19049 if (VT.isVector())
19050 return std::make_pair(x: 0U, y: &PPC::VSRCRegClass);
19051 if (VT == MVT::f32 && Subtarget.hasP8Vector())
19052 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
19053 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
19054 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) {
19055 if (VT == MVT::f32 && Subtarget.hasP8Vector())
19056 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
19057 else
19058 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
19059 } else if (Constraint == "lr") {
19060 if (VT == MVT::i64)
19061 return std::make_pair(x: 0U, y: &PPC::LR8RCRegClass);
19062 else
19063 return std::make_pair(x: 0U, y: &PPC::LRRCRegClass);
19064 }
19065
19066 // Handle special cases of physical registers that are not properly handled
19067 // by the base class.
19068 if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') {
19069 // If we name a VSX register, we can't defer to the base class because it
19070 // will not recognize the correct register (their names will be VSL{0-31}
19071 // and V{0-31} so they won't match). So we match them here.
19072 if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') {
19073 int VSNum = atoi(nptr: Constraint.data() + 3);
19074 assert(VSNum >= 0 && VSNum <= 63 &&
19075 "Attempted to access a vsr out of range");
19076 if (VSNum < 32)
19077 return std::make_pair(x: PPC::VSL0 + VSNum, y: &PPC::VSRCRegClass);
19078 return std::make_pair(x: PPC::V0 + VSNum - 32, y: &PPC::VSRCRegClass);
19079 }
19080
19081 // For float registers, we can't defer to the base class as it will match
19082 // the SPILLTOVSRRC class.
19083 if (Constraint.size() > 3 && Constraint[1] == 'f') {
19084 int RegNum = atoi(nptr: Constraint.data() + 2);
19085 if (RegNum > 31 || RegNum < 0)
19086 report_fatal_error(reason: "Invalid floating point register number");
19087 if (VT == MVT::f32 || VT == MVT::i32)
19088 return Subtarget.hasSPE()
19089 ? std::make_pair(x: PPC::R0 + RegNum, y: &PPC::GPRCRegClass)
19090 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F4RCRegClass);
19091 if (VT == MVT::f64 || VT == MVT::i64)
19092 return Subtarget.hasSPE()
19093 ? std::make_pair(x: PPC::S0 + RegNum, y: &PPC::SPERCRegClass)
19094 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F8RCRegClass);
19095 }
19096 }
19097
19098 std::pair<unsigned, const TargetRegisterClass *> R =
19099 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19100
19101 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers
19102 // (which we call X[0-9]+). If a 64-bit value has been requested, and a
19103 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent
19104 // register.
19105 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use
19106 // the AsmName field from *RegisterInfo.td, then this would not be necessary.
19107 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() &&
19108 PPC::GPRCRegClass.contains(Reg: R.first))
19109 return std::make_pair(x: TRI->getMatchingSuperReg(Reg: R.first,
19110 SubIdx: PPC::sub_32, RC: &PPC::G8RCRegClass),
19111 y: &PPC::G8RCRegClass);
19112
19113 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same.
19114 if (!R.second && StringRef("{cc}").equals_insensitive(RHS: Constraint)) {
19115 R.first = PPC::CR0;
19116 R.second = &PPC::CRRCRegClass;
19117 }
19118 // FIXME: This warning should ideally be emitted in the front end.
19119 if (Subtarget.isAIXABI() && !Subtarget.isAIXExtendedAltivecABI()) {
19120 if (((R.first >= PPC::V20 && R.first <= PPC::V31) ||
19121 (R.first >= PPC::VF20 && R.first <= PPC::VF31)) &&
19122 (R.second == &PPC::VSRCRegClass || R.second == &PPC::VSFRCRegClass))
19123 errs() << "warning: vector registers 20 to 32 are reserved in the "
19124 "default AIX AltiVec ABI and cannot be used\n";
19125 }
19126
19127 return R;
19128}
19129
19130/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
19131/// vector. If it is invalid, don't add anything to Ops.
19132void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
19133 StringRef Constraint,
19134 std::vector<SDValue> &Ops,
19135 SelectionDAG &DAG) const {
19136 SDValue Result;
19137
19138 // Only support length 1 constraints.
19139 if (Constraint.size() > 1)
19140 return;
19141
19142 char Letter = Constraint[0];
19143 switch (Letter) {
19144 default: break;
19145 case 'I':
19146 case 'J':
19147 case 'K':
19148 case 'L':
19149 case 'M':
19150 case 'N':
19151 case 'O':
19152 case 'P': {
19153 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Val&: Op);
19154 if (!CST) return; // Must be an immediate to match.
19155 SDLoc dl(Op);
19156 int64_t Value = CST->getSExtValue();
19157 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative
19158 // numbers are printed as such.
19159 switch (Letter) {
19160 default: llvm_unreachable("Unknown constraint letter!");
19161 case 'I': // "I" is a signed 16-bit constant.
19162 if (isInt<16>(x: Value))
19163 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19164 break;
19165 case 'J': // "J" is a constant with only the high-order 16 bits nonzero.
19166 if (isShiftedUInt<16, 16>(x: Value))
19167 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19168 break;
19169 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits.
19170 if (isShiftedInt<16, 16>(x: Value))
19171 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19172 break;
19173 case 'K': // "K" is a constant with only the low-order 16 bits nonzero.
19174 if (isUInt<16>(x: Value))
19175 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19176 break;
19177 case 'M': // "M" is a constant that is greater than 31.
19178 if (Value > 31)
19179 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19180 break;
19181 case 'N': // "N" is a positive constant that is an exact power of two.
19182 if (Value > 0 && isPowerOf2_64(Value))
19183 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19184 break;
19185 case 'O': // "O" is the constant zero.
19186 if (Value == 0)
19187 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19188 break;
19189 case 'P': // "P" is a constant whose negation is a signed 16-bit constant.
19190 if (isInt<16>(x: -Value))
19191 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19192 break;
19193 }
19194 break;
19195 }
19196 }
19197
19198 if (Result.getNode()) {
19199 Ops.push_back(x: Result);
19200 return;
19201 }
19202
19203 // Handle standard constraint letters.
19204 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
19205}
19206
19207void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I,
19208 SmallVectorImpl<SDValue> &Ops,
19209 SelectionDAG &DAG) const {
19210 if (I.getNumOperands() <= 1)
19211 return;
19212 if (!isa<ConstantSDNode>(Val: Ops[1].getNode()))
19213 return;
19214 auto IntrinsicID = Ops[1].getNode()->getAsZExtVal();
19215 if (IntrinsicID != Intrinsic::ppc_tdw && IntrinsicID != Intrinsic::ppc_tw &&
19216 IntrinsicID != Intrinsic::ppc_trapd && IntrinsicID != Intrinsic::ppc_trap)
19217 return;
19218
19219 if (MDNode *MDN = I.getMetadata(KindID: LLVMContext::MD_annotation))
19220 Ops.push_back(Elt: DAG.getMDNode(MD: MDN));
19221}
19222
19223// isLegalAddressingMode - Return true if the addressing mode represented
19224// by AM is legal for this target, for a load/store of the specified type.
19225bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL,
19226 const AddrMode &AM, Type *Ty,
19227 unsigned AS,
19228 Instruction *I) const {
19229 // Vector type r+i form is supported since power9 as DQ form. We don't check
19230 // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC,
19231 // imm form is preferred and the offset can be adjusted to use imm form later
19232 // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and
19233 // max offset to check legal addressing mode, we should be a little aggressive
19234 // to contain other offsets for that LSRUse.
19235 if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector())
19236 return false;
19237
19238 // PPC allows a sign-extended 16-bit immediate field.
19239 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
19240 return false;
19241
19242 // No global is ever allowed as a base.
19243 if (AM.BaseGV)
19244 return false;
19245
19246 // PPC only support r+r,
19247 switch (AM.Scale) {
19248 case 0: // "r+i" or just "i", depending on HasBaseReg.
19249 break;
19250 case 1:
19251 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
19252 return false;
19253 // Otherwise we have r+r or r+i.
19254 break;
19255 case 2:
19256 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
19257 return false;
19258 // Allow 2*r as r+r.
19259 break;
19260 default:
19261 // No other scales are supported.
19262 return false;
19263 }
19264
19265 return true;
19266}
19267
19268SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
19269 SelectionDAG &DAG) const {
19270 MachineFunction &MF = DAG.getMachineFunction();
19271 MachineFrameInfo &MFI = MF.getFrameInfo();
19272 MFI.setReturnAddressIsTaken(true);
19273
19274 SDLoc dl(Op);
19275 unsigned Depth = Op.getConstantOperandVal(i: 0);
19276
19277 // Make sure the function does not optimize away the store of the RA to
19278 // the stack.
19279 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
19280 FuncInfo->setLRStoreRequired();
19281 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
19282
19283 if (Depth > 0) {
19284 // The link register (return address) is saved in the caller's frame
19285 // not the callee's stack frame. So we must get the caller's frame
19286 // address and load the return address at the LR offset from there.
19287 SDValue FrameAddr =
19288 DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19289 Ptr: LowerFRAMEADDR(Op, DAG), PtrInfo: MachinePointerInfo());
19290 SDValue Offset =
19291 DAG.getConstant(Val: Subtarget.getFrameLowering()->getReturnSaveOffset(), DL: dl,
19292 VT: Subtarget.getScalarIntVT());
19293 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(),
19294 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FrameAddr, N2: Offset),
19295 PtrInfo: MachinePointerInfo());
19296 }
19297
19298 // Just load the return address off the stack.
19299 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
19300 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: RetAddrFI,
19301 PtrInfo: MachinePointerInfo());
19302}
19303
19304SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
19305 SelectionDAG &DAG) const {
19306 SDLoc dl(Op);
19307 unsigned Depth = Op.getConstantOperandVal(i: 0);
19308
19309 MachineFunction &MF = DAG.getMachineFunction();
19310 MachineFrameInfo &MFI = MF.getFrameInfo();
19311 MFI.setFrameAddressIsTaken(true);
19312
19313 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
19314 bool isPPC64 = PtrVT == MVT::i64;
19315
19316 // Naked functions never have a frame pointer, and so we use r1. For all
19317 // other functions, this decision must be delayed until during PEI.
19318 unsigned FrameReg;
19319 if (MF.getFunction().hasFnAttribute(Kind: Attribute::Naked))
19320 FrameReg = isPPC64 ? PPC::X1 : PPC::R1;
19321 else
19322 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP;
19323
19324 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: FrameReg,
19325 VT: PtrVT);
19326 while (Depth--)
19327 FrameAddr = DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19328 Ptr: FrameAddr, PtrInfo: MachinePointerInfo());
19329 return FrameAddr;
19330}
19331
19332#define GET_REGISTER_MATCHER
19333#include "PPCGenAsmMatcher.inc"
19334
19335Register PPCTargetLowering::getRegisterByName(const char *RegName, LLT VT,
19336 const MachineFunction &MF) const {
19337 bool IsPPC64 = Subtarget.isPPC64();
19338
19339 bool Is64Bit = IsPPC64 && VT == LLT::scalar(SizeInBits: 64);
19340 if (!Is64Bit && VT != LLT::scalar(SizeInBits: 32))
19341 report_fatal_error(reason: "Invalid register global variable type");
19342
19343 Register Reg = MatchRegisterName(Name: RegName);
19344 if (!Reg)
19345 return Reg;
19346
19347 // FIXME: Unable to generate code for `-O2` but okay for `-O0`.
19348 // Need followup investigation as to why.
19349 if ((IsPPC64 && Reg == PPC::R2) || Reg == PPC::R0)
19350 report_fatal_error(reason: Twine("Trying to reserve an invalid register \"" +
19351 StringRef(RegName) + "\"."));
19352
19353 // Convert GPR to GP8R register for 64bit.
19354 if (Is64Bit && StringRef(RegName).starts_with_insensitive(Prefix: "r"))
19355 Reg = Reg.id() - PPC::R0 + PPC::X0;
19356
19357 return Reg;
19358}
19359
19360bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
19361 // 32-bit SVR4 ABI access everything as got-indirect.
19362 if (Subtarget.is32BitELFABI())
19363 return true;
19364
19365 // AIX accesses everything indirectly through the TOC, which is similar to
19366 // the GOT.
19367 if (Subtarget.isAIXABI())
19368 return true;
19369
19370 CodeModel::Model CModel = getTargetMachine().getCodeModel();
19371 // If it is small or large code model, module locals are accessed
19372 // indirectly by loading their address from .toc/.got.
19373 if (CModel == CodeModel::Small || CModel == CodeModel::Large)
19374 return true;
19375
19376 // JumpTable and BlockAddress are accessed as got-indirect.
19377 if (isa<JumpTableSDNode>(Val: GA) || isa<BlockAddressSDNode>(Val: GA))
19378 return true;
19379
19380 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: GA))
19381 return Subtarget.isGVIndirectSymbol(GV: G->getGlobal());
19382
19383 return false;
19384}
19385
19386bool
19387PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
19388 // The PowerPC target isn't yet aware of offsets.
19389 return false;
19390}
19391
19392void PPCTargetLowering::getTgtMemIntrinsic(
19393 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
19394 MachineFunction &MF, unsigned Intrinsic) const {
19395 IntrinsicInfo Info;
19396 switch (Intrinsic) {
19397 case Intrinsic::ppc_atomicrmw_xchg_i128:
19398 case Intrinsic::ppc_atomicrmw_add_i128:
19399 case Intrinsic::ppc_atomicrmw_sub_i128:
19400 case Intrinsic::ppc_atomicrmw_nand_i128:
19401 case Intrinsic::ppc_atomicrmw_and_i128:
19402 case Intrinsic::ppc_atomicrmw_or_i128:
19403 case Intrinsic::ppc_atomicrmw_xor_i128:
19404 case Intrinsic::ppc_cmpxchg_i128:
19405 Info.opc = ISD::INTRINSIC_W_CHAIN;
19406 Info.memVT = MVT::i128;
19407 Info.ptrVal = I.getArgOperand(i: 0);
19408 Info.offset = 0;
19409 Info.align = Align(16);
19410 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
19411 MachineMemOperand::MOVolatile;
19412 Infos.push_back(Elt: Info);
19413 return;
19414 case Intrinsic::ppc_atomic_load_i128:
19415 Info.opc = ISD::INTRINSIC_W_CHAIN;
19416 Info.memVT = MVT::i128;
19417 Info.ptrVal = I.getArgOperand(i: 0);
19418 Info.offset = 0;
19419 Info.align = Align(16);
19420 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
19421 Infos.push_back(Elt: Info);
19422 return;
19423 case Intrinsic::ppc_atomic_store_i128:
19424 Info.opc = ISD::INTRINSIC_VOID;
19425 Info.memVT = MVT::i128;
19426 Info.ptrVal = I.getArgOperand(i: 2);
19427 Info.offset = 0;
19428 Info.align = Align(16);
19429 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19430 Infos.push_back(Elt: Info);
19431 return;
19432 case Intrinsic::ppc_altivec_lvx:
19433 case Intrinsic::ppc_altivec_lvxl:
19434 case Intrinsic::ppc_altivec_lvebx:
19435 case Intrinsic::ppc_altivec_lvehx:
19436 case Intrinsic::ppc_altivec_lvewx:
19437 case Intrinsic::ppc_vsx_lxvd2x:
19438 case Intrinsic::ppc_vsx_lxvw4x:
19439 case Intrinsic::ppc_vsx_lxvd2x_be:
19440 case Intrinsic::ppc_vsx_lxvw4x_be:
19441 case Intrinsic::ppc_vsx_lxvl:
19442 case Intrinsic::ppc_vsx_lxvll: {
19443 EVT VT;
19444 switch (Intrinsic) {
19445 case Intrinsic::ppc_altivec_lvebx:
19446 VT = MVT::i8;
19447 break;
19448 case Intrinsic::ppc_altivec_lvehx:
19449 VT = MVT::i16;
19450 break;
19451 case Intrinsic::ppc_altivec_lvewx:
19452 VT = MVT::i32;
19453 break;
19454 case Intrinsic::ppc_vsx_lxvd2x:
19455 case Intrinsic::ppc_vsx_lxvd2x_be:
19456 VT = MVT::v2f64;
19457 break;
19458 default:
19459 VT = MVT::v4i32;
19460 break;
19461 }
19462
19463 Info.opc = ISD::INTRINSIC_W_CHAIN;
19464 Info.memVT = VT;
19465 Info.ptrVal = I.getArgOperand(i: 0);
19466 Info.offset = -VT.getStoreSize()+1;
19467 Info.size = 2*VT.getStoreSize()-1;
19468 Info.align = Align(1);
19469 Info.flags = MachineMemOperand::MOLoad;
19470 Infos.push_back(Elt: Info);
19471 return;
19472 }
19473 case Intrinsic::ppc_altivec_stvx:
19474 case Intrinsic::ppc_altivec_stvxl:
19475 case Intrinsic::ppc_altivec_stvebx:
19476 case Intrinsic::ppc_altivec_stvehx:
19477 case Intrinsic::ppc_altivec_stvewx:
19478 case Intrinsic::ppc_vsx_stxvd2x:
19479 case Intrinsic::ppc_vsx_stxvw4x:
19480 case Intrinsic::ppc_vsx_stxvd2x_be:
19481 case Intrinsic::ppc_vsx_stxvw4x_be:
19482 case Intrinsic::ppc_vsx_stxvl:
19483 case Intrinsic::ppc_vsx_stxvll: {
19484 EVT VT;
19485 switch (Intrinsic) {
19486 case Intrinsic::ppc_altivec_stvebx:
19487 VT = MVT::i8;
19488 break;
19489 case Intrinsic::ppc_altivec_stvehx:
19490 VT = MVT::i16;
19491 break;
19492 case Intrinsic::ppc_altivec_stvewx:
19493 VT = MVT::i32;
19494 break;
19495 case Intrinsic::ppc_vsx_stxvd2x:
19496 case Intrinsic::ppc_vsx_stxvd2x_be:
19497 VT = MVT::v2f64;
19498 break;
19499 default:
19500 VT = MVT::v4i32;
19501 break;
19502 }
19503
19504 Info.opc = ISD::INTRINSIC_VOID;
19505 Info.memVT = VT;
19506 Info.ptrVal = I.getArgOperand(i: 1);
19507 Info.offset = -VT.getStoreSize()+1;
19508 Info.size = 2*VT.getStoreSize()-1;
19509 Info.align = Align(1);
19510 Info.flags = MachineMemOperand::MOStore;
19511 Infos.push_back(Elt: Info);
19512 return;
19513 }
19514 case Intrinsic::ppc_stdcx:
19515 case Intrinsic::ppc_stwcx:
19516 case Intrinsic::ppc_sthcx:
19517 case Intrinsic::ppc_stbcx: {
19518 EVT VT;
19519 auto Alignment = Align(8);
19520 switch (Intrinsic) {
19521 case Intrinsic::ppc_stdcx:
19522 VT = MVT::i64;
19523 break;
19524 case Intrinsic::ppc_stwcx:
19525 VT = MVT::i32;
19526 Alignment = Align(4);
19527 break;
19528 case Intrinsic::ppc_sthcx:
19529 VT = MVT::i16;
19530 Alignment = Align(2);
19531 break;
19532 case Intrinsic::ppc_stbcx:
19533 VT = MVT::i8;
19534 Alignment = Align(1);
19535 break;
19536 }
19537 Info.opc = ISD::INTRINSIC_W_CHAIN;
19538 Info.memVT = VT;
19539 Info.ptrVal = I.getArgOperand(i: 0);
19540 Info.offset = 0;
19541 Info.align = Alignment;
19542 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19543 Infos.push_back(Elt: Info);
19544 return;
19545 }
19546 default:
19547 break;
19548 }
19549}
19550
19551/// It returns EVT::Other if the type should be determined using generic
19552/// target-independent logic.
19553EVT PPCTargetLowering::getOptimalMemOpType(
19554 LLVMContext &Context, const MemOp &Op,
19555 const AttributeList &FuncAttributes) const {
19556 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None) {
19557 // We should use Altivec/VSX loads and stores when available. For unaligned
19558 // addresses, unaligned VSX loads are only fast starting with the P8.
19559 if (Subtarget.hasAltivec() && Op.size() >= 16) {
19560 if (Op.isMemset() && Subtarget.hasVSX()) {
19561 uint64_t TailSize = Op.size() % 16;
19562 // For memset lowering, EXTRACT_VECTOR_ELT tries to return constant
19563 // element if vector element type matches tail store. For tail size
19564 // 3/4, the tail store is i32, v4i32 cannot be used, need a legal one.
19565 if (TailSize > 2 && TailSize <= 4) {
19566 return MVT::v8i16;
19567 }
19568 return MVT::v4i32;
19569 }
19570 if (Op.isAligned(AlignCheck: Align(16)) || Subtarget.hasP8Vector())
19571 return MVT::v4i32;
19572 }
19573 }
19574
19575 if (Subtarget.isPPC64()) {
19576 return MVT::i64;
19577 }
19578
19579 return MVT::i32;
19580}
19581
19582/// Returns true if it is beneficial to convert a load of a constant
19583/// to just the constant itself.
19584bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
19585 Type *Ty) const {
19586 assert(Ty->isIntegerTy());
19587
19588 unsigned BitSize = Ty->getPrimitiveSizeInBits();
19589 return !(BitSize == 0 || BitSize > 64);
19590}
19591
19592bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
19593 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19594 return false;
19595 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
19596 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
19597 return NumBits1 == 64 && NumBits2 == 32;
19598}
19599
19600bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
19601 if (!VT1.isInteger() || !VT2.isInteger())
19602 return false;
19603 unsigned NumBits1 = VT1.getSizeInBits();
19604 unsigned NumBits2 = VT2.getSizeInBits();
19605 return NumBits1 == 64 && NumBits2 == 32;
19606}
19607
19608bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
19609 // Generally speaking, zexts are not free, but they are free when they can be
19610 // folded with other operations.
19611 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) {
19612 EVT MemVT = LD->getMemoryVT();
19613 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 ||
19614 (Subtarget.isPPC64() && MemVT == MVT::i32)) &&
19615 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
19616 LD->getExtensionType() == ISD::ZEXTLOAD))
19617 return true;
19618 }
19619
19620 // FIXME: Add other cases...
19621 // - 32-bit shifts with a zext to i64
19622 // - zext after ctlz, bswap, etc.
19623 // - zext after and by a constant mask
19624
19625 return TargetLowering::isZExtFree(Val, VT2);
19626}
19627
19628bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const {
19629 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
19630 "invalid fpext types");
19631 // Extending to float128 is not free.
19632 if (DestVT == MVT::f128)
19633 return false;
19634 return true;
19635}
19636
19637bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
19638 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19639}
19640
19641bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const {
19642 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19643}
19644
19645bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
19646 MachineMemOperand::Flags,
19647 unsigned *Fast) const {
19648 if (DisablePPCUnaligned)
19649 return false;
19650
19651 // PowerPC supports unaligned memory access for simple non-vector types.
19652 // Although accessing unaligned addresses is not as efficient as accessing
19653 // aligned addresses, it is generally more efficient than manual expansion,
19654 // and generally only traps for software emulation when crossing page
19655 // boundaries.
19656
19657 if (!VT.isSimple())
19658 return false;
19659
19660 if (VT.isFloatingPoint() && !VT.isVector() &&
19661 !Subtarget.allowsUnalignedFPAccess())
19662 return false;
19663
19664 if (VT.getSimpleVT().isVector()) {
19665 if (Subtarget.hasVSX()) {
19666 if (VT != MVT::v2f64 && VT != MVT::v2i64 &&
19667 VT != MVT::v4f32 && VT != MVT::v4i32)
19668 return false;
19669 } else {
19670 return false;
19671 }
19672 }
19673
19674 if (VT == MVT::ppcf128)
19675 return false;
19676
19677 if (Fast)
19678 *Fast = 1;
19679
19680 return true;
19681}
19682
19683bool PPCTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
19684 SDValue C) const {
19685 // Check integral scalar types.
19686 if (!VT.isScalarInteger())
19687 return false;
19688 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
19689 if (!ConstNode->getAPIntValue().isSignedIntN(N: 64))
19690 return false;
19691 // This transformation will generate >= 2 operations. But the following
19692 // cases will generate <= 2 instructions during ISEL. So exclude them.
19693 // 1. If the constant multiplier fits 16 bits, it can be handled by one
19694 // HW instruction, ie. MULLI
19695 // 2. If the multiplier after shifted fits 16 bits, an extra shift
19696 // instruction is needed than case 1, ie. MULLI and RLDICR
19697 int64_t Imm = ConstNode->getSExtValue();
19698 unsigned Shift = llvm::countr_zero<uint64_t>(Val: Imm);
19699 Imm >>= Shift;
19700 if (isInt<16>(x: Imm))
19701 return false;
19702 uint64_t UImm = static_cast<uint64_t>(Imm);
19703 if (isPowerOf2_64(Value: UImm + 1) || isPowerOf2_64(Value: UImm - 1) ||
19704 isPowerOf2_64(Value: 1 - UImm) || isPowerOf2_64(Value: -1 - UImm))
19705 return true;
19706 }
19707 return false;
19708}
19709
19710bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19711 EVT VT) const {
19712 return isFMAFasterThanFMulAndFAdd(
19713 F: MF.getFunction(), Ty: VT.getTypeForEVT(Context&: MF.getFunction().getContext()));
19714}
19715
19716bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
19717 Type *Ty) const {
19718 if (Subtarget.hasSPE() || Subtarget.useSoftFloat())
19719 return false;
19720 switch (Ty->getScalarType()->getTypeID()) {
19721 case Type::FloatTyID:
19722 case Type::DoubleTyID:
19723 return true;
19724 case Type::FP128TyID:
19725 return Subtarget.hasP9Vector();
19726 default:
19727 return false;
19728 }
19729}
19730
19731// FIXME: add more patterns which are not profitable to hoist.
19732bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const {
19733 if (!I->hasOneUse())
19734 return true;
19735
19736 Instruction *User = I->user_back();
19737 assert(User && "A single use instruction with no uses.");
19738
19739 switch (I->getOpcode()) {
19740 case Instruction::FMul: {
19741 // Don't break FMA, PowerPC prefers FMA.
19742 if (User->getOpcode() != Instruction::FSub &&
19743 User->getOpcode() != Instruction::FAdd)
19744 return true;
19745
19746 const Function *F = I->getFunction();
19747 const DataLayout &DL = F->getDataLayout();
19748 Type *Ty = User->getOperand(i: 0)->getType();
19749 bool AllowContract = I->getFastMathFlags().allowContract() &&
19750 User->getFastMathFlags().allowContract();
19751
19752 return !(isFMAFasterThanFMulAndFAdd(F: *F, Ty) &&
19753 isOperationLegalOrCustom(Op: ISD::FMA, VT: getValueType(DL, Ty)) &&
19754 AllowContract);
19755 }
19756 case Instruction::Load: {
19757 // Don't break "store (load float*)" pattern, this pattern will be combined
19758 // to "store (load int32)" in later InstCombine pass. See function
19759 // combineLoadToOperationType. On PowerPC, loading a float point takes more
19760 // cycles than loading a 32 bit integer.
19761 LoadInst *LI = cast<LoadInst>(Val: I);
19762 // For the loads that combineLoadToOperationType does nothing, like
19763 // ordered load, it should be profitable to hoist them.
19764 // For swifterror load, it can only be used for pointer to pointer type, so
19765 // later type check should get rid of this case.
19766 if (!LI->isUnordered())
19767 return true;
19768
19769 if (User->getOpcode() != Instruction::Store)
19770 return true;
19771
19772 if (I->getType()->getTypeID() != Type::FloatTyID)
19773 return true;
19774
19775 return false;
19776 }
19777 default:
19778 return true;
19779 }
19780 return true;
19781}
19782
19783const MCPhysReg *
19784PPCTargetLowering::getScratchRegisters(CallingConv::ID) const {
19785 // LR is a callee-save register, but we must treat it as clobbered by any call
19786 // site. Hence we include LR in the scratch registers, which are in turn added
19787 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies
19788 // to CTR, which is used by any indirect call.
19789 static const MCPhysReg ScratchRegs[] = {
19790 PPC::X12, PPC::LR8, PPC::CTR8, 0
19791 };
19792
19793 return ScratchRegs;
19794}
19795
19796Register PPCTargetLowering::getExceptionPointerRegister(
19797 ExceptionHandling EH, const Constant *PersonalityFn) const {
19798 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3;
19799}
19800
19801Register PPCTargetLowering::getExceptionSelectorRegister(
19802 ExceptionHandling EH, const Constant *PersonalityFn) const {
19803 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4;
19804}
19805
19806bool
19807PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
19808 EVT VT , unsigned DefinedValues) const {
19809 if (VT == MVT::v2i64)
19810 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
19811
19812 if (Subtarget.hasVSX())
19813 return true;
19814
19815 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
19816}
19817
19818Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const {
19819 if (DisableILPPref || Subtarget.enableMachineScheduler())
19820 return TargetLowering::getSchedulingPreference(N);
19821
19822 return Sched::ILP;
19823}
19824
19825// Create a fast isel object.
19826FastISel *PPCTargetLowering::createFastISel(
19827 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo,
19828 const LibcallLoweringInfo *LibcallLowering) const {
19829 return PPC::createFastISel(FuncInfo, LibInfo, LibcallLowering);
19830}
19831
19832// 'Inverted' means the FMA opcode after negating one multiplicand.
19833// For example, (fma -a b c) = (fnmsub a b c)
19834static unsigned invertFMAOpcode(unsigned Opc) {
19835 switch (Opc) {
19836 default:
19837 llvm_unreachable("Invalid FMA opcode for PowerPC!");
19838 case ISD::FMA:
19839 return PPCISD::FNMSUB;
19840 case PPCISD::FNMSUB:
19841 return ISD::FMA;
19842 }
19843}
19844
19845SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
19846 bool LegalOps, bool OptForSize,
19847 NegatibleCost &Cost,
19848 unsigned Depth) const {
19849 if (Depth > SelectionDAG::MaxRecursionDepth)
19850 return SDValue();
19851
19852 unsigned Opc = Op.getOpcode();
19853 EVT VT = Op.getValueType();
19854 SDNodeFlags Flags = Op.getNode()->getFlags();
19855
19856 switch (Opc) {
19857 case PPCISD::FNMSUB:
19858 if (!Op.hasOneUse() || !isTypeLegal(VT))
19859 break;
19860
19861 SDValue N0 = Op.getOperand(i: 0);
19862 SDValue N1 = Op.getOperand(i: 1);
19863 SDValue N2 = Op.getOperand(i: 2);
19864 SDLoc Loc(Op);
19865
19866 NegatibleCost N2Cost = NegatibleCost::Expensive;
19867 SDValue NegN2 =
19868 getNegatedExpression(Op: N2, DAG, LegalOps, OptForSize, Cost&: N2Cost, Depth: Depth + 1);
19869
19870 if (!NegN2)
19871 return SDValue();
19872
19873 // (fneg (fnmsub a b c)) => (fnmsub (fneg a) b (fneg c))
19874 // (fneg (fnmsub a b c)) => (fnmsub a (fneg b) (fneg c))
19875 // These transformations may change sign of zeroes. For example,
19876 // -(-ab-(-c))=-0 while -(-(ab-c))=+0 when a=b=c=1.
19877 if (Flags.hasNoSignedZeros()) {
19878 // Try and choose the cheaper one to negate.
19879 NegatibleCost N0Cost = NegatibleCost::Expensive;
19880 SDValue NegN0 = getNegatedExpression(Op: N0, DAG, LegalOps, OptForSize,
19881 Cost&: N0Cost, Depth: Depth + 1);
19882
19883 NegatibleCost N1Cost = NegatibleCost::Expensive;
19884 SDValue NegN1 = getNegatedExpression(Op: N1, DAG, LegalOps, OptForSize,
19885 Cost&: N1Cost, Depth: Depth + 1);
19886
19887 if (NegN0 && N0Cost <= N1Cost) {
19888 Cost = std::min(a: N0Cost, b: N2Cost);
19889 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: NegN0, N2: N1, N3: NegN2, Flags);
19890 } else if (NegN1) {
19891 Cost = std::min(a: N1Cost, b: N2Cost);
19892 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: N0, N2: NegN1, N3: NegN2, Flags);
19893 }
19894 }
19895
19896 // (fneg (fnmsub a b c)) => (fma a b (fneg c))
19897 if (isOperationLegal(Op: ISD::FMA, VT)) {
19898 Cost = N2Cost;
19899 return DAG.getNode(Opcode: ISD::FMA, DL: Loc, VT, N1: N0, N2: N1, N3: NegN2, Flags);
19900 }
19901
19902 break;
19903 }
19904
19905 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps, OptForSize,
19906 Cost, Depth);
19907}
19908
19909// Override to enable LOAD_STACK_GUARD lowering on Linux.
19910bool PPCTargetLowering::useLoadStackGuardNode(const Module &M) const {
19911 if (M.getStackProtectorGuard() == "tls" || Subtarget.isTargetLinux())
19912 return true;
19913 return TargetLowering::useLoadStackGuardNode(M);
19914}
19915
19916bool PPCTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
19917 bool ForCodeSize) const {
19918 if (!VT.isSimple() || !Subtarget.hasVSX())
19919 return false;
19920
19921 switch(VT.getSimpleVT().SimpleTy) {
19922 default:
19923 // For FP types that are currently not supported by PPC backend, return
19924 // false. Examples: f16, f80.
19925 return false;
19926 case MVT::f32:
19927 case MVT::f64: {
19928 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
19929 // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP.
19930 return true;
19931 }
19932 bool IsExact;
19933 APSInt IntResult(16, false);
19934 // The rounding mode doesn't really matter because we only care about floats
19935 // that can be converted to integers exactly.
19936 Imm.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
19937 // For exact values in the range [-16, 15] we can materialize the float.
19938 if (IsExact && IntResult <= 15 && IntResult >= -16)
19939 return true;
19940 return Imm.isZero();
19941 }
19942 case MVT::ppcf128:
19943 return Imm.isPosZero();
19944 }
19945}
19946
19947// For vector shift operation op, fold
19948// (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y)
19949static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N,
19950 SelectionDAG &DAG) {
19951 SDValue N0 = N->getOperand(Num: 0);
19952 SDValue N1 = N->getOperand(Num: 1);
19953 EVT VT = N0.getValueType();
19954 unsigned OpSizeInBits = VT.getScalarSizeInBits();
19955 unsigned Opcode = N->getOpcode();
19956 unsigned TargetOpcode;
19957
19958 switch (Opcode) {
19959 default:
19960 llvm_unreachable("Unexpected shift operation");
19961 case ISD::SHL:
19962 TargetOpcode = PPCISD::SHL;
19963 break;
19964 case ISD::SRL:
19965 TargetOpcode = PPCISD::SRL;
19966 break;
19967 case ISD::SRA:
19968 TargetOpcode = PPCISD::SRA;
19969 break;
19970 }
19971
19972 if (VT.isVector() && TLI.isOperationLegal(Op: Opcode, VT) &&
19973 N1->getOpcode() == ISD::AND)
19974 if (ConstantSDNode *Mask = isConstOrConstSplat(N: N1->getOperand(Num: 1)))
19975 if (Mask->getZExtValue() == OpSizeInBits - 1)
19976 return DAG.getNode(Opcode: TargetOpcode, DL: SDLoc(N), VT, N1: N0, N2: N1->getOperand(Num: 0));
19977
19978 return SDValue();
19979}
19980
19981SDValue PPCTargetLowering::combineVectorShift(SDNode *N,
19982 DAGCombinerInfo &DCI) const {
19983 EVT VT = N->getValueType(ResNo: 0);
19984 assert(VT.isVector() && "Vector type expected.");
19985
19986 unsigned Opc = N->getOpcode();
19987 assert((Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) &&
19988 "Unexpected opcode.");
19989
19990 if (!isOperationLegal(Op: Opc, VT))
19991 return SDValue();
19992
19993 EVT EltTy = VT.getScalarType();
19994 unsigned EltBits = EltTy.getSizeInBits();
19995 if (EltTy != MVT::i64 && EltTy != MVT::i32)
19996 return SDValue();
19997
19998 SDValue N1 = N->getOperand(Num: 1);
19999 uint64_t SplatBits = 0;
20000 bool AddSplatCase = false;
20001 unsigned OpcN1 = N1.getOpcode();
20002 if (OpcN1 == PPCISD::VADD_SPLAT &&
20003 N1.getConstantOperandVal(i: 1) == VT.getVectorNumElements()) {
20004 AddSplatCase = true;
20005 SplatBits = N1.getConstantOperandVal(i: 0);
20006 }
20007
20008 if (!AddSplatCase) {
20009 if (OpcN1 != ISD::BUILD_VECTOR)
20010 return SDValue();
20011
20012 unsigned SplatBitSize;
20013 bool HasAnyUndefs;
20014 APInt APSplatBits, APSplatUndef;
20015 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val&: N1);
20016 bool BVNIsConstantSplat =
20017 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
20018 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
20019 if (!BVNIsConstantSplat || SplatBitSize != EltBits)
20020 return SDValue();
20021 SplatBits = APSplatBits.getZExtValue();
20022 }
20023
20024 SDLoc DL(N);
20025 SDValue N0 = N->getOperand(Num: 0);
20026 // PPC vector shifts by word/double look at only the low 5/6 bits of the
20027 // shift vector, which means the max value is 31/63. A shift vector of all
20028 // 1s will be truncated to 31/63, which is useful as vspltiw is limited to
20029 // -16 to 15 range.
20030 if (SplatBits == (EltBits - 1)) {
20031 unsigned NewOpc;
20032 switch (Opc) {
20033 case ISD::SHL:
20034 NewOpc = PPCISD::SHL;
20035 break;
20036 case ISD::SRL:
20037 NewOpc = PPCISD::SRL;
20038 break;
20039 case ISD::SRA:
20040 NewOpc = PPCISD::SRA;
20041 break;
20042 }
20043 SDValue SplatOnes = getCanonicalConstSplat(Val: 255, SplatSize: 1, VT, DAG&: DCI.DAG, dl: DL);
20044 return DCI.DAG.getNode(Opcode: NewOpc, DL, VT, N1: N0, N2: SplatOnes);
20045 }
20046
20047 if (Opc != ISD::SHL || !isOperationLegal(Op: ISD::ADD, VT))
20048 return SDValue();
20049
20050 // For 64-bit there is no splat immediate so we want to catch shift by 1 here
20051 // before the BUILD_VECTOR is replaced by a load.
20052 if (EltTy != MVT::i64 || SplatBits != 1)
20053 return SDValue();
20054
20055 return DCI.DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT, N1: N0, N2: N0);
20056}
20057
20058SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const {
20059 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20060 return Value;
20061
20062 if (N->getValueType(ResNo: 0).isVector())
20063 return combineVectorShift(N, DCI);
20064
20065 SDValue N0 = N->getOperand(Num: 0);
20066 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
20067 if (!Subtarget.isISA3_0() || !Subtarget.isPPC64() ||
20068 N0.getOpcode() != ISD::SIGN_EXTEND ||
20069 N0.getOperand(i: 0).getValueType() != MVT::i32 || CN1 == nullptr ||
20070 N->getValueType(ResNo: 0) != MVT::i64)
20071 return SDValue();
20072
20073 // We can't save an operation here if the value is already extended, and
20074 // the existing shift is easier to combine.
20075 SDValue ExtsSrc = N0.getOperand(i: 0);
20076 if (ExtsSrc.getOpcode() == ISD::TRUNCATE &&
20077 ExtsSrc.getOperand(i: 0).getOpcode() == ISD::AssertSext)
20078 return SDValue();
20079
20080 SDLoc DL(N0);
20081 SDValue ShiftBy = SDValue(CN1, 0);
20082 // We want the shift amount to be i32 on the extswli, but the shift could
20083 // have an i64.
20084 if (ShiftBy.getValueType() == MVT::i64)
20085 ShiftBy = DCI.DAG.getConstant(Val: CN1->getZExtValue(), DL, VT: MVT::i32);
20086
20087 return DCI.DAG.getNode(Opcode: PPCISD::EXTSWSLI, DL, VT: MVT::i64, N1: N0->getOperand(Num: 0),
20088 N2: ShiftBy);
20089}
20090
20091SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const {
20092 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20093 return Value;
20094
20095 if (N->getValueType(ResNo: 0).isVector())
20096 return combineVectorShift(N, DCI);
20097
20098 return SDValue();
20099}
20100
20101SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const {
20102 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20103 return Value;
20104
20105 if (N->getValueType(ResNo: 0).isVector())
20106 return combineVectorShift(N, DCI);
20107
20108 return SDValue();
20109}
20110
20111// Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1))
20112// Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0))
20113// When C is zero, the equation (addi Z, -C) can be simplified to Z
20114// Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types
20115static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG,
20116 const PPCSubtarget &Subtarget) {
20117 if (!Subtarget.isPPC64())
20118 return SDValue();
20119
20120 SDValue LHS = N->getOperand(Num: 0);
20121 SDValue RHS = N->getOperand(Num: 1);
20122
20123 auto isZextOfCompareWithConstant = [](SDValue Op) {
20124 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() ||
20125 Op.getValueType() != MVT::i64)
20126 return false;
20127
20128 SDValue Cmp = Op.getOperand(i: 0);
20129 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() ||
20130 Cmp.getOperand(i: 0).getValueType() != MVT::i64)
20131 return false;
20132
20133 if (auto *Constant = dyn_cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1))) {
20134 int64_t NegConstant = 0 - Constant->getSExtValue();
20135 // Due to the limitations of the addi instruction,
20136 // -C is required to be [-32768, 32767].
20137 return isInt<16>(x: NegConstant);
20138 }
20139
20140 return false;
20141 };
20142
20143 bool LHSHasPattern = isZextOfCompareWithConstant(LHS);
20144 bool RHSHasPattern = isZextOfCompareWithConstant(RHS);
20145
20146 // If there is a pattern, canonicalize a zext operand to the RHS.
20147 if (LHSHasPattern && !RHSHasPattern)
20148 std::swap(a&: LHS, b&: RHS);
20149 else if (!LHSHasPattern && !RHSHasPattern)
20150 return SDValue();
20151
20152 SDLoc DL(N);
20153 EVT CarryType = Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
20154 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: CarryType);
20155 SDValue Cmp = RHS.getOperand(i: 0);
20156 SDValue Z = Cmp.getOperand(i: 0);
20157 auto *Constant = cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1));
20158 int64_t NegConstant = 0 - Constant->getSExtValue();
20159
20160 switch(cast<CondCodeSDNode>(Val: Cmp.getOperand(i: 2))->get()) {
20161 default: break;
20162 case ISD::SETNE: {
20163 // when C == 0
20164 // --> addze X, (addic Z, -1).carry
20165 // /
20166 // add X, (zext(setne Z, C))--
20167 // \ when -32768 <= -C <= 32767 && C != 0
20168 // --> addze X, (addic (addi Z, -C), -1).carry
20169 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20170 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20171 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20172 SDValue Addc =
20173 DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20174 N1: AddOrZ, N2: DAG.getAllOnesConstant(DL, VT: MVT::i64),
20175 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20176 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20177 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64),
20178 N3: SDValue(Addc.getNode(), 1));
20179 }
20180 case ISD::SETEQ: {
20181 // when C == 0
20182 // --> addze X, (subfic Z, 0).carry
20183 // /
20184 // add X, (zext(sete Z, C))--
20185 // \ when -32768 <= -C <= 32767 && C != 0
20186 // --> addze X, (subfic (addi Z, -C), 0).carry
20187 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20188 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20189 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20190 SDValue Subc =
20191 DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20192 N1: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N2: AddOrZ,
20193 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20194 SDValue Invert = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Subc.getValue(R: 1),
20195 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
20196 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20197 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N3: Invert);
20198 }
20199 }
20200
20201 return SDValue();
20202}
20203
20204// Transform
20205// (add C1, (MAT_PCREL_ADDR GlobalAddr+C2)) to
20206// (MAT_PCREL_ADDR GlobalAddr+(C1+C2))
20207// In this case both C1 and C2 must be known constants.
20208// C1+C2 must fit into a 34 bit signed integer.
20209static SDValue combineADDToMAT_PCREL_ADDR(SDNode *N, SelectionDAG &DAG,
20210 const PPCSubtarget &Subtarget) {
20211 if (!Subtarget.isUsingPCRelativeCalls())
20212 return SDValue();
20213
20214 // Check both Operand 0 and Operand 1 of the ADD node for the PCRel node.
20215 // If we find that node try to cast the Global Address and the Constant.
20216 SDValue LHS = N->getOperand(Num: 0);
20217 SDValue RHS = N->getOperand(Num: 1);
20218
20219 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20220 std::swap(a&: LHS, b&: RHS);
20221
20222 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20223 return SDValue();
20224
20225 // Operand zero of PPCISD::MAT_PCREL_ADDR is the GA node.
20226 GlobalAddressSDNode *GSDN = dyn_cast<GlobalAddressSDNode>(Val: LHS.getOperand(i: 0));
20227 ConstantSDNode* ConstNode = dyn_cast<ConstantSDNode>(Val&: RHS);
20228
20229 // Check that both casts succeeded.
20230 if (!GSDN || !ConstNode)
20231 return SDValue();
20232
20233 int64_t NewOffset = GSDN->getOffset() + ConstNode->getSExtValue();
20234 SDLoc DL(GSDN);
20235
20236 // The signed int offset needs to fit in 34 bits.
20237 if (!isInt<34>(x: NewOffset))
20238 return SDValue();
20239
20240 // The new global address is a copy of the old global address except
20241 // that it has the updated Offset.
20242 SDValue GA =
20243 DAG.getTargetGlobalAddress(GV: GSDN->getGlobal(), DL, VT: GSDN->getValueType(ResNo: 0),
20244 offset: NewOffset, TargetFlags: GSDN->getTargetFlags());
20245 SDValue MatPCRel =
20246 DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: GSDN->getValueType(ResNo: 0), Operand: GA);
20247 return MatPCRel;
20248}
20249
20250// Transform (add X, (build_vector (T 1), (T 1), ...)) -> (sub X, (XXLEQVOnes))
20251// XXLEQVOnes creates an all-1s vector (0xFFFFFFFF...) efficiently via xxleqv
20252// Mathematical identity: X + 1 = X - (-1)
20253// Applies to v4i32, v2i64, v8i16, v16i8 where all elements are constant 1
20254// Requirement: VSX feature for efficient xxleqv generation
20255static SDValue combineADDToSUB(SDNode *N, SelectionDAG &DAG,
20256 const PPCSubtarget &Subtarget) {
20257
20258 EVT VT = N->getValueType(ResNo: 0);
20259 if (!Subtarget.hasVSX())
20260 return SDValue();
20261
20262 // Handle v2i64, v4i32, v8i16 and v16i8 types
20263 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
20264 VT == MVT::v2i64))
20265 return SDValue();
20266
20267 SDValue LHS = N->getOperand(Num: 0);
20268 SDValue RHS = N->getOperand(Num: 1);
20269
20270 // Check if RHS is BUILD_VECTOR
20271 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
20272 return SDValue();
20273
20274 // Check if all the elements are 1
20275 unsigned NumOfEles = RHS.getNumOperands();
20276 for (unsigned i = 0; i < NumOfEles; ++i) {
20277 auto *CN = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i));
20278 if (!CN || CN->getSExtValue() != 1)
20279 return SDValue();
20280 }
20281 SDLoc DL(N);
20282
20283 SDValue MinusOne = DAG.getConstant(Val: APInt::getAllOnes(numBits: 32), DL, VT: MVT::i32);
20284 SmallVector<SDValue, 4> Ops(4, MinusOne);
20285 SDValue AllOnesVec = DAG.getBuildVector(VT: MVT::v4i32, DL, Ops);
20286
20287 // Bitcast to the target vector type
20288 SDValue Bitcast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: AllOnesVec);
20289
20290 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Bitcast);
20291}
20292
20293SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const {
20294 if (auto Value = combineADDToADDZE(N, DAG&: DCI.DAG, Subtarget))
20295 return Value;
20296
20297 if (auto Value = combineADDToMAT_PCREL_ADDR(N, DAG&: DCI.DAG, Subtarget))
20298 return Value;
20299
20300 if (auto Value = combineADDToSUB(N, DAG&: DCI.DAG, Subtarget))
20301 return Value;
20302 return SDValue();
20303}
20304
20305// Detect TRUNCATE operations on bitcasts of float128 values.
20306// What we are looking for here is the situtation where we extract a subset
20307// of bits from a 128 bit float.
20308// This can be of two forms:
20309// 1) BITCAST of f128 feeding TRUNCATE
20310// 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE
20311// The reason this is required is because we do not have a legal i128 type
20312// and so we want to prevent having to store the f128 and then reload part
20313// of it.
20314SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N,
20315 DAGCombinerInfo &DCI) const {
20316 // If we are using CRBits then try that first.
20317 if (Subtarget.useCRBits()) {
20318 // Check if CRBits did anything and return that if it did.
20319 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI))
20320 return CRTruncValue;
20321 }
20322
20323 SDLoc dl(N);
20324 SDValue Op0 = N->getOperand(Num: 0);
20325
20326 // Looking for a truncate of i128 to i64.
20327 if (Op0.getValueType() != MVT::i128 || N->getValueType(ResNo: 0) != MVT::i64)
20328 return SDValue();
20329
20330 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0;
20331
20332 // SRL feeding TRUNCATE.
20333 if (Op0.getOpcode() == ISD::SRL) {
20334 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
20335 // The right shift has to be by 64 bits.
20336 if (!ConstNode || ConstNode->getZExtValue() != 64)
20337 return SDValue();
20338
20339 // Switch the element number to extract.
20340 EltToExtract = EltToExtract ? 0 : 1;
20341 // Update Op0 past the SRL.
20342 Op0 = Op0.getOperand(i: 0);
20343 }
20344
20345 // BITCAST feeding a TRUNCATE possibly via SRL.
20346 if (Op0.getOpcode() == ISD::BITCAST &&
20347 Op0.getValueType() == MVT::i128 &&
20348 Op0.getOperand(i: 0).getValueType() == MVT::f128) {
20349 SDValue Bitcast = DCI.DAG.getBitcast(VT: MVT::v2i64, V: Op0.getOperand(i: 0));
20350 return DCI.DAG.getNode(
20351 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Bitcast,
20352 N2: DCI.DAG.getTargetConstant(Val: EltToExtract, DL: dl, VT: MVT::i32));
20353 }
20354 return SDValue();
20355}
20356
20357SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const {
20358 SelectionDAG &DAG = DCI.DAG;
20359
20360 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N: N->getOperand(Num: 1));
20361 if (!ConstOpOrElement)
20362 return SDValue();
20363
20364 // An imul is usually smaller than the alternative sequence for legal type.
20365 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
20366 isOperationLegal(Op: ISD::MUL, VT: N->getValueType(ResNo: 0)))
20367 return SDValue();
20368
20369 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool {
20370 switch (this->Subtarget.getCPUDirective()) {
20371 default:
20372 // TODO: enhance the condition for subtarget before pwr8
20373 return false;
20374 case PPC::DIR_PWR8:
20375 // type mul add shl
20376 // scalar 4 1 1
20377 // vector 7 2 2
20378 return true;
20379 case PPC::DIR_PWR9:
20380 case PPC::DIR_PWR10:
20381 case PPC::DIR_PWR11:
20382 case PPC::DIR_PWR_FUTURE:
20383 // type mul add shl
20384 // scalar 5 2 2
20385 // vector 7 2 2
20386
20387 // The cycle RATIO of related operations are showed as a table above.
20388 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both
20389 // scalar and vector type. For 2 instrs patterns, add/sub + shl
20390 // are 4, it is always profitable; but for 3 instrs patterns
20391 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6.
20392 // So we should only do it for vector type.
20393 return IsAddOne && IsNeg ? VT.isVector() : true;
20394 }
20395 };
20396
20397 EVT VT = N->getValueType(ResNo: 0);
20398 SDLoc DL(N);
20399
20400 const APInt &MulAmt = ConstOpOrElement->getAPIntValue();
20401 bool IsNeg = MulAmt.isNegative();
20402 APInt MulAmtAbs = MulAmt.abs();
20403
20404 if ((MulAmtAbs - 1).isPowerOf2()) {
20405 // (mul x, 2^N + 1) => (add (shl x, N), x)
20406 // (mul x, -(2^N + 1)) => -(add (shl x, N), x)
20407
20408 if (!IsProfitable(IsNeg, true, VT))
20409 return SDValue();
20410
20411 SDValue Op0 = N->getOperand(Num: 0);
20412 SDValue Op1 =
20413 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20414 N2: DAG.getConstant(Val: (MulAmtAbs - 1).logBase2(), DL, VT));
20415 SDValue Res = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: Op1);
20416
20417 if (!IsNeg)
20418 return Res;
20419
20420 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Res);
20421 } else if ((MulAmtAbs + 1).isPowerOf2()) {
20422 // (mul x, 2^N - 1) => (sub (shl x, N), x)
20423 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
20424
20425 if (!IsProfitable(IsNeg, false, VT))
20426 return SDValue();
20427
20428 SDValue Op0 = N->getOperand(Num: 0);
20429 SDValue Op1 =
20430 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20431 N2: DAG.getConstant(Val: (MulAmtAbs + 1).logBase2(), DL, VT));
20432
20433 if (!IsNeg)
20434 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op1, N2: Op0);
20435 else
20436 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0, N2: Op1);
20437
20438 } else {
20439 return SDValue();
20440 }
20441}
20442
20443// Combine fma-like op (like fnmsub) with fnegs to appropriate op. Do this
20444// in combiner since we need to check SD flags and other subtarget features.
20445SDValue PPCTargetLowering::combineFMALike(SDNode *N,
20446 DAGCombinerInfo &DCI) const {
20447 SDValue N0 = N->getOperand(Num: 0);
20448 SDValue N1 = N->getOperand(Num: 1);
20449 SDValue N2 = N->getOperand(Num: 2);
20450 SDNodeFlags Flags = N->getFlags();
20451 EVT VT = N->getValueType(ResNo: 0);
20452 SelectionDAG &DAG = DCI.DAG;
20453 unsigned Opc = N->getOpcode();
20454 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
20455 bool LegalOps = !DCI.isBeforeLegalizeOps();
20456 SDLoc Loc(N);
20457
20458 if (!isOperationLegal(Op: ISD::FMA, VT))
20459 return SDValue();
20460
20461 // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
20462 // since (fnmsub a b c)=-0 while c-ab=+0.
20463 if (!Flags.hasNoSignedZeros())
20464 return SDValue();
20465
20466 // (fma (fneg a) b c) => (fnmsub a b c)
20467 // (fnmsub (fneg a) b c) => (fma a b c)
20468 if (SDValue NegN0 = getCheaperNegatedExpression(Op: N0, DAG, LegalOps, OptForSize: CodeSize))
20469 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: NegN0, N2: N1, N3: N2, Flags);
20470
20471 // (fma a (fneg b) c) => (fnmsub a b c)
20472 // (fnmsub a (fneg b) c) => (fma a b c)
20473 if (SDValue NegN1 = getCheaperNegatedExpression(Op: N1, DAG, LegalOps, OptForSize: CodeSize))
20474 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: N0, N2: NegN1, N3: N2, Flags);
20475
20476 return SDValue();
20477}
20478
20479bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
20480 // Only duplicate to increase tail-calls for the 64bit SysV ABIs.
20481 if (!Subtarget.is64BitELFABI())
20482 return false;
20483
20484 // If not a tail call then no need to proceed.
20485 if (!CI->isTailCall())
20486 return false;
20487
20488 // If sibling calls have been disabled and tail-calls aren't guaranteed
20489 // there is no reason to duplicate.
20490 auto &TM = getTargetMachine();
20491 if (!TM.Options.GuaranteedTailCallOpt && DisableSCO)
20492 return false;
20493
20494 // Can't tail call a function called indirectly, or if it has variadic args.
20495 const Function *Callee = CI->getCalledFunction();
20496 if (!Callee || Callee->isVarArg())
20497 return false;
20498
20499 // Make sure the callee and caller calling conventions are eligible for tco.
20500 const Function *Caller = CI->getParent()->getParent();
20501 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC: Caller->getCallingConv(),
20502 CalleeCC: CI->getCallingConv()))
20503 return false;
20504
20505 // If the function is local then we have a good chance at tail-calling it
20506 return getTargetMachine().shouldAssumeDSOLocal(GV: Callee);
20507}
20508
20509bool PPCTargetLowering::
20510isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
20511 const Value *Mask = AndI.getOperand(i: 1);
20512 // If the mask is suitable for andi. or andis. we should sink the and.
20513 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Mask)) {
20514 // Can't handle constants wider than 64-bits.
20515 if (CI->getBitWidth() > 64)
20516 return false;
20517 int64_t ConstVal = CI->getZExtValue();
20518 return isUInt<16>(x: ConstVal) ||
20519 (isUInt<16>(x: ConstVal >> 16) && !(ConstVal & 0xFFFF));
20520 }
20521
20522 // For non-constant masks, we can always use the record-form and.
20523 return true;
20524}
20525
20526/// getAddrModeForFlags - Based on the set of address flags, select the most
20527/// optimal instruction format to match by.
20528PPC::AddrMode PPCTargetLowering::getAddrModeForFlags(unsigned Flags) const {
20529 // This is not a node we should be handling here.
20530 if (Flags == PPC::MOF_None)
20531 return PPC::AM_None;
20532 // Unaligned D-Forms are tried first, followed by the aligned D-Forms.
20533 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DForm))
20534 if ((Flags & FlagSet) == FlagSet)
20535 return PPC::AM_DForm;
20536 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DSForm))
20537 if ((Flags & FlagSet) == FlagSet)
20538 return PPC::AM_DSForm;
20539 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DQForm))
20540 if ((Flags & FlagSet) == FlagSet)
20541 return PPC::AM_DQForm;
20542 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_PrefixDForm))
20543 if ((Flags & FlagSet) == FlagSet)
20544 return PPC::AM_PrefixDForm;
20545 // If no other forms are selected, return an X-Form as it is the most
20546 // general addressing mode.
20547 return PPC::AM_XForm;
20548}
20549
20550/// Set alignment flags based on whether or not the Frame Index is aligned.
20551/// Utilized when computing flags for address computation when selecting
20552/// load and store instructions.
20553static void setAlignFlagsForFI(SDValue N, unsigned &FlagSet,
20554 SelectionDAG &DAG) {
20555 bool IsAdd = ((N.getOpcode() == ISD::ADD) || (N.getOpcode() == ISD::OR));
20556 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: IsAdd ? N.getOperand(i: 0) : N);
20557 if (!FI)
20558 return;
20559 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
20560 unsigned FrameIndexAlign = MFI.getObjectAlign(ObjectIdx: FI->getIndex()).value();
20561 // If this is (add $FI, $S16Imm), the alignment flags are already set
20562 // based on the immediate. We just need to clear the alignment flags
20563 // if the FI alignment is weaker.
20564 if ((FrameIndexAlign % 4) != 0)
20565 FlagSet &= ~PPC::MOF_RPlusSImm16Mult4;
20566 if ((FrameIndexAlign % 16) != 0)
20567 FlagSet &= ~PPC::MOF_RPlusSImm16Mult16;
20568 // If the address is a plain FrameIndex, set alignment flags based on
20569 // FI alignment.
20570 if (!IsAdd) {
20571 if ((FrameIndexAlign % 4) == 0)
20572 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20573 if ((FrameIndexAlign % 16) == 0)
20574 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20575 }
20576}
20577
20578/// Given a node, compute flags that are used for address computation when
20579/// selecting load and store instructions. The flags computed are stored in
20580/// FlagSet. This function takes into account whether the node is a constant,
20581/// an ADD, OR, or a constant, and computes the address flags accordingly.
20582static void computeFlagsForAddressComputation(SDValue N, unsigned &FlagSet,
20583 SelectionDAG &DAG) {
20584 // Set the alignment flags for the node depending on if the node is
20585 // 4-byte or 16-byte aligned.
20586 auto SetAlignFlagsForImm = [&](uint64_t Imm) {
20587 if ((Imm & 0x3) == 0)
20588 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20589 if ((Imm & 0xf) == 0)
20590 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20591 };
20592
20593 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
20594 // All 32-bit constants can be computed as LIS + Disp.
20595 const APInt &ConstImm = CN->getAPIntValue();
20596 if (ConstImm.isSignedIntN(N: 32)) { // Flag to handle 32-bit constants.
20597 FlagSet |= PPC::MOF_AddrIsSImm32;
20598 SetAlignFlagsForImm(ConstImm.getZExtValue());
20599 setAlignFlagsForFI(N, FlagSet, DAG);
20600 }
20601 if (ConstImm.isSignedIntN(N: 34)) // Flag to handle 34-bit constants.
20602 FlagSet |= PPC::MOF_RPlusSImm34;
20603 else // Let constant materialization handle large constants.
20604 FlagSet |= PPC::MOF_NotAddNorCst;
20605 } else if (N.getOpcode() == ISD::ADD || provablyDisjointOr(DAG, N)) {
20606 // This address can be represented as an addition of:
20607 // - Register + Imm16 (possibly a multiple of 4/16)
20608 // - Register + Imm34
20609 // - Register + PPCISD::Lo
20610 // - Register + Register
20611 // In any case, we won't have to match this as Base + Zero.
20612 SDValue RHS = N.getOperand(i: 1);
20613 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: RHS)) {
20614 const APInt &ConstImm = CN->getAPIntValue();
20615 if (ConstImm.isSignedIntN(N: 16)) {
20616 FlagSet |= PPC::MOF_RPlusSImm16; // Signed 16-bit immediates.
20617 SetAlignFlagsForImm(ConstImm.getZExtValue());
20618 setAlignFlagsForFI(N, FlagSet, DAG);
20619 }
20620 if (ConstImm.isSignedIntN(N: 34))
20621 FlagSet |= PPC::MOF_RPlusSImm34; // Signed 34-bit immediates.
20622 else
20623 FlagSet |= PPC::MOF_RPlusR; // Register.
20624 } else if (RHS.getOpcode() == PPCISD::Lo && !RHS.getConstantOperandVal(i: 1))
20625 FlagSet |= PPC::MOF_RPlusLo; // PPCISD::Lo.
20626 else
20627 FlagSet |= PPC::MOF_RPlusR;
20628 } else { // The address computation is not a constant or an addition.
20629 setAlignFlagsForFI(N, FlagSet, DAG);
20630 FlagSet |= PPC::MOF_NotAddNorCst;
20631 }
20632}
20633
20634static bool isPCRelNode(SDValue N) {
20635 return (N.getOpcode() == PPCISD::MAT_PCREL_ADDR ||
20636 isValidPCRelNode<ConstantPoolSDNode>(N) ||
20637 isValidPCRelNode<GlobalAddressSDNode>(N) ||
20638 isValidPCRelNode<JumpTableSDNode>(N) ||
20639 isValidPCRelNode<BlockAddressSDNode>(N));
20640}
20641
20642/// computeMOFlags - Given a node N and it's Parent (a MemSDNode), compute
20643/// the address flags of the load/store instruction that is to be matched.
20644unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N,
20645 SelectionDAG &DAG) const {
20646 unsigned FlagSet = PPC::MOF_None;
20647
20648 // Compute subtarget flags.
20649 if (!Subtarget.hasP9Vector())
20650 FlagSet |= PPC::MOF_SubtargetBeforeP9;
20651 else
20652 FlagSet |= PPC::MOF_SubtargetP9;
20653
20654 if (Subtarget.hasPrefixInstrs())
20655 FlagSet |= PPC::MOF_SubtargetP10;
20656
20657 if (Subtarget.hasSPE())
20658 FlagSet |= PPC::MOF_SubtargetSPE;
20659
20660 // Check if we have a PCRel node and return early.
20661 if ((FlagSet & PPC::MOF_SubtargetP10) && isPCRelNode(N))
20662 return FlagSet;
20663
20664 // If the node is the paired load/store intrinsics, compute flags for
20665 // address computation and return early.
20666 unsigned ParentOp = Parent->getOpcode();
20667 if (Subtarget.isISA3_1() && ((ParentOp == ISD::INTRINSIC_W_CHAIN) ||
20668 (ParentOp == ISD::INTRINSIC_VOID))) {
20669 unsigned ID = Parent->getConstantOperandVal(Num: 1);
20670 if ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) {
20671 SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp)
20672 ? Parent->getOperand(Num: 2)
20673 : Parent->getOperand(Num: 3);
20674 computeFlagsForAddressComputation(N: IntrinOp, FlagSet, DAG);
20675 FlagSet |= PPC::MOF_Vector;
20676 return FlagSet;
20677 }
20678 }
20679
20680 // Mark this as something we don't want to handle here if it is atomic
20681 // or pre-increment instruction.
20682 if (const LSBaseSDNode *LSB = dyn_cast<LSBaseSDNode>(Val: Parent))
20683 if (LSB->isIndexed())
20684 return PPC::MOF_None;
20685
20686 // Compute in-memory type flags. This is based on if there are scalars,
20687 // floats or vectors.
20688 const MemSDNode *MN = dyn_cast<MemSDNode>(Val: Parent);
20689 assert(MN && "Parent should be a MemSDNode!");
20690 EVT MemVT = MN->getMemoryVT();
20691 unsigned Size = MemVT.getSizeInBits();
20692 if (MemVT.isScalarInteger()) {
20693 assert(Size <= 128 &&
20694 "Not expecting scalar integers larger than 16 bytes!");
20695 if (Size < 32)
20696 FlagSet |= PPC::MOF_SubWordInt;
20697 else if (Size == 32)
20698 FlagSet |= PPC::MOF_WordInt;
20699 else
20700 FlagSet |= PPC::MOF_DoubleWordInt;
20701 } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors.
20702 if (Size == 128)
20703 FlagSet |= PPC::MOF_Vector;
20704 else if (Size == 256) {
20705 assert(Subtarget.pairedVectorMemops() &&
20706 "256-bit vectors are only available when paired vector memops is "
20707 "enabled!");
20708 FlagSet |= PPC::MOF_Vector;
20709 } else
20710 llvm_unreachable("Not expecting illegal vectors!");
20711 } else { // Floating point type: can be scalar, f128 or vector types.
20712 if (Size == 32 || Size == 64)
20713 FlagSet |= PPC::MOF_ScalarFloat;
20714 else if (MemVT == MVT::f128 || MemVT.isVector())
20715 FlagSet |= PPC::MOF_Vector;
20716 else
20717 llvm_unreachable("Not expecting illegal scalar floats!");
20718 }
20719
20720 // Compute flags for address computation.
20721 computeFlagsForAddressComputation(N, FlagSet, DAG);
20722
20723 // Compute type extension flags.
20724 if (const LoadSDNode *LN = dyn_cast<LoadSDNode>(Val: Parent)) {
20725 switch (LN->getExtensionType()) {
20726 case ISD::SEXTLOAD:
20727 FlagSet |= PPC::MOF_SExt;
20728 break;
20729 case ISD::EXTLOAD:
20730 case ISD::ZEXTLOAD:
20731 FlagSet |= PPC::MOF_ZExt;
20732 break;
20733 case ISD::NON_EXTLOAD:
20734 FlagSet |= PPC::MOF_NoExt;
20735 break;
20736 }
20737 } else
20738 FlagSet |= PPC::MOF_NoExt;
20739
20740 // For integers, no extension is the same as zero extension.
20741 // We set the extension mode to zero extension so we don't have
20742 // to add separate entries in AddrModesMap for loads and stores.
20743 if (MemVT.isScalarInteger() && (FlagSet & PPC::MOF_NoExt)) {
20744 FlagSet |= PPC::MOF_ZExt;
20745 FlagSet &= ~PPC::MOF_NoExt;
20746 }
20747
20748 // If we don't have prefixed instructions, 34-bit constants should be
20749 // treated as PPC::MOF_NotAddNorCst so they can match D-Forms.
20750 bool IsNonP1034BitConst =
20751 ((PPC::MOF_RPlusSImm34 | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubtargetP10) &
20752 FlagSet) == PPC::MOF_RPlusSImm34;
20753 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::OR &&
20754 IsNonP1034BitConst)
20755 FlagSet |= PPC::MOF_NotAddNorCst;
20756
20757 return FlagSet;
20758}
20759
20760/// SelectForceXFormMode - Given the specified address, force it to be
20761/// represented as an indexed [r+r] operation (an XForm instruction).
20762PPC::AddrMode PPCTargetLowering::SelectForceXFormMode(SDValue N, SDValue &Disp,
20763 SDValue &Base,
20764 SelectionDAG &DAG) const {
20765
20766 PPC::AddrMode Mode = PPC::AM_XForm;
20767 int16_t ForceXFormImm = 0;
20768 if (provablyDisjointOr(DAG, N) &&
20769 !isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm)) {
20770 Disp = N.getOperand(i: 0);
20771 Base = N.getOperand(i: 1);
20772 return Mode;
20773 }
20774
20775 // If the address is the result of an add, we will utilize the fact that the
20776 // address calculation includes an implicit add. However, we can reduce
20777 // register pressure if we do not materialize a constant just for use as the
20778 // index register. We only get rid of the add if it is not an add of a
20779 // value and a 16-bit signed constant and both have a single use.
20780 if (N.getOpcode() == ISD::ADD &&
20781 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm) ||
20782 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
20783 Disp = N.getOperand(i: 0);
20784 Base = N.getOperand(i: 1);
20785 return Mode;
20786 }
20787
20788 // Otherwise, use R0 as the base register.
20789 Disp = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
20790 VT: N.getValueType());
20791 Base = N;
20792
20793 return Mode;
20794}
20795
20796bool PPCTargetLowering::splitValueIntoRegisterParts(
20797 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
20798 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
20799 EVT ValVT = Val.getValueType();
20800 // If we are splitting a scalar integer into f64 parts (i.e. so they
20801 // can be placed into VFRC registers), we need to zero extend and
20802 // bitcast the values. This will ensure the value is placed into a
20803 // VSR using direct moves or stack operations as needed.
20804 if (PartVT == MVT::f64 &&
20805 (ValVT == MVT::i32 || ValVT == MVT::i16 || ValVT == MVT::i8)) {
20806 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: Val);
20807 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Val);
20808 Parts[0] = Val;
20809 return true;
20810 }
20811 return false;
20812}
20813
20814SDValue PPCTargetLowering::lowerToLibCall(const char *LibCallName, SDValue Op,
20815 SelectionDAG &DAG) const {
20816 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20817 TargetLowering::CallLoweringInfo CLI(DAG);
20818 EVT RetVT = Op.getValueType();
20819 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
20820 SDValue Callee =
20821 DAG.getExternalSymbol(Sym: LibCallName, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
20822 bool SignExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: false);
20823 TargetLowering::ArgListTy Args;
20824 for (const SDValue &N : Op->op_values()) {
20825 EVT ArgVT = N.getValueType();
20826 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
20827 TargetLowering::ArgListEntry Entry(N, ArgTy);
20828 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: SignExtend);
20829 Entry.IsZExt = !Entry.IsSExt;
20830 Args.push_back(x: Entry);
20831 }
20832
20833 SDValue InChain = DAG.getEntryNode();
20834 SDValue TCChain = InChain;
20835 const Function &F = DAG.getMachineFunction().getFunction();
20836 bool isTailCall =
20837 TLI.isInTailCallPosition(DAG, Node: Op.getNode(), Chain&: TCChain) &&
20838 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
20839 if (isTailCall)
20840 InChain = TCChain;
20841 CLI.setDebugLoc(SDLoc(Op))
20842 .setChain(InChain)
20843 .setLibCallee(CC: CallingConv::C, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
20844 .setTailCall(isTailCall)
20845 .setSExtResult(SignExtend)
20846 .setZExtResult(!SignExtend)
20847 .setIsPostTypeLegalization(true);
20848 return TLI.LowerCallTo(CLI).first;
20849}
20850
20851SDValue PPCTargetLowering::lowerLibCallBasedOnType(
20852 const char *LibCallFloatName, const char *LibCallDoubleName, SDValue Op,
20853 SelectionDAG &DAG) const {
20854 if (Op.getValueType() == MVT::f32)
20855 return lowerToLibCall(LibCallName: LibCallFloatName, Op, DAG);
20856
20857 if (Op.getValueType() == MVT::f64)
20858 return lowerToLibCall(LibCallName: LibCallDoubleName, Op, DAG);
20859
20860 return SDValue();
20861}
20862
20863bool PPCTargetLowering::isLowringToMASSFiniteSafe(SDValue Op) const {
20864 SDNodeFlags Flags = Op.getNode()->getFlags();
20865 return isLowringToMASSSafe(Op) && Flags.hasNoSignedZeros() &&
20866 Flags.hasNoNaNs() && Flags.hasNoInfs();
20867}
20868
20869bool PPCTargetLowering::isLowringToMASSSafe(SDValue Op) const {
20870 return Op.getNode()->getFlags().hasApproximateFuncs();
20871}
20872
20873bool PPCTargetLowering::isScalarMASSConversionEnabled() const {
20874 return getTargetMachine().getOptLevel() == CodeGenOptLevel::Aggressive &&
20875 EnablePPCGenScalarMASSEntries;
20876}
20877
20878SDValue PPCTargetLowering::lowerLibCallBase(const char *LibCallDoubleName,
20879 const char *LibCallFloatName,
20880 const char *LibCallDoubleNameFinite,
20881 const char *LibCallFloatNameFinite,
20882 SDValue Op,
20883 SelectionDAG &DAG) const {
20884 if (!isScalarMASSConversionEnabled() || !isLowringToMASSSafe(Op))
20885 return SDValue();
20886
20887 if (!isLowringToMASSFiniteSafe(Op))
20888 return lowerLibCallBasedOnType(LibCallFloatName, LibCallDoubleName, Op,
20889 DAG);
20890
20891 return lowerLibCallBasedOnType(LibCallFloatName: LibCallFloatNameFinite,
20892 LibCallDoubleName: LibCallDoubleNameFinite, Op, DAG);
20893}
20894
20895SDValue PPCTargetLowering::lowerPow(SDValue Op, SelectionDAG &DAG) const {
20896 return lowerLibCallBase(LibCallDoubleName: "__xl_pow", LibCallFloatName: "__xl_powf", LibCallDoubleNameFinite: "__xl_pow_finite",
20897 LibCallFloatNameFinite: "__xl_powf_finite", Op, DAG);
20898}
20899
20900SDValue PPCTargetLowering::lowerSin(SDValue Op, SelectionDAG &DAG) const {
20901 return lowerLibCallBase(LibCallDoubleName: "__xl_sin", LibCallFloatName: "__xl_sinf", LibCallDoubleNameFinite: "__xl_sin_finite",
20902 LibCallFloatNameFinite: "__xl_sinf_finite", Op, DAG);
20903}
20904
20905SDValue PPCTargetLowering::lowerCos(SDValue Op, SelectionDAG &DAG) const {
20906 return lowerLibCallBase(LibCallDoubleName: "__xl_cos", LibCallFloatName: "__xl_cosf", LibCallDoubleNameFinite: "__xl_cos_finite",
20907 LibCallFloatNameFinite: "__xl_cosf_finite", Op, DAG);
20908}
20909
20910SDValue PPCTargetLowering::lowerLog(SDValue Op, SelectionDAG &DAG) const {
20911 return lowerLibCallBase(LibCallDoubleName: "__xl_log", LibCallFloatName: "__xl_logf", LibCallDoubleNameFinite: "__xl_log_finite",
20912 LibCallFloatNameFinite: "__xl_logf_finite", Op, DAG);
20913}
20914
20915SDValue PPCTargetLowering::lowerLog10(SDValue Op, SelectionDAG &DAG) const {
20916 return lowerLibCallBase(LibCallDoubleName: "__xl_log10", LibCallFloatName: "__xl_log10f", LibCallDoubleNameFinite: "__xl_log10_finite",
20917 LibCallFloatNameFinite: "__xl_log10f_finite", Op, DAG);
20918}
20919
20920SDValue PPCTargetLowering::lowerExp(SDValue Op, SelectionDAG &DAG) const {
20921 return lowerLibCallBase(LibCallDoubleName: "__xl_exp", LibCallFloatName: "__xl_expf", LibCallDoubleNameFinite: "__xl_exp_finite",
20922 LibCallFloatNameFinite: "__xl_expf_finite", Op, DAG);
20923}
20924
20925// If we happen to match to an aligned D-Form, check if the Frame Index is
20926// adequately aligned. If it is not, reset the mode to match to X-Form.
20927static void setXFormForUnalignedFI(SDValue N, unsigned Flags,
20928 PPC::AddrMode &Mode) {
20929 if (!isa<FrameIndexSDNode>(Val: N))
20930 return;
20931 if ((Mode == PPC::AM_DSForm && !(Flags & PPC::MOF_RPlusSImm16Mult4)) ||
20932 (Mode == PPC::AM_DQForm && !(Flags & PPC::MOF_RPlusSImm16Mult16)))
20933 Mode = PPC::AM_XForm;
20934}
20935
20936/// SelectOptimalAddrMode - Based on a node N and it's Parent (a MemSDNode),
20937/// compute the address flags of the node, get the optimal address mode based
20938/// on the flags, and set the Base and Disp based on the address mode.
20939PPC::AddrMode PPCTargetLowering::SelectOptimalAddrMode(const SDNode *Parent,
20940 SDValue N, SDValue &Disp,
20941 SDValue &Base,
20942 SelectionDAG &DAG,
20943 MaybeAlign Align) const {
20944 SDLoc DL(Parent);
20945
20946 // Compute the address flags.
20947 unsigned Flags = computeMOFlags(Parent, N, DAG);
20948
20949 // Get the optimal address mode based on the Flags.
20950 PPC::AddrMode Mode = getAddrModeForFlags(Flags);
20951
20952 // If the address mode is DS-Form or DQ-Form, check if the FI is aligned.
20953 // Select an X-Form load if it is not.
20954 setXFormForUnalignedFI(N, Flags, Mode);
20955
20956 // Set the mode to PC-Relative addressing mode if we have a valid PC-Rel node.
20957 if ((Mode == PPC::AM_XForm) && isPCRelNode(N)) {
20958 assert(Subtarget.isUsingPCRelativeCalls() &&
20959 "Must be using PC-Relative calls when a valid PC-Relative node is "
20960 "present!");
20961 Mode = PPC::AM_PCRel;
20962 }
20963
20964 // Set Base and Disp accordingly depending on the address mode.
20965 switch (Mode) {
20966 case PPC::AM_DForm:
20967 case PPC::AM_DSForm:
20968 case PPC::AM_DQForm: {
20969 // This is a register plus a 16-bit immediate. The base will be the
20970 // register and the displacement will be the immediate unless it
20971 // isn't sufficiently aligned.
20972 if (Flags & PPC::MOF_RPlusSImm16) {
20973 SDValue Op0 = N.getOperand(i: 0);
20974 SDValue Op1 = N.getOperand(i: 1);
20975 int16_t Imm = Op1->getAsZExtVal();
20976 if (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm)) {
20977 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: N.getValueType());
20978 Base = Op0;
20979 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op0)) {
20980 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
20981 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
20982 }
20983 break;
20984 }
20985 }
20986 // This is a register plus the @lo relocation. The base is the register
20987 // and the displacement is the global address.
20988 else if (Flags & PPC::MOF_RPlusLo) {
20989 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
20990 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
20991 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
20992 Disp.getOpcode() == ISD::TargetConstantPool ||
20993 Disp.getOpcode() == ISD::TargetJumpTable);
20994 Base = N.getOperand(i: 0);
20995 break;
20996 }
20997 // This is a constant address at most 32 bits. The base will be
20998 // zero or load-immediate-shifted and the displacement will be
20999 // the low 16 bits of the address.
21000 else if (Flags & PPC::MOF_AddrIsSImm32) {
21001 auto *CN = cast<ConstantSDNode>(Val&: N);
21002 EVT CNType = CN->getValueType(ResNo: 0);
21003 uint64_t CNImm = CN->getZExtValue();
21004 // If this address fits entirely in a 16-bit sext immediate field, codegen
21005 // this as "d, 0".
21006 int16_t Imm;
21007 if (isIntS16Immediate(N: CN, Imm) && (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm))) {
21008 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: CNType);
21009 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
21010 VT: CNType);
21011 break;
21012 }
21013 // Handle 32-bit sext immediate with LIS + Addr mode.
21014 if ((CNType == MVT::i32 || isInt<32>(x: CNImm)) &&
21015 (!Align || isAligned(Lhs: *Align, SizeInBytes: CNImm))) {
21016 int32_t Addr = (int32_t)CNImm;
21017 // Otherwise, break this down into LIS + Disp.
21018 Disp = DAG.getSignedTargetConstant(Val: (int16_t)Addr, DL, VT: MVT::i32);
21019 Base = DAG.getSignedTargetConstant(Val: (Addr - (int16_t)Addr) >> 16, DL,
21020 VT: MVT::i32);
21021 uint32_t LIS = CNType == MVT::i32 ? PPC::LIS : PPC::LIS8;
21022 Base = SDValue(DAG.getMachineNode(Opcode: LIS, dl: DL, VT: CNType, Op1: Base), 0);
21023 break;
21024 }
21025 }
21026 // Otherwise, the PPC:MOF_NotAdd flag is set. Load/Store is Non-foldable.
21027 Disp = DAG.getTargetConstant(Val: 0, DL, VT: getPointerTy(DL: DAG.getDataLayout()));
21028 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
21029 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
21030 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
21031 } else
21032 Base = N;
21033 break;
21034 }
21035 case PPC::AM_PrefixDForm: {
21036 int64_t Imm34 = 0;
21037 unsigned Opcode = N.getOpcode();
21038 if (((Opcode == ISD::ADD) || (Opcode == ISD::OR)) &&
21039 (isIntS34Immediate(Op: N.getOperand(i: 1), Imm&: Imm34))) {
21040 // N is an Add/OR Node, and it's operand is a 34-bit signed immediate.
21041 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
21042 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
21043 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
21044 else
21045 Base = N.getOperand(i: 0);
21046 } else if (isIntS34Immediate(Op: N, Imm&: Imm34)) {
21047 // The address is a 34-bit signed immediate.
21048 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
21049 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
21050 }
21051 break;
21052 }
21053 case PPC::AM_PCRel: {
21054 // When selecting PC-Relative instructions, "Base" is not utilized as
21055 // we select the address as [PC+imm].
21056 Disp = N;
21057 break;
21058 }
21059 case PPC::AM_None:
21060 break;
21061 default: { // By default, X-Form is always available to be selected.
21062 // When a frame index is not aligned, we also match by XForm.
21063 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N);
21064 Base = FI ? N : N.getOperand(i: 1);
21065 Disp = FI ? DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
21066 VT: N.getValueType())
21067 : N.getOperand(i: 0);
21068 break;
21069 }
21070 }
21071 return Mode;
21072}
21073
21074CCAssignFn *PPCTargetLowering::ccAssignFnForCall(CallingConv::ID CC,
21075 bool Return,
21076 bool IsVarArg) const {
21077 switch (CC) {
21078 case CallingConv::Cold:
21079 return (Return ? RetCC_PPC_Cold : CC_PPC64_ELF);
21080 default:
21081 return CC_PPC64_ELF;
21082 }
21083}
21084
21085bool PPCTargetLowering::shouldInlineQuadwordAtomics() const {
21086 return Subtarget.isPPC64() && Subtarget.hasQuadwordAtomics();
21087}
21088
21089TargetLowering::AtomicExpansionKind
21090PPCTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
21091 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21092 if (shouldInlineQuadwordAtomics() && Size == 128)
21093 return AtomicExpansionKind::MaskedIntrinsic;
21094
21095 switch (AI->getOperation()) {
21096 case AtomicRMWInst::UIncWrap:
21097 case AtomicRMWInst::UDecWrap:
21098 case AtomicRMWInst::USubCond:
21099 case AtomicRMWInst::USubSat:
21100 return AtomicExpansionKind::CmpXChg;
21101 default:
21102 return TargetLowering::shouldExpandAtomicRMWInIR(RMW: AI);
21103 }
21104
21105 llvm_unreachable("unreachable atomicrmw operation");
21106}
21107
21108TargetLowering::AtomicExpansionKind
21109PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(
21110 const AtomicCmpXchgInst *AI) const {
21111 unsigned Size = AI->getNewValOperand()->getType()->getPrimitiveSizeInBits();
21112 if (shouldInlineQuadwordAtomics() && Size == 128)
21113 return AtomicExpansionKind::MaskedIntrinsic;
21114 return AtomicExpansionKind::LLSC;
21115}
21116
21117static Intrinsic::ID
21118getIntrinsicForAtomicRMWBinOp128(AtomicRMWInst::BinOp BinOp) {
21119 switch (BinOp) {
21120 default:
21121 llvm_unreachable("Unexpected AtomicRMW BinOp");
21122 case AtomicRMWInst::Xchg:
21123 return Intrinsic::ppc_atomicrmw_xchg_i128;
21124 case AtomicRMWInst::Add:
21125 return Intrinsic::ppc_atomicrmw_add_i128;
21126 case AtomicRMWInst::Sub:
21127 return Intrinsic::ppc_atomicrmw_sub_i128;
21128 case AtomicRMWInst::And:
21129 return Intrinsic::ppc_atomicrmw_and_i128;
21130 case AtomicRMWInst::Or:
21131 return Intrinsic::ppc_atomicrmw_or_i128;
21132 case AtomicRMWInst::Xor:
21133 return Intrinsic::ppc_atomicrmw_xor_i128;
21134 case AtomicRMWInst::Nand:
21135 return Intrinsic::ppc_atomicrmw_nand_i128;
21136 }
21137}
21138
21139Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic(
21140 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
21141 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
21142 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21143 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21144 Type *ValTy = Incr->getType();
21145 assert(ValTy->getPrimitiveSizeInBits() == 128);
21146 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21147 Value *IncrLo = Builder.CreateTrunc(V: Incr, DestTy: Int64Ty, Name: "incr_lo");
21148 Value *IncrHi =
21149 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: Incr, RHS: 64), DestTy: Int64Ty, Name: "incr_hi");
21150 Value *LoHi = Builder.CreateIntrinsic(
21151 ID: getIntrinsicForAtomicRMWBinOp128(BinOp: AI->getOperation()), OverloadTypes: {},
21152 Args: {AlignedAddr, IncrLo, IncrHi});
21153 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21154 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21155 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21156 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21157 return Builder.CreateOr(
21158 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21159}
21160
21161Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
21162 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
21163 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
21164 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21165 Module *M = Builder.GetInsertBlock()->getParent()->getParent();
21166 Type *ValTy = CmpVal->getType();
21167 assert(ValTy->getPrimitiveSizeInBits() == 128);
21168 Function *IntCmpXchg =
21169 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::ppc_cmpxchg_i128);
21170 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21171 Value *CmpLo = Builder.CreateTrunc(V: CmpVal, DestTy: Int64Ty, Name: "cmp_lo");
21172 Value *CmpHi =
21173 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: CmpVal, RHS: 64), DestTy: Int64Ty, Name: "cmp_hi");
21174 Value *NewLo = Builder.CreateTrunc(V: NewVal, DestTy: Int64Ty, Name: "new_lo");
21175 Value *NewHi =
21176 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: NewVal, RHS: 64), DestTy: Int64Ty, Name: "new_hi");
21177 emitLeadingFence(Builder, Inst: CI, Ord);
21178 Value *LoHi =
21179 Builder.CreateCall(Callee: IntCmpXchg, Args: {AlignedAddr, CmpLo, CmpHi, NewLo, NewHi});
21180 emitTrailingFence(Builder, Inst: CI, Ord);
21181 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21182 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21183 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21184 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21185 return Builder.CreateOr(
21186 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21187}
21188
21189bool PPCTargetLowering::hasMultipleConditionRegisters(EVT VT) const {
21190 return Subtarget.useCRBits();
21191}
21192
21193/// Shuffle masks for vectors of bits are not legal as such vectors are
21194/// reserved for MMA/DM.
21195bool PPCTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const {
21196 if (VT.getScalarType() == MVT::i1)
21197 return false;
21198 return TargetLowering::isShuffleMaskLegal(Mask, VT);
21199}
21200
21201// Optimize the following patterns using vbpermq/vbpermd:
21202// i16 = bitcast(v16i1 truncate(v16i8))
21203// i8 = bitcast(v8i1 truncate(v8i16))
21204// i8 = bitcast(v8i1 truncate(v8i8))
21205SDValue PPCTargetLowering::DAGCombineBitcast(SDNode *N,
21206 DAGCombinerInfo &DCI) const {
21207 SDValue Op0 = N->getOperand(Num: 0);
21208 if (Op0.getOpcode() != ISD::TRUNCATE)
21209 return SDValue();
21210 SDValue Src = Op0.getOperand(i: 0);
21211 EVT ResVT = N->getValueType(ResNo: 0);
21212 EVT TruncResVT = Op0.getValueType();
21213 EVT SrcVT = Src.getValueType();
21214 SDLoc dl(N);
21215 SelectionDAG &DAG = DCI.DAG;
21216 bool IsLittleEndian = Subtarget.isLittleEndian();
21217
21218 if (ResVT != MVT::i16 && ResVT != MVT::i8)
21219 return SDValue();
21220 SDValue VBPerm =
21221 GenerateVBPERM(DAG, dl, Src, SrcVT, ResVT: TruncResVT, IsLE: IsLittleEndian);
21222 if (!VBPerm)
21223 return SDValue();
21224 SDValue ForExtract = DAG.getBitcast(VT: MVT::v4i32, V: VBPerm);
21225 SDValue Extracted =
21226 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: ForExtract,
21227 N2: DAG.getIntPtrConstant(Val: IsLittleEndian ? 2 : 1, DL: dl));
21228 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT, Operand: Extracted);
21229}
21230
21231SDValue PPCTargetLowering::GenerateVBPERM(SelectionDAG &DAG, SDLoc dl,
21232 SDValue Src, EVT SrcVT, EVT ResVT,
21233 bool IsLE) const {
21234 bool IsV16i8 = (ResVT == MVT::v16i1 && SrcVT == MVT::v16i8);
21235 bool IsV8i16 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i16);
21236 bool IsV8i8 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i8);
21237
21238 if (!IsV16i8 && !IsV8i16 && !IsV8i8)
21239 return SDValue();
21240
21241 if (IsV8i8) {
21242 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: MVT::v16i8,
21243 N1: DAG.getUNDEF(VT: MVT::v16i8), N2: Src,
21244 N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
21245 }
21246 SmallVector<int, 16> BitIndices(16, 128);
21247 unsigned NumElts = SrcVT.getVectorNumElements();
21248 unsigned EltSize = SrcVT.getScalarType().getSizeInBits();
21249 for (int Idx = 0, End = SrcVT.getVectorNumElements(); Idx < End; Idx++) {
21250 BitIndices[Idx] = EltSize * (NumElts - Idx) - 1;
21251 if (IsV8i8 && IsLE)
21252 BitIndices[Idx] += 64;
21253 }
21254 if (!IsLE)
21255 std::reverse(first: BitIndices.begin(), last: BitIndices.end());
21256 SmallVector<SDValue, 16> BVOps;
21257 for (auto Idx : BitIndices)
21258 BVOps.push_back(Elt: DAG.getConstant(Val: Idx, DL: dl, VT: MVT::i8));
21259 SDValue VRB = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: BVOps);
21260 return DAG.getNode(
21261 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v16i8,
21262 N1: DAG.getConstant(Val: Intrinsic::ppc_altivec_vbpermq, DL: dl, VT: MVT::i32),
21263 N2: DAG.getBitcast(VT: MVT::v16i8, V: Src), N3: VRB);
21264}
21265
21266// For Power8/9, optimize vec splats of small FP values that can be
21267// represented as integers. Use vspltisw + xvcvsxwdp/xvcvsxwsp instead of
21268// loading from constant pool.
21269SDValue PPCTargetLowering::LowerVecSplatSmallFP(SDValue Op, SelectionDAG &DAG,
21270 bool BVNIsConstantSplat,
21271 unsigned SplatBitSize) const {
21272
21273 if (!BVNIsConstantSplat || !Subtarget.hasVSX() || !Subtarget.hasP8Vector() ||
21274 Subtarget.hasP10Vector())
21275 return SDValue();
21276
21277 EVT VT = Op->getValueType(ResNo: 0);
21278 if (!((SplatBitSize == 64 && VT == MVT::v2f64) ||
21279 (SplatBitSize == 32 && VT == MVT::v4f32)))
21280 return SDValue();
21281
21282 auto *CN = dyn_cast<ConstantFPSDNode>(Val: Op.getOperand(i: 0));
21283 if (!CN)
21284 return SDValue();
21285
21286 APFloat APFloatVal = CN->getValueAPF();
21287 bool IsExact;
21288 APSInt IntResult(16, false);
21289 APFloatVal.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
21290
21291 if (!(IsExact && IntResult <= 15 && IntResult >= -16 && !APFloatVal.isZero()))
21292 return SDValue();
21293
21294 int64_t IntVal = IntResult.getSExtValue();
21295
21296 SDLoc dl(Op);
21297 SDValue IntSplat = getCanonicalConstSplat(Val: IntVal, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
21298
21299 if (SplatBitSize == 64)
21300 return DAG.getNode(
21301 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v2f64,
21302 N1: DAG.getConstant(Val: Intrinsic::ppc_vsx_xvcvsxwdp, DL: dl, VT: MVT::i32), N2: IntSplat);
21303
21304 return DAG.getNode(Opcode: PPCISD::XVCVSXWSP, DL: dl, VT: MVT::v4f32, Operand: IntSplat);
21305}
21306