1//===-- PPCISelLowering.cpp - PPC DAG Lowering Implementation -------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the PPCISelLowering class.
10//
11//===----------------------------------------------------------------------===//
12
13#include "PPCISelLowering.h"
14#include "MCTargetDesc/PPCMCTargetDesc.h"
15#include "MCTargetDesc/PPCPredicates.h"
16#include "PPC.h"
17#include "PPCCallingConv.h"
18#include "PPCFrameLowering.h"
19#include "PPCInstrInfo.h"
20#include "PPCMachineFunctionInfo.h"
21#include "PPCPerfectShuffle.h"
22#include "PPCRegisterInfo.h"
23#include "PPCSelectionDAGInfo.h"
24#include "PPCSubtarget.h"
25#include "PPCTargetMachine.h"
26#include "llvm/ADT/APFloat.h"
27#include "llvm/ADT/APInt.h"
28#include "llvm/ADT/APSInt.h"
29#include "llvm/ADT/ArrayRef.h"
30#include "llvm/ADT/DenseMap.h"
31#include "llvm/ADT/STLExtras.h"
32#include "llvm/ADT/SmallPtrSet.h"
33#include "llvm/ADT/SmallVector.h"
34#include "llvm/ADT/Statistic.h"
35#include "llvm/ADT/StringRef.h"
36#include "llvm/CodeGen/CallingConvLower.h"
37#include "llvm/CodeGen/ISDOpcodes.h"
38#include "llvm/CodeGen/LivePhysRegs.h"
39#include "llvm/CodeGen/MachineBasicBlock.h"
40#include "llvm/CodeGen/MachineFrameInfo.h"
41#include "llvm/CodeGen/MachineFunction.h"
42#include "llvm/CodeGen/MachineInstr.h"
43#include "llvm/CodeGen/MachineInstrBuilder.h"
44#include "llvm/CodeGen/MachineJumpTableInfo.h"
45#include "llvm/CodeGen/MachineLoopInfo.h"
46#include "llvm/CodeGen/MachineMemOperand.h"
47#include "llvm/CodeGen/MachineModuleInfo.h"
48#include "llvm/CodeGen/MachineOperand.h"
49#include "llvm/CodeGen/MachineRegisterInfo.h"
50#include "llvm/CodeGen/SelectionDAG.h"
51#include "llvm/CodeGen/SelectionDAGNodes.h"
52#include "llvm/CodeGen/TargetInstrInfo.h"
53#include "llvm/CodeGen/TargetLowering.h"
54#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
55#include "llvm/CodeGen/TargetRegisterInfo.h"
56#include "llvm/CodeGen/ValueTypes.h"
57#include "llvm/CodeGenTypes/MachineValueType.h"
58#include "llvm/IR/CallingConv.h"
59#include "llvm/IR/Constant.h"
60#include "llvm/IR/Constants.h"
61#include "llvm/IR/DataLayout.h"
62#include "llvm/IR/DebugLoc.h"
63#include "llvm/IR/DerivedTypes.h"
64#include "llvm/IR/Function.h"
65#include "llvm/IR/GlobalValue.h"
66#include "llvm/IR/IRBuilder.h"
67#include "llvm/IR/Instructions.h"
68#include "llvm/IR/Intrinsics.h"
69#include "llvm/IR/IntrinsicsPowerPC.h"
70#include "llvm/IR/Module.h"
71#include "llvm/IR/Type.h"
72#include "llvm/IR/Use.h"
73#include "llvm/IR/Value.h"
74#include "llvm/MC/MCContext.h"
75#include "llvm/MC/MCExpr.h"
76#include "llvm/MC/MCSectionXCOFF.h"
77#include "llvm/MC/MCSymbolXCOFF.h"
78#include "llvm/Support/AtomicOrdering.h"
79#include "llvm/Support/BranchProbability.h"
80#include "llvm/Support/Casting.h"
81#include "llvm/Support/CodeGen.h"
82#include "llvm/Support/CommandLine.h"
83#include "llvm/Support/Compiler.h"
84#include "llvm/Support/Debug.h"
85#include "llvm/Support/ErrorHandling.h"
86#include "llvm/Support/Format.h"
87#include "llvm/Support/KnownBits.h"
88#include "llvm/Support/MathExtras.h"
89#include "llvm/Support/raw_ostream.h"
90#include "llvm/Target/TargetMachine.h"
91#include "llvm/Target/TargetOptions.h"
92#include <algorithm>
93#include <cassert>
94#include <cstdint>
95#include <iterator>
96#include <list>
97#include <optional>
98#include <utility>
99#include <vector>
100
101using namespace llvm;
102
103#define DEBUG_TYPE "ppc-lowering"
104
105extern cl::opt<bool> EnablePPCGenScalarMASSEntries;
106
107static cl::opt<bool> EnableP10StoreForward(
108 "ppc-p10-store-forward",
109 cl::desc("enable P10 store forward-friendly conversion"), cl::init(Val: true),
110 cl::Hidden);
111
112static cl::opt<bool> EnablePPCPreinc(
113 "ppc-preinc", cl::desc("enable preincrement load/store generation on PPC"),
114 cl::init(Val: true), cl::Hidden);
115
116static cl::opt<bool>
117 EnableILPPref("ppc-ilp-pref",
118 cl::desc("Set the node scheduling preference to ILP on PPC"),
119 cl::init(Val: true), cl::Hidden);
120
121static cl::opt<bool> EnablePPCUnaligned(
122 "ppc-unaligned", cl::desc("enable unaligned load/store generation on PPC"),
123 cl::init(Val: true), cl::Hidden);
124
125static cl::opt<bool>
126 EnableSCO("ppc-sco", cl::desc("enable sibling call optimization on ppc"),
127 cl::init(Val: true), cl::Hidden);
128
129static cl::opt<bool> EnableInnermostLoopAlign32(
130 "ppc-innermost-loop-align32",
131 cl::desc("Align innermost loops to 32 bytes on ppc"), cl::init(Val: true),
132 cl::Hidden);
133
134static cl::opt<bool> UseAbsoluteJumpTables("ppc-use-absolute-jumptables",
135cl::desc("use absolute jump tables on ppc"), cl::Hidden);
136
137static cl::opt<bool>
138 EnablePerfectShuffle("ppc-perfect-shuffle",
139 cl::desc("enable vector permute decomposition"),
140 cl::init(Val: false), cl::Hidden);
141
142cl::opt<bool> EnableAutoPairedVecSt(
143 "ppc-auto-paired-vec-st",
144 cl::desc("enable automatically generated 32byte paired vector stores"),
145 cl::init(Val: false), cl::Hidden);
146
147static cl::opt<unsigned> PPCMinimumJumpTableEntries(
148 "ppc-min-jump-table-entries", cl::init(Val: 64), cl::Hidden,
149 cl::desc("Set minimum number of entries to use a jump table on PPC"));
150
151static cl::opt<unsigned> PPCMinimumBitTestCmps(
152 "ppc-min-bit-test-cmps", cl::init(Val: 3), cl::Hidden,
153 cl::desc("Set minimum of largest number of comparisons to use bit test for "
154 "switch on PPC."));
155
156static cl::opt<unsigned> PPCGatherAllAliasesMaxDepth(
157 "ppc-gather-alias-max-depth", cl::init(Val: 18), cl::Hidden,
158 cl::desc("max depth when checking alias info in GatherAllAliases()"));
159
160static cl::opt<unsigned> PPCAIXTLSModelOptUseIEForLDLimit(
161 "ppc-aix-shared-lib-tls-model-opt-limit", cl::init(Val: 1), cl::Hidden,
162 cl::desc("Set inclusive limit count of TLS local-dynamic access(es) in a "
163 "function to use initial-exec"));
164
165STATISTIC(NumTailCalls, "Number of tail calls");
166STATISTIC(NumSiblingCalls, "Number of sibling calls");
167STATISTIC(ShufflesHandledWithVPERM,
168 "Number of shuffles lowered to a VPERM or XXPERM");
169STATISTIC(NumDynamicAllocaProbed, "Number of dynamic stack allocation probed");
170
171static bool isNByteElemShuffleMask(ShuffleVectorSDNode *, unsigned, int);
172
173static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl);
174
175static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
176 unsigned OpIdx, bool IsByte,
177 const PPCInstrInfo *TII);
178
179// A faster local-[exec|dynamic] TLS access sequence (enabled with the
180// -maix-small-local-[exec|dynamic]-tls option) can be produced for TLS
181// variables; consistent with the IBM XL compiler, we apply a max size of
182// slightly under 32KB.
183constexpr uint64_t AIXSmallTlsPolicySizeLimit = 32751;
184
185// FIXME: Remove this once the bug has been fixed!
186extern cl::opt<bool> ANDIGlueBug;
187
188PPCTargetLowering::PPCTargetLowering(const PPCTargetMachine &TM,
189 const PPCSubtarget &STI)
190 : TargetLowering(TM, STI), Subtarget(STI) {
191 // Initialize map that relates the PPC addressing modes to the computed flags
192 // of a load/store instruction. The map is used to determine the optimal
193 // addressing mode when selecting load and stores.
194 initializeAddrModeMap();
195 // On PPC32/64, arguments smaller than 4/8 bytes are extended, so all
196 // arguments are at least 4/8 bytes aligned.
197 bool isPPC64 = Subtarget.isPPC64();
198 setMinStackArgumentAlignment(isPPC64 ? Align(8) : Align(4));
199 const MVT RegVT = Subtarget.getScalarIntVT();
200
201 // Set up the register classes.
202 addRegisterClass(VT: MVT::i32, RC: &PPC::GPRCRegClass);
203 if (!useSoftFloat()) {
204 if (hasSPE()) {
205 addRegisterClass(VT: MVT::f32, RC: &PPC::GPRCRegClass);
206 // EFPU2 APU only supports f32
207 if (!Subtarget.hasEFPU2())
208 addRegisterClass(VT: MVT::f64, RC: &PPC::SPERCRegClass);
209 } else {
210 addRegisterClass(VT: MVT::f32, RC: &PPC::F4RCRegClass);
211 addRegisterClass(VT: MVT::f64, RC: &PPC::F8RCRegClass);
212 }
213 }
214
215 setOperationAction(Op: ISD::UADDO, VT: RegVT, Action: Custom);
216 setOperationAction(Op: ISD::USUBO, VT: RegVT, Action: Custom);
217
218 // PowerPC uses addo_carry,subo_carry to propagate carry.
219 setOperationAction(Op: ISD::UADDO_CARRY, VT: RegVT, Action: Custom);
220 setOperationAction(Op: ISD::USUBO_CARRY, VT: RegVT, Action: Custom);
221
222 // On P10, the default lowering generates better code using the
223 // setbc instruction.
224 if (!Subtarget.hasP10Vector()) {
225 setOperationAction(Op: ISD::SSUBO, VT: MVT::i32, Action: Custom);
226 setOperationAction(Op: ISD::SADDO, VT: MVT::i32, Action: Custom);
227 if (isPPC64) {
228 setOperationAction(Op: ISD::SSUBO, VT: MVT::i64, Action: Custom);
229 setOperationAction(Op: ISD::SADDO, VT: MVT::i64, Action: Custom);
230 }
231 }
232
233 // Match BITREVERSE to customized fast code sequence in the td file.
234 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i32, Action: Legal);
235 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i64, Action: Legal);
236
237 // Custom lower inline assembly to check for special registers.
238 setOperationAction(Op: ISD::INLINEASM, VT: MVT::Other, Action: Custom);
239 setOperationAction(Op: ISD::INLINEASM_BR, VT: MVT::Other, Action: Custom);
240
241 // PowerPC has an i16 but no i8 (or i1) SEXTLOAD.
242 for (MVT VT : MVT::integer_valuetypes()) {
243 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
244 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i8, Action: Expand);
245 }
246
247 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
248 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f128, Action: Expand);
249
250 if (Subtarget.isISA3_0()) {
251 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Legal);
252 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
253 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
254 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Legal);
255 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Legal);
256 } else {
257 // No extending loads from f16 or HW conversions back and forth.
258 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: MVT::f16, Action: Expand);
259 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f128, Action: Expand);
260 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
261 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
262 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Expand);
263 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
264 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Expand);
265 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Expand);
266 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
267 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
268 }
269
270 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
271
272 // PowerPC has pre-inc load and store's.
273 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
274 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
275 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
276 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
277 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
278 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i1, Action: Legal);
279 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i8, Action: Legal);
280 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i16, Action: Legal);
281 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i32, Action: Legal);
282 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::i64, Action: Legal);
283 if (!Subtarget.hasSPE()) {
284 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
285 setIndexedLoadAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
286 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f32, Action: Legal);
287 setIndexedStoreAction(IdxModes: ISD::PRE_INC, VT: MVT::f64, Action: Legal);
288 }
289
290 if (Subtarget.useCRBits()) {
291 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
292
293 if (isPPC64 || Subtarget.hasFPCVT()) {
294 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Promote);
295 AddPromotedToType(Opc: ISD::STRICT_SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
296 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Promote);
297 AddPromotedToType(Opc: ISD::STRICT_UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
298
299 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Promote);
300 AddPromotedToType(Opc: ISD::SINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
301 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Promote);
302 AddPromotedToType(Opc: ISD::UINT_TO_FP, OrigVT: MVT::i1, DestVT: RegVT);
303
304 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i1, Action: Promote);
305 AddPromotedToType(Opc: ISD::STRICT_FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
306 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i1, Action: Promote);
307 AddPromotedToType(Opc: ISD::STRICT_FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
308
309 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i1, Action: Promote);
310 AddPromotedToType(Opc: ISD::FP_TO_SINT, OrigVT: MVT::i1, DestVT: RegVT);
311 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i1, Action: Promote);
312 AddPromotedToType(Opc: ISD::FP_TO_UINT, OrigVT: MVT::i1, DestVT: RegVT);
313 } else {
314 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i1, Action: Custom);
315 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i1, Action: Custom);
316 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i1, Action: Custom);
317 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i1, Action: Custom);
318 }
319
320 // PowerPC does not support direct load/store of condition registers.
321 setOperationAction(Op: ISD::LOAD, VT: MVT::i1, Action: Custom);
322 setOperationAction(Op: ISD::STORE, VT: MVT::i1, Action: Custom);
323
324 // FIXME: Remove this once the ANDI glue bug is fixed:
325 if (ANDIGlueBug)
326 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::i1, Action: Custom);
327
328 for (MVT VT : MVT::integer_valuetypes()) {
329 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
330 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: MVT::i1, Action: Promote);
331 setTruncStoreAction(ValVT: VT, MemVT: MVT::i1, Action: Expand);
332 }
333
334 addRegisterClass(VT: MVT::i1, RC: &PPC::CRBITRCRegClass);
335 }
336
337 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
338 // PPC (the libcall is not available).
339 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
340 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
341 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::ppcf128, Action: Custom);
342 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::ppcf128, Action: Custom);
343
344 // We do not currently implement these libm ops for PowerPC.
345 setOperationAction(Op: ISD::FFLOOR, VT: MVT::ppcf128, Action: Expand);
346 setOperationAction(Op: ISD::FCEIL, VT: MVT::ppcf128, Action: Expand);
347 setOperationAction(Op: ISD::FTRUNC, VT: MVT::ppcf128, Action: Expand);
348 setOperationAction(Op: ISD::FRINT, VT: MVT::ppcf128, Action: Expand);
349 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::ppcf128, Action: Expand);
350 setOperationAction(Op: ISD::FREM, VT: MVT::ppcf128, Action: LibCall);
351
352 // PowerPC has no SREM/UREM instructions unless we are on P9
353 // On P9 we may use a hardware instruction to compute the remainder.
354 // When the result of both the remainder and the division is required it is
355 // more efficient to compute the remainder from the result of the division
356 // rather than use the remainder instruction. The instructions are legalized
357 // directly because the DivRemPairsPass performs the transformation at the IR
358 // level.
359 if (Subtarget.isISA3_0()) {
360 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Legal);
361 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Legal);
362 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Legal);
363 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Legal);
364 } else {
365 setOperationAction(Op: ISD::SREM, VT: MVT::i32, Action: Expand);
366 setOperationAction(Op: ISD::UREM, VT: MVT::i32, Action: Expand);
367 setOperationAction(Op: ISD::SREM, VT: MVT::i64, Action: Expand);
368 setOperationAction(Op: ISD::UREM, VT: MVT::i64, Action: Expand);
369 }
370
371 // Don't use SMUL_LOHI/UMUL_LOHI or SDIVREM/UDIVREM to lower SREM/UREM.
372 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i32, Action: Expand);
373 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i32, Action: Expand);
374 setOperationAction(Op: ISD::UMUL_LOHI, VT: MVT::i64, Action: Expand);
375 setOperationAction(Op: ISD::SMUL_LOHI, VT: MVT::i64, Action: Expand);
376 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i32, Action: Expand);
377 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i32, Action: Expand);
378 setOperationAction(Op: ISD::UDIVREM, VT: MVT::i64, Action: Expand);
379 setOperationAction(Op: ISD::SDIVREM, VT: MVT::i64, Action: Expand);
380
381 // Handle constrained floating-point operations of scalar.
382 // TODO: Handle SPE specific operation.
383 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f32, Action: Legal);
384 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f32, Action: Legal);
385 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f32, Action: Legal);
386 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f32, Action: Legal);
387 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
388
389 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f64, Action: Legal);
390 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f64, Action: Legal);
391 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f64, Action: Legal);
392 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f64, Action: Legal);
393
394 if (!Subtarget.hasSPE()) {
395 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f32, Action: Legal);
396 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f64, Action: Legal);
397 }
398
399 if (Subtarget.hasVSX()) {
400 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f32, Action: Legal);
401 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f64, Action: Legal);
402 }
403
404 if (Subtarget.hasFSQRT()) {
405 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f32, Action: Legal);
406 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f64, Action: Legal);
407 }
408
409 if (Subtarget.hasFPRND()) {
410 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f32, Action: Legal);
411 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f32, Action: Legal);
412 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f32, Action: Legal);
413 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f32, Action: Legal);
414
415 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f64, Action: Legal);
416 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f64, Action: Legal);
417 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f64, Action: Legal);
418 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f64, Action: Legal);
419 }
420
421 // We don't support sin/cos/sqrt/fmod/pow
422 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Expand);
423 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Expand);
424 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f64, Action: Expand);
425 setOperationAction(Op: ISD::FREM, VT: MVT::f64, Action: LibCall);
426 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Expand);
427 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Expand);
428 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Expand);
429 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f32, Action: Expand);
430 setOperationAction(Op: ISD::FREM, VT: MVT::f32, Action: LibCall);
431 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Expand);
432
433 // MASS transformation for LLVM intrinsics with replicating fast-math flag
434 // to be consistent to PPCGenScalarMASSEntries pass
435 if (TM.getOptLevel() == CodeGenOptLevel::Aggressive) {
436 setOperationAction(Op: ISD::FSIN , VT: MVT::f64, Action: Custom);
437 setOperationAction(Op: ISD::FCOS , VT: MVT::f64, Action: Custom);
438 setOperationAction(Op: ISD::FPOW , VT: MVT::f64, Action: Custom);
439 setOperationAction(Op: ISD::FLOG, VT: MVT::f64, Action: Custom);
440 setOperationAction(Op: ISD::FLOG10, VT: MVT::f64, Action: Custom);
441 setOperationAction(Op: ISD::FEXP, VT: MVT::f64, Action: Custom);
442 setOperationAction(Op: ISD::FSIN , VT: MVT::f32, Action: Custom);
443 setOperationAction(Op: ISD::FCOS , VT: MVT::f32, Action: Custom);
444 setOperationAction(Op: ISD::FPOW , VT: MVT::f32, Action: Custom);
445 setOperationAction(Op: ISD::FLOG, VT: MVT::f32, Action: Custom);
446 setOperationAction(Op: ISD::FLOG10, VT: MVT::f32, Action: Custom);
447 setOperationAction(Op: ISD::FEXP, VT: MVT::f32, Action: Custom);
448 }
449
450 if (Subtarget.hasSPE()) {
451 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Expand);
452 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Expand);
453 } else {
454 setOperationAction(Op: ISD::FMA , VT: MVT::f64, Action: Legal);
455 setOperationAction(Op: ISD::FMA , VT: MVT::f32, Action: Legal);
456 setOperationAction(Op: ISD::GET_ROUNDING, VT: MVT::i32, Action: Custom);
457 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
458 }
459
460 if (Subtarget.hasSPE())
461 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
462
463 // If we're enabling GP optimizations, use hardware square root
464 if (!Subtarget.hasFSQRT() && !(Subtarget.hasFRSQRTE() && Subtarget.hasFRE()))
465 setOperationAction(Op: ISD::FSQRT, VT: MVT::f64, Action: Expand);
466
467 if (!Subtarget.hasFSQRT() &&
468 !(Subtarget.hasFRSQRTES() && Subtarget.hasFRES()))
469 setOperationAction(Op: ISD::FSQRT, VT: MVT::f32, Action: Expand);
470
471 if (Subtarget.hasFCPSGN()) {
472 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Legal);
473 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Legal);
474 } else {
475 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f64, Action: Expand);
476 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f32, Action: Expand);
477 }
478
479 if (Subtarget.hasFPRND()) {
480 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f64, Action: Legal);
481 setOperationAction(Op: ISD::FCEIL, VT: MVT::f64, Action: Legal);
482 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f64, Action: Legal);
483 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
484
485 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f32, Action: Legal);
486 setOperationAction(Op: ISD::FCEIL, VT: MVT::f32, Action: Legal);
487 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f32, Action: Legal);
488 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
489 }
490
491 // Prior to P10, PowerPC does not have BSWAP, but we can use vector BSWAP
492 // instruction xxbrd to speed up scalar BSWAP64.
493 if (Subtarget.isISA3_1()) {
494 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Legal);
495 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64, Action: Legal);
496 } else {
497 setOperationAction(Op: ISD::BSWAP, VT: MVT::i32, Action: Expand);
498 setOperationAction(Op: ISD::BSWAP, VT: MVT::i64,
499 Action: ((Subtarget.hasP8Vector()) && isPPC64) ? Custom
500 : Expand);
501 }
502
503 // CTPOP or CTTZ were introduced in P8/P9 respectively
504 if (Subtarget.isISA3_0()) {
505 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Legal);
506 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Legal);
507 } else {
508 setOperationAction(Op: ISD::CTTZ , VT: MVT::i32 , Action: Expand);
509 setOperationAction(Op: ISD::CTTZ , VT: MVT::i64 , Action: Expand);
510 }
511
512 if (Subtarget.hasPOPCNTD() == PPCSubtarget::POPCNTD_Fast) {
513 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Legal);
514 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Legal);
515 } else {
516 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32 , Action: Expand);
517 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64 , Action: Expand);
518 }
519
520 // PowerPC does not have ROTR
521 setOperationAction(Op: ISD::ROTR, VT: MVT::i32 , Action: Expand);
522 setOperationAction(Op: ISD::ROTR, VT: MVT::i64 , Action: Expand);
523
524 if (!Subtarget.useCRBits()) {
525 // PowerPC does not have Select
526 setOperationAction(Op: ISD::SELECT, VT: MVT::i32, Action: Expand);
527 setOperationAction(Op: ISD::SELECT, VT: MVT::i64, Action: Expand);
528 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Expand);
529 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Expand);
530 }
531
532 // PowerPC wants to turn select_cc of FP into fsel when possible.
533 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Custom);
534 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Custom);
535
536 // PowerPC wants to optimize integer setcc a bit
537 if (!Subtarget.useCRBits())
538 setOperationAction(Op: ISD::SETCC, VT: MVT::i32, Action: Custom);
539
540 if (Subtarget.hasFPU()) {
541 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Legal);
542 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Legal);
543 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Legal);
544
545 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Legal);
546 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Legal);
547 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Legal);
548 }
549
550 // PowerPC does not have BRCOND which requires SetCC
551 if (!Subtarget.useCRBits())
552 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Expand);
553
554 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
555
556 if (Subtarget.hasSPE()) {
557 // SPE has built-in conversions
558 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Legal);
559 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Legal);
560 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Legal);
561 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Legal);
562 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Legal);
563 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Legal);
564
565 // SPE supports signaling compare of f32/f64.
566 // But it doesn't comply IEEE-754 rules for comparing
567 // special values like NaNs, Infs.
568 setOperationAction(Op: ISD::SETCC, VT: MVT::f32, Action: Custom);
569 setOperationAction(Op: ISD::SETCC, VT: MVT::f64, Action: Custom);
570 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f32, Action: Custom);
571 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f64, Action: Custom);
572 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f32, Action: Custom);
573 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f64, Action: Custom);
574 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Custom);
575 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Custom);
576 } else {
577 // PowerPC turns FP_TO_SINT into FCTIWZ and some load/stores.
578 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
579 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
580
581 // PowerPC does not have [U|S]INT_TO_FP
582 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Expand);
583 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Expand);
584 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Expand);
585 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Expand);
586 }
587
588 if (Subtarget.hasDirectMove() && isPPC64) {
589 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Legal);
590 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Legal);
591 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Legal);
592 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Legal);
593
594 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f64, Action: Custom);
595 setOperationAction(Op: ISD::STRICT_LRINT, VT: MVT::f32, Action: Custom);
596 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f64, Action: Custom);
597 setOperationAction(Op: ISD::STRICT_LLRINT, VT: MVT::f32, Action: Custom);
598 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f64, Action: Custom);
599 setOperationAction(Op: ISD::STRICT_LROUND, VT: MVT::f32, Action: Custom);
600 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f64, Action: Custom);
601 setOperationAction(Op: ISD::STRICT_LLROUND, VT: MVT::f32, Action: Custom);
602 } else {
603 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Expand);
604 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Expand);
605 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Expand);
606 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Expand);
607 }
608
609 // We cannot sextinreg(i1). Expand to shifts.
610 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
611
612 // Custom handling for PowerPC ucmp and scmp instructions
613 setOperationAction(Op: ISD::UCMP, VT: MVT::i32, Action: Custom);
614 setOperationAction(Op: ISD::UCMP, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
615 setOperationAction(Op: ISD::ABDU, VT: MVT::i32, Action: Custom);
616 setOperationAction(Op: ISD::ABDU, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
617 setOperationAction(Op: ISD::SCMP, VT: MVT::i32, Action: Custom);
618 setOperationAction(Op: ISD::SCMP, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
619
620 // NOTE: EH_SJLJ_SETJMP/_LONGJMP supported here is NOT intended to support
621 // SjLj exception handling but a light-weight setjmp/longjmp replacement to
622 // support continuation, user-level threading, and etc.. As a result, no
623 // other SjLj exception interfaces are implemented and please don't build
624 // your own exception handling based on them.
625 // LLVM/Clang supports zero-cost DWARF exception handling.
626 setOperationAction(Op: ISD::EH_SJLJ_SETJMP, VT: MVT::i32, Action: Custom);
627 setOperationAction(Op: ISD::EH_SJLJ_LONGJMP, VT: MVT::Other, Action: Custom);
628
629 // We want to legalize GlobalAddress and ConstantPool nodes into the
630 // appropriate instructions to materialize the address.
631 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i32, Action: Custom);
632 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i32, Action: Custom);
633 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i32, Action: Custom);
634 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i32, Action: Custom);
635 setOperationAction(Op: ISD::JumpTable, VT: MVT::i32, Action: Custom);
636 setOperationAction(Op: ISD::GlobalAddress, VT: MVT::i64, Action: Custom);
637 setOperationAction(Op: ISD::GlobalTLSAddress, VT: MVT::i64, Action: Custom);
638 setOperationAction(Op: ISD::BlockAddress, VT: MVT::i64, Action: Custom);
639 setOperationAction(Op: ISD::ConstantPool, VT: MVT::i64, Action: Custom);
640 setOperationAction(Op: ISD::JumpTable, VT: MVT::i64, Action: Custom);
641
642 // TRAP is legal.
643 setOperationAction(Op: ISD::TRAP, VT: MVT::Other, Action: Legal);
644
645 // TRAMPOLINE is custom lowered.
646 setOperationAction(Op: ISD::INIT_TRAMPOLINE, VT: MVT::Other, Action: Custom);
647 setOperationAction(Op: ISD::ADJUST_TRAMPOLINE, VT: MVT::Other, Action: Custom);
648
649 // VASTART needs to be custom lowered to use the VarArgsFrameIndex
650 setOperationAction(Op: ISD::VASTART , VT: MVT::Other, Action: Custom);
651
652 if (Subtarget.is64BitELFABI()) {
653 // VAARG always uses double-word chunks, so promote anything smaller.
654 setOperationAction(Op: ISD::VAARG, VT: MVT::i1, Action: Promote);
655 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i1, DestVT: MVT::i64);
656 setOperationAction(Op: ISD::VAARG, VT: MVT::i8, Action: Promote);
657 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i8, DestVT: MVT::i64);
658 setOperationAction(Op: ISD::VAARG, VT: MVT::i16, Action: Promote);
659 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i16, DestVT: MVT::i64);
660 setOperationAction(Op: ISD::VAARG, VT: MVT::i32, Action: Promote);
661 AddPromotedToType(Opc: ISD::VAARG, OrigVT: MVT::i32, DestVT: MVT::i64);
662 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
663 } else if (Subtarget.is32BitELFABI()) {
664 // VAARG is custom lowered with the 32-bit SVR4 ABI.
665 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Custom);
666 setOperationAction(Op: ISD::VAARG, VT: MVT::i64, Action: Custom);
667 } else
668 setOperationAction(Op: ISD::VAARG, VT: MVT::Other, Action: Expand);
669
670 // VACOPY is custom lowered with the 32-bit SVR4 ABI.
671 if (Subtarget.is32BitELFABI())
672 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Custom);
673 else
674 setOperationAction(Op: ISD::VACOPY , VT: MVT::Other, Action: Expand);
675
676 // Use the default implementation.
677 setOperationAction(Op: ISD::VAEND , VT: MVT::Other, Action: Expand);
678 setOperationAction(Op: ISD::STACKSAVE , VT: MVT::Other, Action: Expand);
679 setOperationAction(Op: ISD::STACKRESTORE , VT: MVT::Other, Action: Custom);
680 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i32 , Action: Custom);
681 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: MVT::i64 , Action: Custom);
682 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i32, Action: Custom);
683 setOperationAction(Op: ISD::GET_DYNAMIC_AREA_OFFSET, VT: MVT::i64, Action: Custom);
684 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
685 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i64, Action: Custom);
686
687 if (Subtarget.isISA3_0() && isPPC64) {
688 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v16i1, Action: Custom);
689 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v8i1, Action: Custom);
690 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v4i1, Action: Custom);
691 setOperationAction(Op: ISD::VP_STORE, VT: MVT::v2i1, Action: Custom);
692 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v16i1, Action: Custom);
693 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v8i1, Action: Custom);
694 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v4i1, Action: Custom);
695 setOperationAction(Op: ISD::VP_LOAD, VT: MVT::v2i1, Action: Custom);
696 }
697
698 // We want to custom lower some of our intrinsics.
699 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
700 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::f64, Action: Custom);
701 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::ppcf128, Action: Custom);
702 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v4f32, Action: Custom);
703 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::v2f64, Action: Custom);
704
705 // To handle counter-based loop conditions.
706 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::i1, Action: Custom);
707 setOperationAction(Op: ISD::INTRINSIC_W_CHAIN, VT: MVT::Other, Action: Custom);
708
709 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i8, Action: Custom);
710 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i16, Action: Custom);
711 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i32, Action: Custom);
712 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::Other, Action: Custom);
713
714 // Comparisons that require checking two conditions.
715 if (Subtarget.hasSPE()) {
716 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f32, Action: Expand);
717 setCondCodeAction(CCs: ISD::SETO, VT: MVT::f64, Action: Expand);
718 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f32, Action: Expand);
719 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::f64, Action: Expand);
720 }
721 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f32, Action: Expand);
722 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f64, Action: Expand);
723 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f32, Action: Expand);
724 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f64, Action: Expand);
725 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f32, Action: Expand);
726 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f64, Action: Expand);
727 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f32, Action: Expand);
728 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f64, Action: Expand);
729 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f32, Action: Expand);
730 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f64, Action: Expand);
731 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f32, Action: Expand);
732 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f64, Action: Expand);
733
734 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
735 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
736
737 if (Subtarget.has64BitSupport()) {
738 // They also have instructions for converting between i64 and fp.
739 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
740 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Expand);
741 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
742 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Expand);
743 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
744 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Expand);
745 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
746 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Expand);
747 // This is just the low 32 bits of a (signed) fp->i64 conversion.
748 // We cannot do this with Promote because i64 is not a legal type.
749 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
750 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
751
752 if (Subtarget.hasLFIWAX() || isPPC64) {
753 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
754 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
755 }
756 } else {
757 // PowerPC does not have FP_TO_UINT on 32-bit implementations.
758 if (Subtarget.hasSPE()) {
759 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Legal);
760 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Legal);
761 } else {
762 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Expand);
763 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Expand);
764 }
765 }
766
767 // With the instructions enabled under FPCVT, we can do everything.
768 if (Subtarget.hasFPCVT()) {
769 if (Subtarget.has64BitSupport()) {
770 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i64, Action: Custom);
771 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i64, Action: Custom);
772 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i64, Action: Custom);
773 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i64, Action: Custom);
774 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i64, Action: Custom);
775 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i64, Action: Custom);
776 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i64, Action: Custom);
777 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i64, Action: Custom);
778 }
779
780 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::i32, Action: Custom);
781 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::i32, Action: Custom);
782 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::i32, Action: Custom);
783 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::i32, Action: Custom);
784 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::i32, Action: Custom);
785 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::i32, Action: Custom);
786 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::i32, Action: Custom);
787 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::i32, Action: Custom);
788 }
789
790 if (Subtarget.use64BitRegs()) {
791 // 64-bit PowerPC implementations can support i64 types directly
792 addRegisterClass(VT: MVT::i64, RC: &PPC::G8RCRegClass);
793 // BUILD_PAIR can't be handled natively, and should be expanded to shl/or
794 setOperationAction(Op: ISD::BUILD_PAIR, VT: MVT::i64, Action: Expand);
795 // 64-bit PowerPC wants to expand i128 shifts itself.
796 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i64, Action: Custom);
797 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i64, Action: Custom);
798 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i64, Action: Custom);
799 } else {
800 // 32-bit PowerPC wants to expand i64 shifts itself.
801 setOperationAction(Op: ISD::SHL_PARTS, VT: MVT::i32, Action: Custom);
802 setOperationAction(Op: ISD::SRA_PARTS, VT: MVT::i32, Action: Custom);
803 setOperationAction(Op: ISD::SRL_PARTS, VT: MVT::i32, Action: Custom);
804 }
805
806 // PowerPC has better expansions for funnel shifts than the generic
807 // TargetLowering::expandFunnelShift.
808 if (Subtarget.has64BitSupport()) {
809 setOperationAction(Op: ISD::FSHL, VT: MVT::i64, Action: Custom);
810 setOperationAction(Op: ISD::FSHR, VT: MVT::i64, Action: Custom);
811 }
812 setOperationAction(Op: ISD::FSHL, VT: MVT::i32, Action: Custom);
813 setOperationAction(Op: ISD::FSHR, VT: MVT::i32, Action: Custom);
814
815 if (Subtarget.hasVSX()) {
816 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f64, Action: Legal);
817 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT: MVT::f32, Action: Legal);
818 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f64, Action: Legal);
819 setOperationAction(Op: ISD::FMINNUM_IEEE, VT: MVT::f32, Action: Legal);
820 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f64, Action: Legal);
821 setOperationAction(Op: ISD::FMAXNUM, VT: MVT::f32, Action: Legal);
822 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f64, Action: Legal);
823 setOperationAction(Op: ISD::FMINNUM, VT: MVT::f32, Action: Legal);
824 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f64, Action: Legal);
825 setOperationAction(Op: ISD::FCANONICALIZE, VT: MVT::f32, Action: Legal);
826 }
827
828 if (Subtarget.hasAltivec()) {
829 for (MVT VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
830 setOperationAction(Op: ISD::AVGCEILS, VT, Action: Legal);
831 setOperationAction(Op: ISD::AVGCEILU, VT, Action: Legal);
832 setOperationAction(Op: ISD::SADDSAT, VT, Action: Legal);
833 setOperationAction(Op: ISD::SSUBSAT, VT, Action: Legal);
834 setOperationAction(Op: ISD::UADDSAT, VT, Action: Legal);
835 setOperationAction(Op: ISD::USUBSAT, VT, Action: Legal);
836 }
837 // First set operation action for all vector types to expand. Then we
838 // will selectively turn on ones that can be effectively codegen'd.
839 for (MVT VT : MVT::fixedlen_vector_valuetypes()) {
840 // add/sub are legal for all supported vector VT's.
841 setOperationAction(Op: ISD::ADD, VT, Action: Legal);
842 setOperationAction(Op: ISD::SUB, VT, Action: Legal);
843
844 // For v2i64, these are only valid with P8Vector. This is corrected after
845 // the loop.
846 if (VT.getSizeInBits() <= 128 && VT.getScalarSizeInBits() <= 64) {
847 setOperationAction(Op: ISD::SMAX, VT, Action: Legal);
848 setOperationAction(Op: ISD::SMIN, VT, Action: Legal);
849 setOperationAction(Op: ISD::UMAX, VT, Action: Legal);
850 setOperationAction(Op: ISD::UMIN, VT, Action: Legal);
851 }
852 else {
853 setOperationAction(Op: ISD::SMAX, VT, Action: Expand);
854 setOperationAction(Op: ISD::SMIN, VT, Action: Expand);
855 setOperationAction(Op: ISD::UMAX, VT, Action: Expand);
856 setOperationAction(Op: ISD::UMIN, VT, Action: Expand);
857 }
858
859 if (Subtarget.hasVSX()) {
860 setOperationAction(Op: ISD::FMAXNUM_IEEE, VT, Action: Legal);
861 setOperationAction(Op: ISD::FMINNUM_IEEE, VT, Action: Legal);
862 setOperationAction(Op: ISD::FMAXNUM, VT, Action: Legal);
863 setOperationAction(Op: ISD::FMINNUM, VT, Action: Legal);
864 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Legal);
865 }
866
867 // Vector instructions introduced in P8
868 if (Subtarget.hasP8Altivec() && (VT.SimpleTy != MVT::v1i128)) {
869 setOperationAction(Op: ISD::CTPOP, VT, Action: Legal);
870 setOperationAction(Op: ISD::CTLZ, VT, Action: Legal);
871 }
872 else {
873 setOperationAction(Op: ISD::CTPOP, VT, Action: Expand);
874 setOperationAction(Op: ISD::CTLZ, VT, Action: Expand);
875 }
876
877 // Vector instructions introduced in P9
878 if (Subtarget.hasP9Altivec() && (VT.SimpleTy != MVT::v1i128))
879 setOperationAction(Op: ISD::CTTZ, VT, Action: Legal);
880 else
881 setOperationAction(Op: ISD::CTTZ, VT, Action: Expand);
882
883 // We promote all shuffles to v16i8.
884 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Promote);
885 AddPromotedToType (Opc: ISD::VECTOR_SHUFFLE, OrigVT: VT, DestVT: MVT::v16i8);
886
887 // We promote all non-typed operations to v4i32.
888 setOperationAction(Op: ISD::AND , VT, Action: Promote);
889 AddPromotedToType (Opc: ISD::AND , OrigVT: VT, DestVT: MVT::v4i32);
890 setOperationAction(Op: ISD::OR , VT, Action: Promote);
891 AddPromotedToType (Opc: ISD::OR , OrigVT: VT, DestVT: MVT::v4i32);
892 setOperationAction(Op: ISD::XOR , VT, Action: Promote);
893 AddPromotedToType (Opc: ISD::XOR , OrigVT: VT, DestVT: MVT::v4i32);
894 setOperationAction(Op: ISD::LOAD , VT, Action: Promote);
895 AddPromotedToType (Opc: ISD::LOAD , OrigVT: VT, DestVT: MVT::v4i32);
896 setOperationAction(Op: ISD::SELECT, VT, Action: Promote);
897 AddPromotedToType (Opc: ISD::SELECT, OrigVT: VT, DestVT: MVT::v4i32);
898 setOperationAction(Op: ISD::VSELECT, VT, Action: Legal);
899 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Promote);
900 AddPromotedToType (Opc: ISD::SELECT_CC, OrigVT: VT, DestVT: MVT::v4i32);
901 setOperationAction(Op: ISD::STORE, VT, Action: Promote);
902 AddPromotedToType (Opc: ISD::STORE, OrigVT: VT, DestVT: MVT::v4i32);
903
904 // No other operations are legal.
905 setOperationAction(Op: ISD::MUL , VT, Action: Expand);
906 setOperationAction(Op: ISD::SDIV, VT, Action: Expand);
907 setOperationAction(Op: ISD::SREM, VT, Action: Expand);
908 setOperationAction(Op: ISD::UDIV, VT, Action: Expand);
909 setOperationAction(Op: ISD::UREM, VT, Action: Expand);
910 setOperationAction(Op: ISD::FDIV, VT, Action: Expand);
911 setOperationAction(Op: ISD::FREM, VT, Action: Expand);
912 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
913 setOperationAction(Op: ISD::FSQRT, VT, Action: Expand);
914 setOperationAction(Op: ISD::FLOG, VT, Action: Expand);
915 setOperationAction(Op: ISD::FLOG10, VT, Action: Expand);
916 setOperationAction(Op: ISD::FLOG2, VT, Action: Expand);
917 setOperationAction(Op: ISD::FEXP, VT, Action: Expand);
918 setOperationAction(Op: ISD::FEXP2, VT, Action: Expand);
919 setOperationAction(Op: ISD::FSIN, VT, Action: Expand);
920 setOperationAction(Op: ISD::FCOS, VT, Action: Expand);
921 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
922 setOperationAction(Op: ISD::FFLOOR, VT, Action: Expand);
923 setOperationAction(Op: ISD::FCEIL, VT, Action: Expand);
924 setOperationAction(Op: ISD::FTRUNC, VT, Action: Expand);
925 setOperationAction(Op: ISD::FRINT, VT, Action: Expand);
926 setOperationAction(Op: ISD::FLDEXP, VT, Action: Expand);
927 setOperationAction(Op: ISD::FNEARBYINT, VT, Action: Expand);
928 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT, Action: Expand);
929 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT, Action: Expand);
930 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Expand);
931 setOperationAction(Op: ISD::MULHU, VT, Action: Expand);
932 setOperationAction(Op: ISD::MULHS, VT, Action: Expand);
933 setOperationAction(Op: ISD::UMUL_LOHI, VT, Action: Expand);
934 setOperationAction(Op: ISD::SMUL_LOHI, VT, Action: Expand);
935 setOperationAction(Op: ISD::UDIVREM, VT, Action: Expand);
936 setOperationAction(Op: ISD::SDIVREM, VT, Action: Expand);
937 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Expand);
938 setOperationAction(Op: ISD::FPOW, VT, Action: Expand);
939 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
940 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT, Action: Expand);
941 setOperationAction(Op: ISD::ROTL, VT, Action: Expand);
942 setOperationAction(Op: ISD::ROTR, VT, Action: Expand);
943
944 for (MVT InnerVT : MVT::fixedlen_vector_valuetypes()) {
945 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
946 setLoadExtAction(ExtType: ISD::SEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
947 setLoadExtAction(ExtType: ISD::ZEXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
948 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
949 }
950 }
951 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::v4i32, Action: Expand);
952 if (!Subtarget.hasP8Vector()) {
953 setOperationAction(Op: ISD::SMAX, VT: MVT::v2i64, Action: Expand);
954 setOperationAction(Op: ISD::SMIN, VT: MVT::v2i64, Action: Expand);
955 setOperationAction(Op: ISD::UMAX, VT: MVT::v2i64, Action: Expand);
956 setOperationAction(Op: ISD::UMIN, VT: MVT::v2i64, Action: Expand);
957 }
958
959 // We can custom expand all VECTOR_SHUFFLEs to VPERM, others we can handle
960 // with merges, splats, etc.
961 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v16i8, Action: Custom);
962
963 // Vector truncates to sub-word integer that fit in an Altivec/VSX register
964 // are cheap, so handle them before they get expanded to scalar.
965 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v8i8, Action: Custom);
966 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i8, Action: Custom);
967 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i8, Action: Custom);
968 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v4i16, Action: Custom);
969 setOperationAction(Op: ISD::TRUNCATE, VT: MVT::v2i16, Action: Custom);
970
971 setOperationAction(Op: ISD::AND , VT: MVT::v4i32, Action: Legal);
972 setOperationAction(Op: ISD::OR , VT: MVT::v4i32, Action: Legal);
973 setOperationAction(Op: ISD::XOR , VT: MVT::v4i32, Action: Legal);
974 setOperationAction(Op: ISD::LOAD , VT: MVT::v4i32, Action: Legal);
975 setOperationAction(Op: ISD::SELECT, VT: MVT::v4i32,
976 Action: Subtarget.useCRBits() ? Legal : Expand);
977 setOperationAction(Op: ISD::STORE , VT: MVT::v4i32, Action: Legal);
978 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
979 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
980 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
981 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
982 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v4i32, Action: Legal);
983 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v4i32, Action: Legal);
984 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i32, Action: Legal);
985 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i32, Action: Legal);
986 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v4f32, Action: Legal);
987 setOperationAction(Op: ISD::FCEIL, VT: MVT::v4f32, Action: Legal);
988 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v4f32, Action: Legal);
989 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::v4f32, Action: Legal);
990
991 // Custom lowering ROTL v1i128 to VECTOR_SHUFFLE v16i8.
992 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Custom);
993 // With hasAltivec set, we can lower ISD::ROTL to vrl(b|h|w).
994 if (Subtarget.hasAltivec())
995 for (auto VT : {MVT::v4i32, MVT::v8i16, MVT::v16i8})
996 setOperationAction(Op: ISD::ROTL, VT, Action: Legal);
997 // With hasP8Altivec set, we can lower ISD::ROTL to vrld.
998 if (Subtarget.hasP8Altivec())
999 setOperationAction(Op: ISD::ROTL, VT: MVT::v2i64, Action: Legal);
1000
1001 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VRRCRegClass);
1002 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VRRCRegClass);
1003 addRegisterClass(VT: MVT::v8i16, RC: &PPC::VRRCRegClass);
1004 addRegisterClass(VT: MVT::v16i8, RC: &PPC::VRRCRegClass);
1005
1006 setOperationAction(Op: ISD::MUL, VT: MVT::v4f32, Action: Legal);
1007 setOperationAction(Op: ISD::FMA, VT: MVT::v4f32, Action: Legal);
1008
1009 if (Subtarget.hasVSX()) {
1010 setOperationAction(Op: ISD::FDIV, VT: MVT::v4f32, Action: Legal);
1011 setOperationAction(Op: ISD::FSQRT, VT: MVT::v4f32, Action: Legal);
1012 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2f64, Action: Custom);
1013 }
1014
1015 if (Subtarget.hasP8Altivec())
1016 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Legal);
1017 else
1018 setOperationAction(Op: ISD::MUL, VT: MVT::v4i32, Action: Custom);
1019
1020 if (Subtarget.isISA3_1()) {
1021 setOperationAction(Op: ISD::MUL, VT: MVT::v2i64, Action: Legal);
1022 setOperationAction(Op: ISD::MULHS, VT: MVT::v2i64, Action: Legal);
1023 setOperationAction(Op: ISD::MULHU, VT: MVT::v2i64, Action: Legal);
1024 setOperationAction(Op: ISD::MULHS, VT: MVT::v4i32, Action: Legal);
1025 setOperationAction(Op: ISD::MULHU, VT: MVT::v4i32, Action: Legal);
1026 setOperationAction(Op: ISD::UDIV, VT: MVT::v2i64, Action: Legal);
1027 setOperationAction(Op: ISD::SDIV, VT: MVT::v2i64, Action: Legal);
1028 setOperationAction(Op: ISD::UDIV, VT: MVT::v4i32, Action: Legal);
1029 setOperationAction(Op: ISD::SDIV, VT: MVT::v4i32, Action: Legal);
1030 setOperationAction(Op: ISD::UREM, VT: MVT::v2i64, Action: Legal);
1031 setOperationAction(Op: ISD::SREM, VT: MVT::v2i64, Action: Legal);
1032 setOperationAction(Op: ISD::UREM, VT: MVT::v4i32, Action: Legal);
1033 setOperationAction(Op: ISD::SREM, VT: MVT::v4i32, Action: Legal);
1034 setOperationAction(Op: ISD::UREM, VT: MVT::v1i128, Action: Legal);
1035 setOperationAction(Op: ISD::SREM, VT: MVT::v1i128, Action: Legal);
1036 setOperationAction(Op: ISD::UDIV, VT: MVT::v1i128, Action: Legal);
1037 setOperationAction(Op: ISD::SDIV, VT: MVT::v1i128, Action: Legal);
1038 setOperationAction(Op: ISD::ROTL, VT: MVT::v1i128, Action: Legal);
1039 }
1040
1041 setOperationAction(Op: ISD::MUL, VT: MVT::v8i16, Action: Legal);
1042 setOperationAction(Op: ISD::MUL, VT: MVT::v16i8, Action: Custom);
1043
1044 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Custom);
1045 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Custom);
1046 // LE is P8+/64-bit so direct moves are supported and these operations
1047 // are legal. The custom transformation requires 64-bit since we need a
1048 // pair of stores that will cover a 128-bit load for P10.
1049 if (EnableP10StoreForward && isPPC64 && !Subtarget.isLittleEndian()) {
1050 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Custom);
1051 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Custom);
1052 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Custom);
1053 }
1054
1055 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v16i8, Action: Custom);
1056 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v8i16, Action: Custom);
1057 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i32, Action: Custom);
1058 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4f32, Action: Custom);
1059
1060 // Altivec does not contain unordered floating-point compare instructions
1061 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v4f32, Action: Expand);
1062 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v4f32, Action: Expand);
1063 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v4f32, Action: Expand);
1064 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v4f32, Action: Expand);
1065
1066 if (Subtarget.hasVSX()) {
1067 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2f64, Action: Legal);
1068 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1069 if (Subtarget.hasP8Vector()) {
1070 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4f32, Action: Legal);
1071 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4f32, Action: Legal);
1072 }
1073 if (Subtarget.hasDirectMove() && isPPC64) {
1074 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v16i8, Action: Legal);
1075 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v8i16, Action: Legal);
1076 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v4i32, Action: Legal);
1077 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT: MVT::v2i64, Action: Legal);
1078 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1079 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1080 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1081 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1082 }
1083 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2f64, Action: Legal);
1084
1085 // The nearbyint variants are not allowed to raise the inexact exception
1086 // so we can only code-gen them with fpexcept.ignore.
1087 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f64, Action: Custom);
1088 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f32, Action: Custom);
1089 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v2f64, Action: Custom);
1090 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::v4f32, Action: Custom);
1091
1092 setOperationAction(Op: ISD::FFLOOR, VT: MVT::v2f64, Action: Legal);
1093 setOperationAction(Op: ISD::FCEIL, VT: MVT::v2f64, Action: Legal);
1094 setOperationAction(Op: ISD::FTRUNC, VT: MVT::v2f64, Action: Legal);
1095 setOperationAction(Op: ISD::FRINT, VT: MVT::v2f64, Action: Legal);
1096 setOperationAction(Op: ISD::FROUND, VT: MVT::v2f64, Action: Legal);
1097 setOperationAction(Op: ISD::FROUND, VT: MVT::f64, Action: Legal);
1098 setOperationAction(Op: ISD::FRINT, VT: MVT::f64, Action: Legal);
1099
1100 setOperationAction(Op: ISD::FRINT, VT: MVT::v4f32, Action: Legal);
1101 setOperationAction(Op: ISD::FROUND, VT: MVT::v4f32, Action: Legal);
1102 setOperationAction(Op: ISD::FROUND, VT: MVT::f32, Action: Legal);
1103 setOperationAction(Op: ISD::FRINT, VT: MVT::f32, Action: Legal);
1104
1105 setOperationAction(Op: ISD::MUL, VT: MVT::v2f64, Action: Legal);
1106 setOperationAction(Op: ISD::FMA, VT: MVT::v2f64, Action: Legal);
1107
1108 setOperationAction(Op: ISD::FDIV, VT: MVT::v2f64, Action: Legal);
1109 setOperationAction(Op: ISD::FSQRT, VT: MVT::v2f64, Action: Legal);
1110
1111 // Share the Altivec comparison restrictions.
1112 setCondCodeAction(CCs: ISD::SETUO, VT: MVT::v2f64, Action: Expand);
1113 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::v2f64, Action: Expand);
1114 setCondCodeAction(CCs: ISD::SETO, VT: MVT::v2f64, Action: Expand);
1115 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::v2f64, Action: Expand);
1116
1117 setOperationAction(Op: ISD::LOAD, VT: MVT::v2f64, Action: Legal);
1118 setOperationAction(Op: ISD::STORE, VT: MVT::v2f64, Action: Legal);
1119
1120 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2f64, Action: Custom);
1121
1122 if (Subtarget.hasP8Vector())
1123 addRegisterClass(VT: MVT::f32, RC: &PPC::VSSRCRegClass);
1124
1125 addRegisterClass(VT: MVT::f64, RC: &PPC::VSFRCRegClass);
1126
1127 addRegisterClass(VT: MVT::v4i32, RC: &PPC::VSRCRegClass);
1128 addRegisterClass(VT: MVT::v4f32, RC: &PPC::VSRCRegClass);
1129 addRegisterClass(VT: MVT::v2f64, RC: &PPC::VSRCRegClass);
1130
1131 if (Subtarget.hasP8Altivec()) {
1132 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Legal);
1133 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Legal);
1134 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Legal);
1135
1136 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1137 // SRL, but not for SRA because of the instructions available:
1138 // VS{RL} and VS{RL}O. However due to direct move costs, it's not worth
1139 // doing
1140 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Expand);
1141 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Expand);
1142 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1143
1144 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Legal);
1145 }
1146 else {
1147 setOperationAction(Op: ISD::SHL, VT: MVT::v2i64, Action: Expand);
1148 setOperationAction(Op: ISD::SRA, VT: MVT::v2i64, Action: Expand);
1149 setOperationAction(Op: ISD::SRL, VT: MVT::v2i64, Action: Expand);
1150
1151 setOperationAction(Op: ISD::SETCC, VT: MVT::v2i64, Action: Custom);
1152
1153 // VSX v2i64 only supports non-arithmetic operations.
1154 setOperationAction(Op: ISD::ADD, VT: MVT::v2i64, Action: Expand);
1155 setOperationAction(Op: ISD::SUB, VT: MVT::v2i64, Action: Expand);
1156 }
1157
1158 if (Subtarget.isISA3_1())
1159 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Legal);
1160 else
1161 setOperationAction(Op: ISD::SETCC, VT: MVT::v1i128, Action: Expand);
1162
1163 setOperationAction(Op: ISD::LOAD, VT: MVT::v2i64, Action: Promote);
1164 AddPromotedToType (Opc: ISD::LOAD, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1165 setOperationAction(Op: ISD::STORE, VT: MVT::v2i64, Action: Promote);
1166 AddPromotedToType (Opc: ISD::STORE, OrigVT: MVT::v2i64, DestVT: MVT::v2f64);
1167
1168 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT: MVT::v2i64, Action: Custom);
1169
1170 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1171 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1172 setOperationAction(Op: ISD::STRICT_FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1173 setOperationAction(Op: ISD::STRICT_FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1174 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1175 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i64, Action: Legal);
1176 setOperationAction(Op: ISD::FP_TO_SINT, VT: MVT::v2i64, Action: Legal);
1177 setOperationAction(Op: ISD::FP_TO_UINT, VT: MVT::v2i64, Action: Legal);
1178
1179 // Custom handling for partial vectors of integers converted to
1180 // floating point. We already have optimal handling for v2i32 through
1181 // the DAG combine, so those aren't necessary.
1182 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1183 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1184 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1185 setOperationAction(Op: ISD::STRICT_UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1186 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1187 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1188 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1189 setOperationAction(Op: ISD::STRICT_SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1190 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1191 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1192 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1193 setOperationAction(Op: ISD::UINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1194 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i8, Action: Custom);
1195 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i8, Action: Custom);
1196 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v2i16, Action: Custom);
1197 setOperationAction(Op: ISD::SINT_TO_FP, VT: MVT::v4i16, Action: Custom);
1198
1199 setOperationAction(Op: ISD::FNEG, VT: MVT::v4f32, Action: Legal);
1200 setOperationAction(Op: ISD::FNEG, VT: MVT::v2f64, Action: Legal);
1201 setOperationAction(Op: ISD::FABS, VT: MVT::v4f32, Action: Legal);
1202 setOperationAction(Op: ISD::FABS, VT: MVT::v2f64, Action: Legal);
1203 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v4f32, Action: Legal);
1204 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::v2f64, Action: Legal);
1205
1206 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2i64, Action: Custom);
1207 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v2f64, Action: Custom);
1208
1209 // Handle constrained floating-point operations of vector.
1210 // The predictor is `hasVSX` because altivec instruction has
1211 // no exception but VSX vector instruction has.
1212 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v4f32, Action: Legal);
1213 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v4f32, Action: Legal);
1214 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v4f32, Action: Legal);
1215 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v4f32, Action: Legal);
1216 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v4f32, Action: Legal);
1217 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v4f32, Action: Legal);
1218 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v4f32, Action: Legal);
1219 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v4f32, Action: Legal);
1220 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v4f32, Action: Legal);
1221 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v4f32, Action: Legal);
1222 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v4f32, Action: Legal);
1223 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v4f32, Action: Legal);
1224 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v4f32, Action: Legal);
1225
1226 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::v2f64, Action: Legal);
1227 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::v2f64, Action: Legal);
1228 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::v2f64, Action: Legal);
1229 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::v2f64, Action: Legal);
1230 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::v2f64, Action: Legal);
1231 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::v2f64, Action: Legal);
1232 setOperationAction(Op: ISD::STRICT_FMAXNUM, VT: MVT::v2f64, Action: Legal);
1233 setOperationAction(Op: ISD::STRICT_FMINNUM, VT: MVT::v2f64, Action: Legal);
1234 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::v2f64, Action: Legal);
1235 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::v2f64, Action: Legal);
1236 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::v2f64, Action: Legal);
1237 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::v2f64, Action: Legal);
1238 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::v2f64, Action: Legal);
1239
1240 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VSRCRegClass);
1241 addRegisterClass(VT: MVT::f128, RC: &PPC::VRRCRegClass);
1242
1243 for (MVT FPT : MVT::fp_valuetypes())
1244 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f128, MemVT: FPT, Action: Expand);
1245
1246 // Expand the SELECT to SELECT_CC
1247 setOperationAction(Op: ISD::SELECT, VT: MVT::f128, Action: Expand);
1248
1249 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f64, Action: Expand);
1250 setTruncStoreAction(ValVT: MVT::f128, MemVT: MVT::f32, Action: Expand);
1251
1252 // No implementation for these ops for PowerPC.
1253 setOperationAction(Op: ISD::FSINCOS, VT: MVT::f128, Action: Expand);
1254 setOperationAction(Op: ISD::FSIN, VT: MVT::f128, Action: Expand);
1255 setOperationAction(Op: ISD::FCOS, VT: MVT::f128, Action: Expand);
1256 setOperationAction(Op: ISD::FPOW, VT: MVT::f128, Action: Expand);
1257 setOperationAction(Op: ISD::FPOWI, VT: MVT::f128, Action: Expand);
1258 setOperationAction(Op: ISD::FREM, VT: MVT::f128, Action: LibCall);
1259 }
1260
1261 if (Subtarget.hasP8Altivec()) {
1262 addRegisterClass(VT: MVT::v2i64, RC: &PPC::VRRCRegClass);
1263 addRegisterClass(VT: MVT::v1i128, RC: &PPC::VRRCRegClass);
1264 }
1265
1266 if (Subtarget.hasP9Vector()) {
1267 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Custom);
1268 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4f32, Action: Custom);
1269
1270 // Test data class instructions store results in CR bits.
1271 if (Subtarget.useCRBits()) {
1272 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Custom);
1273 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Custom);
1274 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f128, Action: Custom);
1275 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::ppcf128, Action: Custom);
1276 }
1277
1278 // 128 bit shifts can be accomplished via 3 instructions for SHL and
1279 // SRL, but not for SRA because of the instructions available:
1280 // VS{RL} and VS{RL}O.
1281 setOperationAction(Op: ISD::SHL, VT: MVT::v1i128, Action: Legal);
1282 setOperationAction(Op: ISD::SRL, VT: MVT::v1i128, Action: Legal);
1283 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Expand);
1284
1285 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: Legal);
1286 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: Legal);
1287 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Legal);
1288 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Legal);
1289 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Legal);
1290
1291 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Legal);
1292 setCondCodeAction(CCs: ISD::SETULT, VT: MVT::f128, Action: Expand);
1293 setCondCodeAction(CCs: ISD::SETUGT, VT: MVT::f128, Action: Expand);
1294 setCondCodeAction(CCs: ISD::SETUEQ, VT: MVT::f128, Action: Expand);
1295 setCondCodeAction(CCs: ISD::SETOGE, VT: MVT::f128, Action: Expand);
1296 setCondCodeAction(CCs: ISD::SETOLE, VT: MVT::f128, Action: Expand);
1297 setCondCodeAction(CCs: ISD::SETONE, VT: MVT::f128, Action: Expand);
1298
1299 setOperationAction(Op: ISD::FTRUNC, VT: MVT::f128, Action: Legal);
1300 setOperationAction(Op: ISD::FRINT, VT: MVT::f128, Action: Legal);
1301 setOperationAction(Op: ISD::FFLOOR, VT: MVT::f128, Action: Legal);
1302 setOperationAction(Op: ISD::FCEIL, VT: MVT::f128, Action: Legal);
1303 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f128, Action: Legal);
1304 setOperationAction(Op: ISD::FROUND, VT: MVT::f128, Action: Legal);
1305
1306 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f64, Action: Legal);
1307 setOperationAction(Op: ISD::FP_ROUND, VT: MVT::f32, Action: Legal);
1308 setOperationAction(Op: ISD::BITCAST, VT: MVT::i128, Action: Custom);
1309
1310 // Handle constrained floating-point operations of fp128
1311 setOperationAction(Op: ISD::STRICT_FADD, VT: MVT::f128, Action: Legal);
1312 setOperationAction(Op: ISD::STRICT_FSUB, VT: MVT::f128, Action: Legal);
1313 setOperationAction(Op: ISD::STRICT_FMUL, VT: MVT::f128, Action: Legal);
1314 setOperationAction(Op: ISD::STRICT_FDIV, VT: MVT::f128, Action: Legal);
1315 setOperationAction(Op: ISD::STRICT_FMA, VT: MVT::f128, Action: Legal);
1316 setOperationAction(Op: ISD::STRICT_FSQRT, VT: MVT::f128, Action: Legal);
1317 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Legal);
1318 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f64, Action: Legal);
1319 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
1320 setOperationAction(Op: ISD::STRICT_FRINT, VT: MVT::f128, Action: Legal);
1321 setOperationAction(Op: ISD::STRICT_FNEARBYINT, VT: MVT::f128, Action: Legal);
1322 setOperationAction(Op: ISD::STRICT_FFLOOR, VT: MVT::f128, Action: Legal);
1323 setOperationAction(Op: ISD::STRICT_FCEIL, VT: MVT::f128, Action: Legal);
1324 setOperationAction(Op: ISD::STRICT_FTRUNC, VT: MVT::f128, Action: Legal);
1325 setOperationAction(Op: ISD::STRICT_FROUND, VT: MVT::f128, Action: Legal);
1326 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::v2f32, Action: Custom);
1327 setOperationAction(Op: ISD::BSWAP, VT: MVT::v8i16, Action: Legal);
1328 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i32, Action: Legal);
1329 setOperationAction(Op: ISD::BSWAP, VT: MVT::v2i64, Action: Legal);
1330 setOperationAction(Op: ISD::BSWAP, VT: MVT::v1i128, Action: Legal);
1331 } else if (Subtarget.hasVSX()) {
1332 setOperationAction(Op: ISD::LOAD, VT: MVT::f128, Action: Promote);
1333 setOperationAction(Op: ISD::STORE, VT: MVT::f128, Action: Promote);
1334
1335 AddPromotedToType(Opc: ISD::LOAD, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1336 AddPromotedToType(Opc: ISD::STORE, OrigVT: MVT::f128, DestVT: MVT::v4i32);
1337
1338 // Set FADD/FSUB as libcall to avoid the legalizer to expand the
1339 // fp_to_uint and int_to_fp.
1340 setOperationAction(Op: ISD::FADD, VT: MVT::f128, Action: LibCall);
1341 setOperationAction(Op: ISD::FSUB, VT: MVT::f128, Action: LibCall);
1342
1343 setOperationAction(Op: ISD::FMUL, VT: MVT::f128, Action: Expand);
1344 setOperationAction(Op: ISD::FDIV, VT: MVT::f128, Action: Expand);
1345 setOperationAction(Op: ISD::FNEG, VT: MVT::f128, Action: Expand);
1346 setOperationAction(Op: ISD::FABS, VT: MVT::f128, Action: Expand);
1347 setOperationAction(Op: ISD::FSQRT, VT: MVT::f128, Action: Expand);
1348 setOperationAction(Op: ISD::FMA, VT: MVT::f128, Action: Expand);
1349 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f128, Action: Expand);
1350
1351 // Expand the fp_extend if the target type is fp128.
1352 setOperationAction(Op: ISD::FP_EXTEND, VT: MVT::f128, Action: Expand);
1353 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f128, Action: Expand);
1354
1355 // Expand the fp_round if the source type is fp128.
1356 for (MVT VT : {MVT::f32, MVT::f64}) {
1357 setOperationAction(Op: ISD::FP_ROUND, VT, Action: Custom);
1358 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT, Action: Custom);
1359 }
1360
1361 setOperationAction(Op: ISD::SETCC, VT: MVT::f128, Action: Custom);
1362 setOperationAction(Op: ISD::STRICT_FSETCC, VT: MVT::f128, Action: Custom);
1363 setOperationAction(Op: ISD::STRICT_FSETCCS, VT: MVT::f128, Action: Custom);
1364 setOperationAction(Op: ISD::BR_CC, VT: MVT::f128, Action: Custom);
1365
1366 // Lower following f128 select_cc pattern:
1367 // select_cc x, y, tv, fv, cc -> select_cc (setcc x, y, cc), 0, tv, fv, NE
1368 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1369
1370 // We need to handle f128 SELECT_CC with integer result type.
1371 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i32, Action: Custom);
1372 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::i64, Action: isPPC64 ? Custom : Expand);
1373 }
1374
1375 if (Subtarget.hasP9Altivec()) {
1376 if (Subtarget.isISA3_1()) {
1377 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v2i64, Action: Legal);
1378 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Legal);
1379 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Legal);
1380 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v4i32, Action: Legal);
1381 } else {
1382 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v8i16, Action: Custom);
1383 setOperationAction(Op: ISD::INSERT_VECTOR_ELT, VT: MVT::v16i8, Action: Custom);
1384 }
1385 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i8, Action: Legal);
1386 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i16, Action: Legal);
1387 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v4i32, Action: Legal);
1388 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i8, Action: Legal);
1389 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i16, Action: Legal);
1390 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i32, Action: Legal);
1391 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::v2i64, Action: Legal);
1392
1393 setOperationAction(Op: ISD::ABDU, VT: MVT::v16i8, Action: Legal);
1394 setOperationAction(Op: ISD::ABDU, VT: MVT::v8i16, Action: Legal);
1395 setOperationAction(Op: ISD::ABDU, VT: MVT::v4i32, Action: Legal);
1396 setOperationAction(Op: ISD::ABDS, VT: MVT::v4i32, Action: Legal);
1397 }
1398
1399 if (Subtarget.hasP10Vector()) {
1400 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f128, Action: Custom);
1401 }
1402
1403 setOperationAction(Op: ISD::PARTIAL_REDUCE_UMLA, VT: MVT::v16i32, Action: Custom);
1404 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1405 Action: Legal);
1406 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SMLA, AccVT: MVT::v4i32, InputVT: MVT::v8i16,
1407 Action: Legal);
1408 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_UMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1409 Action: Legal);
1410 setPartialReduceMLAAction(Opc: ISD::PARTIAL_REDUCE_SUMLA, AccVT: MVT::v4i32, InputVT: MVT::v16i8,
1411 Action: Legal);
1412 }
1413
1414 if (Subtarget.pairedVectorMemops()) {
1415 addRegisterClass(VT: MVT::v256i1, RC: &PPC::VSRpRCRegClass);
1416 setOperationAction(Op: ISD::LOAD, VT: MVT::v256i1, Action: Custom);
1417 setOperationAction(Op: ISD::STORE, VT: MVT::v256i1, Action: Custom);
1418 }
1419 if (Subtarget.hasMMA()) {
1420 if (Subtarget.isISAFuture()) {
1421 addRegisterClass(VT: MVT::v512i1, RC: &PPC::WACCRCRegClass);
1422 addRegisterClass(VT: MVT::v1024i1, RC: &PPC::DMRRCRegClass);
1423 addRegisterClass(VT: MVT::v2048i1, RC: &PPC::DMRpRCRegClass);
1424 setOperationAction(Op: ISD::LOAD, VT: MVT::v1024i1, Action: Custom);
1425 setOperationAction(Op: ISD::STORE, VT: MVT::v1024i1, Action: Custom);
1426 setOperationAction(Op: ISD::LOAD, VT: MVT::v2048i1, Action: Custom);
1427 setOperationAction(Op: ISD::STORE, VT: MVT::v2048i1, Action: Custom);
1428 } else {
1429 addRegisterClass(VT: MVT::v512i1, RC: &PPC::UACCRCRegClass);
1430 }
1431 setOperationAction(Op: ISD::LOAD, VT: MVT::v512i1, Action: Custom);
1432 setOperationAction(Op: ISD::STORE, VT: MVT::v512i1, Action: Custom);
1433 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v512i1, Action: Custom);
1434 }
1435
1436 if (Subtarget.has64BitSupport())
1437 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Legal);
1438
1439 if (Subtarget.isISA3_1())
1440 setOperationAction(Op: ISD::SRA, VT: MVT::v1i128, Action: Legal);
1441
1442 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64, Action: isPPC64 ? Legal : Custom);
1443
1444 if (!isPPC64) {
1445 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i64, Action: Expand);
1446 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i64, Action: Expand);
1447 }
1448
1449 if (shouldInlineQuadwordAtomics()) {
1450 setOperationAction(Op: ISD::ATOMIC_LOAD, VT: MVT::i128, Action: Custom);
1451 setOperationAction(Op: ISD::ATOMIC_STORE, VT: MVT::i128, Action: Custom);
1452 setOperationAction(Op: ISD::INTRINSIC_VOID, VT: MVT::i128, Action: Custom);
1453 }
1454
1455 setBooleanContents(ZeroOrOneBooleanContent);
1456
1457 if (Subtarget.hasAltivec()) {
1458 // Altivec instructions set fields to all zeros or all ones.
1459 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
1460 }
1461
1462 if (shouldInlineQuadwordAtomics())
1463 setMaxAtomicSizeInBitsSupported(128);
1464 else if (isPPC64)
1465 setMaxAtomicSizeInBitsSupported(64);
1466 else
1467 setMaxAtomicSizeInBitsSupported(32);
1468
1469 setStackPointerRegisterToSaveRestore(isPPC64 ? PPC::X1 : PPC::R1);
1470
1471 // We have target-specific dag combine patterns for the following nodes:
1472 setTargetDAGCombine({ISD::AND, ISD::ADD, ISD::XOR, ISD::SHL, ISD::SRA,
1473 ISD::SRL, ISD::MUL, ISD::FMA, ISD::SINT_TO_FP,
1474 ISD::BUILD_VECTOR});
1475 if (Subtarget.hasFPCVT())
1476 setTargetDAGCombine(ISD::UINT_TO_FP);
1477 setTargetDAGCombine({ISD::LOAD, ISD::STORE, ISD::BR_CC});
1478 if (Subtarget.useCRBits())
1479 setTargetDAGCombine(ISD::BRCOND);
1480 setTargetDAGCombine({ISD::BSWAP, ISD::INTRINSIC_WO_CHAIN,
1481 ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID});
1482
1483 setTargetDAGCombine({ISD::SIGN_EXTEND, ISD::ZERO_EXTEND, ISD::ANY_EXTEND});
1484
1485 setTargetDAGCombine({ISD::TRUNCATE, ISD::VECTOR_SHUFFLE});
1486
1487 if (Subtarget.useCRBits()) {
1488 setTargetDAGCombine({ISD::SETCC, ISD::SELECT_CC});
1489 }
1490
1491 if (Subtarget.hasP8Vector())
1492 setTargetDAGCombine(ISD::BITCAST);
1493
1494 // With 32 condition bits, we don't need to sink (and duplicate) compares
1495 // aggressively in CodeGenPrep.
1496 if (Subtarget.useCRBits()) {
1497 setJumpIsExpensive();
1498 }
1499
1500 // TODO: The default entry number is set to 64. This stops most jump table
1501 // generation on PPC. But it is good for current PPC HWs because the indirect
1502 // branch instruction mtctr to the jump table may lead to bad branch predict.
1503 // Re-evaluate this value on future HWs that can do better with mtctr.
1504 setMinimumJumpTableEntries(PPCMinimumJumpTableEntries);
1505
1506 // The default minimum of largest number in a BitTest cluster is 3.
1507 setMinimumBitTestCmps(PPCMinimumBitTestCmps);
1508
1509 setMinFunctionAlignment(Align(4));
1510 setMinCmpXchgSizeInBits(Subtarget.hasPartwordAtomics() ? 8 : 32);
1511
1512 auto CPUDirective = Subtarget.getCPUDirective();
1513 switch (CPUDirective) {
1514 default: break;
1515 case PPC::DIR_970:
1516 case PPC::DIR_A2:
1517 case PPC::DIR_E500:
1518 case PPC::DIR_E500mc:
1519 case PPC::DIR_E5500:
1520 case PPC::DIR_PWR4:
1521 case PPC::DIR_PWR5:
1522 case PPC::DIR_PWR5X:
1523 case PPC::DIR_PWR6:
1524 case PPC::DIR_PWR6X:
1525 case PPC::DIR_PWR7:
1526 case PPC::DIR_PWR8:
1527 case PPC::DIR_PWR9:
1528 case PPC::DIR_PWR10:
1529 case PPC::DIR_PWR11:
1530 case PPC::DIR_PWR_FUTURE:
1531 setPrefLoopAlignment(Align(16));
1532 setPrefFunctionAlignment(Align(16));
1533 break;
1534 }
1535
1536 if (Subtarget.enableMachineScheduler())
1537 setSchedulingPreference(Sched::Source);
1538 else
1539 setSchedulingPreference(Sched::Hybrid);
1540
1541 computeRegisterProperties(TRI: STI.getRegisterInfo());
1542
1543 // The Freescale cores do better with aggressive inlining of memcpy and
1544 // friends. GCC uses same threshold of 128 bytes (= 32 word stores).
1545 if (CPUDirective == PPC::DIR_E500mc || CPUDirective == PPC::DIR_E5500) {
1546 MaxStoresPerMemset = 32;
1547 MaxStoresPerMemsetOptSize = 16;
1548 MaxStoresPerMemcpy = 32;
1549 MaxStoresPerMemcpyOptSize = 8;
1550 MaxStoresPerMemmove = 32;
1551 MaxStoresPerMemmoveOptSize = 8;
1552 } else if (CPUDirective == PPC::DIR_A2) {
1553 // The A2 also benefits from (very) aggressive inlining of memcpy and
1554 // friends. The overhead of a the function call, even when warm, can be
1555 // over one hundred cycles.
1556 MaxStoresPerMemset = 128;
1557 MaxStoresPerMemcpy = 128;
1558 MaxStoresPerMemmove = 128;
1559 MaxLoadsPerMemcmp = 128;
1560 } else {
1561 MaxLoadsPerMemcmp = 8;
1562 MaxLoadsPerMemcmpOptSize = 4;
1563 }
1564
1565 // Enable generation of STXVP instructions by default for mcpu=future.
1566 if (CPUDirective == PPC::DIR_PWR_FUTURE &&
1567 EnableAutoPairedVecSt.getNumOccurrences() == 0)
1568 EnableAutoPairedVecSt = true;
1569
1570 IsStrictFPEnabled = true;
1571
1572 // Let the subtarget (CPU) decide if a predictable select is more expensive
1573 // than the corresponding branch. This information is used in CGP to decide
1574 // when to convert selects into branches.
1575 PredictableSelectIsExpensive = Subtarget.isPredictableSelectIsExpensive();
1576
1577 GatherAllAliasesMaxDepth = PPCGatherAllAliasesMaxDepth;
1578}
1579
1580// *********************************** NOTE ************************************
1581// For selecting load and store instructions, the addressing modes are defined
1582// as ComplexPatterns in PPCInstrInfo.td, which are then utilized in the TD
1583// patterns to match the load the store instructions.
1584//
1585// The TD definitions for the addressing modes correspond to their respective
1586// Select<AddrMode>Form() function in PPCISelDAGToDAG.cpp. These functions rely
1587// on SelectOptimalAddrMode(), which calls computeMOFlags() to compute the
1588// address mode flags of a particular node. Afterwards, the computed address
1589// flags are passed into getAddrModeForFlags() in order to retrieve the optimal
1590// addressing mode. SelectOptimalAddrMode() then sets the Base and Displacement
1591// accordingly, based on the preferred addressing mode.
1592//
1593// Within PPCISelLowering.h, there are two enums: MemOpFlags and AddrMode.
1594// MemOpFlags contains all the possible flags that can be used to compute the
1595// optimal addressing mode for load and store instructions.
1596// AddrMode contains all the possible load and store addressing modes available
1597// on Power (such as DForm, DSForm, DQForm, XForm, etc.)
1598//
1599// When adding new load and store instructions, it is possible that new address
1600// flags may need to be added into MemOpFlags, and a new addressing mode will
1601// need to be added to AddrMode. An entry of the new addressing mode (consisting
1602// of the minimal and main distinguishing address flags for the new load/store
1603// instructions) will need to be added into initializeAddrModeMap() below.
1604// Finally, when adding new addressing modes, the getAddrModeForFlags() will
1605// need to be updated to account for selecting the optimal addressing mode.
1606// *****************************************************************************
1607/// Initialize the map that relates the different addressing modes of the load
1608/// and store instructions to a set of flags. This ensures the load/store
1609/// instruction is correctly matched during instruction selection.
1610void PPCTargetLowering::initializeAddrModeMap() {
1611 AddrModesMap[PPC::AM_DForm] = {
1612 // LWZ, STW
1613 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_WordInt,
1614 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_WordInt,
1615 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1616 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1617 // LBZ, LHZ, STB, STH
1618 PPC::MOF_ZExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1619 PPC::MOF_ZExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1620 PPC::MOF_ZExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1621 PPC::MOF_ZExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1622 // LHA
1623 PPC::MOF_SExt | PPC::MOF_RPlusSImm16 | PPC::MOF_SubWordInt,
1624 PPC::MOF_SExt | PPC::MOF_RPlusLo | PPC::MOF_SubWordInt,
1625 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_SubWordInt,
1626 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubWordInt,
1627 // LFS, LFD, STFS, STFD
1628 PPC::MOF_RPlusSImm16 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1629 PPC::MOF_RPlusLo | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1630 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1631 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetBeforeP9,
1632 };
1633 AddrModesMap[PPC::AM_DSForm] = {
1634 // LWA
1635 PPC::MOF_SExt | PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_WordInt,
1636 PPC::MOF_SExt | PPC::MOF_NotAddNorCst | PPC::MOF_WordInt,
1637 PPC::MOF_SExt | PPC::MOF_AddrIsSImm32 | PPC::MOF_WordInt,
1638 // LD, STD
1639 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_DoubleWordInt,
1640 PPC::MOF_NotAddNorCst | PPC::MOF_DoubleWordInt,
1641 PPC::MOF_AddrIsSImm32 | PPC::MOF_DoubleWordInt,
1642 // DFLOADf32, DFLOADf64, DSTOREf32, DSTOREf64
1643 PPC::MOF_RPlusSImm16Mult4 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1644 PPC::MOF_NotAddNorCst | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1645 PPC::MOF_AddrIsSImm32 | PPC::MOF_ScalarFloat | PPC::MOF_SubtargetP9,
1646 };
1647 AddrModesMap[PPC::AM_DQForm] = {
1648 // LXV, STXV
1649 PPC::MOF_RPlusSImm16Mult16 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1650 PPC::MOF_NotAddNorCst | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1651 PPC::MOF_AddrIsSImm32 | PPC::MOF_Vector | PPC::MOF_SubtargetP9,
1652 };
1653 AddrModesMap[PPC::AM_PrefixDForm] = {PPC::MOF_RPlusSImm34 |
1654 PPC::MOF_SubtargetP10};
1655 // TODO: Add mapping for quadword load/store.
1656}
1657
1658/// getMaxByValAlign - Helper for getByValTypeAlignment to determine
1659/// the desired ByVal argument alignment.
1660static void getMaxByValAlign(Type *Ty, Align &MaxAlign, Align MaxMaxAlign) {
1661 if (MaxAlign == MaxMaxAlign)
1662 return;
1663 if (VectorType *VTy = dyn_cast<VectorType>(Val: Ty)) {
1664 if (MaxMaxAlign >= 32 &&
1665 VTy->getPrimitiveSizeInBits().getFixedValue() >= 256)
1666 MaxAlign = Align(32);
1667 else if (VTy->getPrimitiveSizeInBits().getFixedValue() >= 128 &&
1668 MaxAlign < 16)
1669 MaxAlign = Align(16);
1670 } else if (ArrayType *ATy = dyn_cast<ArrayType>(Val: Ty)) {
1671 Align EltAlign;
1672 getMaxByValAlign(Ty: ATy->getElementType(), MaxAlign&: EltAlign, MaxMaxAlign);
1673 if (EltAlign > MaxAlign)
1674 MaxAlign = EltAlign;
1675 } else if (StructType *STy = dyn_cast<StructType>(Val: Ty)) {
1676 for (auto *EltTy : STy->elements()) {
1677 Align EltAlign;
1678 getMaxByValAlign(Ty: EltTy, MaxAlign&: EltAlign, MaxMaxAlign);
1679 if (EltAlign > MaxAlign)
1680 MaxAlign = EltAlign;
1681 if (MaxAlign == MaxMaxAlign)
1682 break;
1683 }
1684 }
1685}
1686
1687/// getByValTypeAlignment - Return the desired alignment for ByVal aggregate
1688/// function arguments in the caller parameter area.
1689Align PPCTargetLowering::getByValTypeAlignment(Type *Ty,
1690 const DataLayout &DL) const {
1691 // 16byte and wider vectors are passed on 16byte boundary.
1692 // The rest is 8 on PPC64 and 4 on PPC32 boundary.
1693 Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
1694 if (Subtarget.hasAltivec())
1695 getMaxByValAlign(Ty, MaxAlign&: Alignment, MaxMaxAlign: Align(16));
1696 return Alignment;
1697}
1698
1699bool PPCTargetLowering::useSoftFloat() const {
1700 return Subtarget.useSoftFloat();
1701}
1702
1703bool PPCTargetLowering::hasSPE() const {
1704 return Subtarget.hasSPE();
1705}
1706
1707bool PPCTargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
1708 return VT.isScalarInteger();
1709}
1710
1711bool PPCTargetLowering::shallExtractConstSplatVectorElementToStore(
1712 Type *VectorTy, unsigned ElemSizeInBits, unsigned &Index) const {
1713 if (!Subtarget.isPPC64() || !Subtarget.hasVSX())
1714 return false;
1715
1716 if (auto *VTy = dyn_cast<VectorType>(Val: VectorTy)) {
1717 if (VTy->getScalarType()->isIntegerTy()) {
1718 // ElemSizeInBits 8/16 can fit in immediate field, not needed here.
1719 if (ElemSizeInBits == 32) {
1720 Index = Subtarget.isLittleEndian() ? 2 : 1;
1721 return true;
1722 }
1723 if (ElemSizeInBits == 64) {
1724 Index = Subtarget.isLittleEndian() ? 1 : 0;
1725 return true;
1726 }
1727 }
1728 }
1729 return false;
1730}
1731
1732EVT PPCTargetLowering::getSetCCResultType(const DataLayout &DL, LLVMContext &C,
1733 EVT VT) const {
1734 if (!VT.isVector())
1735 return Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
1736
1737 return VT.changeVectorElementTypeToInteger();
1738}
1739
1740bool PPCTargetLowering::enableAggressiveFMAFusion(EVT VT) const {
1741 assert(VT.isFloatingPoint() && "Non-floating-point FMA?");
1742 return true;
1743}
1744
1745//===----------------------------------------------------------------------===//
1746// Node matching predicates, for use by the tblgen matching code.
1747//===----------------------------------------------------------------------===//
1748
1749/// isFloatingPointZero - Return true if this is 0.0 or -0.0.
1750static bool isFloatingPointZero(SDValue Op) {
1751 if (ConstantFPSDNode *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op))
1752 return CFP->getValueAPF().isZero();
1753 else if (ISD::isEXTLoad(N: Op.getNode()) || ISD::isNON_EXTLoad(N: Op.getNode())) {
1754 // Maybe this has already been legalized into the constant pool?
1755 if (ConstantPoolSDNode *CP = dyn_cast<ConstantPoolSDNode>(Val: Op.getOperand(i: 1)))
1756 if (const ConstantFP *CFP = dyn_cast<ConstantFP>(Val: CP->getConstVal()))
1757 return CFP->getValueAPF().isZero();
1758 }
1759 return false;
1760}
1761
1762/// isConstantOrUndef - Op is either an undef node or a ConstantSDNode. Return
1763/// true if Op is undef or if it matches the specified value.
1764static bool isConstantOrUndef(int Op, int Val) {
1765 return Op < 0 || Op == Val;
1766}
1767
1768/// isVPKUHUMShuffleMask - Return true if this is the shuffle mask for a
1769/// VPKUHUM instruction.
1770/// The ShuffleKind distinguishes between big-endian operations with
1771/// two different inputs (0), either-endian operations with two identical
1772/// inputs (1), and little-endian operations with two different inputs (2).
1773/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1774bool PPC::isVPKUHUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1775 SelectionDAG &DAG) {
1776 bool IsLE = DAG.getDataLayout().isLittleEndian();
1777 if (ShuffleKind == 0) {
1778 if (IsLE)
1779 return false;
1780 for (unsigned i = 0; i != 16; ++i)
1781 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+1))
1782 return false;
1783 } else if (ShuffleKind == 2) {
1784 if (!IsLE)
1785 return false;
1786 for (unsigned i = 0; i != 16; ++i)
1787 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2))
1788 return false;
1789 } else if (ShuffleKind == 1) {
1790 unsigned j = IsLE ? 0 : 1;
1791 for (unsigned i = 0; i != 8; ++i)
1792 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i), Val: i*2+j) ||
1793 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j))
1794 return false;
1795 }
1796 return true;
1797}
1798
1799/// isVPKUWUMShuffleMask - Return true if this is the shuffle mask for a
1800/// VPKUWUM instruction.
1801/// The ShuffleKind distinguishes between big-endian operations with
1802/// two different inputs (0), either-endian operations with two identical
1803/// inputs (1), and little-endian operations with two different inputs (2).
1804/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1805bool PPC::isVPKUWUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1806 SelectionDAG &DAG) {
1807 bool IsLE = DAG.getDataLayout().isLittleEndian();
1808 if (ShuffleKind == 0) {
1809 if (IsLE)
1810 return false;
1811 for (unsigned i = 0; i != 16; i += 2)
1812 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+2) ||
1813 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+3))
1814 return false;
1815 } else if (ShuffleKind == 2) {
1816 if (!IsLE)
1817 return false;
1818 for (unsigned i = 0; i != 16; i += 2)
1819 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1820 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1))
1821 return false;
1822 } else if (ShuffleKind == 1) {
1823 unsigned j = IsLE ? 0 : 2;
1824 for (unsigned i = 0; i != 8; i += 2)
1825 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1826 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1827 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1828 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1))
1829 return false;
1830 }
1831 return true;
1832}
1833
1834/// isVPKUDUMShuffleMask - Return true if this is the shuffle mask for a
1835/// VPKUDUM instruction, AND the VPKUDUM instruction exists for the
1836/// current subtarget.
1837///
1838/// The ShuffleKind distinguishes between big-endian operations with
1839/// two different inputs (0), either-endian operations with two identical
1840/// inputs (1), and little-endian operations with two different inputs (2).
1841/// For the latter, the input operands are swapped (see PPCInstrAltivec.td).
1842bool PPC::isVPKUDUMShuffleMask(ShuffleVectorSDNode *N, unsigned ShuffleKind,
1843 SelectionDAG &DAG) {
1844 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
1845 if (!Subtarget.hasP8Vector())
1846 return false;
1847
1848 bool IsLE = DAG.getDataLayout().isLittleEndian();
1849 if (ShuffleKind == 0) {
1850 if (IsLE)
1851 return false;
1852 for (unsigned i = 0; i != 16; i += 4)
1853 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+4) ||
1854 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+5) ||
1855 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+6) ||
1856 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+7))
1857 return false;
1858 } else if (ShuffleKind == 2) {
1859 if (!IsLE)
1860 return false;
1861 for (unsigned i = 0; i != 16; i += 4)
1862 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2) ||
1863 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+1) ||
1864 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+2) ||
1865 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+3))
1866 return false;
1867 } else if (ShuffleKind == 1) {
1868 unsigned j = IsLE ? 0 : 4;
1869 for (unsigned i = 0; i != 8; i += 4)
1870 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i ), Val: i*2+j) ||
1871 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+1), Val: i*2+j+1) ||
1872 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+2), Val: i*2+j+2) ||
1873 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+3), Val: i*2+j+3) ||
1874 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+8), Val: i*2+j) ||
1875 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+9), Val: i*2+j+1) ||
1876 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+10), Val: i*2+j+2) ||
1877 !isConstantOrUndef(Op: N->getMaskElt(Idx: i+11), Val: i*2+j+3))
1878 return false;
1879 }
1880 return true;
1881}
1882
1883/// isVMerge - Common function, used to match vmrg* shuffles.
1884///
1885static bool isVMerge(ShuffleVectorSDNode *N, unsigned UnitSize,
1886 unsigned LHSStart, unsigned RHSStart) {
1887 if (N->getValueType(ResNo: 0) != MVT::v16i8)
1888 return false;
1889 assert((UnitSize == 1 || UnitSize == 2 || UnitSize == 4) &&
1890 "Unsupported merge size!");
1891
1892 for (unsigned i = 0; i != 8/UnitSize; ++i) // Step over units
1893 for (unsigned j = 0; j != UnitSize; ++j) { // Step over bytes within unit
1894 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+j),
1895 Val: LHSStart+j+i*UnitSize) ||
1896 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*UnitSize*2+UnitSize+j),
1897 Val: RHSStart+j+i*UnitSize))
1898 return false;
1899 }
1900 return true;
1901}
1902
1903/// isVMRGLShuffleMask - Return true if this is a shuffle mask suitable for
1904/// a VMRGL* instruction with the specified unit size (1,2 or 4 bytes).
1905/// The ShuffleKind distinguishes between big-endian merges with two
1906/// different inputs (0), either-endian merges with two identical inputs (1),
1907/// and little-endian merges with two different inputs (2). For the latter,
1908/// the input operands are swapped (see PPCInstrAltivec.td).
1909bool PPC::isVMRGLShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1910 unsigned ShuffleKind, SelectionDAG &DAG) {
1911 if (DAG.getDataLayout().isLittleEndian()) {
1912 if (ShuffleKind == 1) // unary
1913 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1914 else if (ShuffleKind == 2) // swapped
1915 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1916 else
1917 return false;
1918 } else {
1919 if (ShuffleKind == 1) // unary
1920 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1921 else if (ShuffleKind == 0) // normal
1922 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1923 else
1924 return false;
1925 }
1926}
1927
1928/// isVMRGHShuffleMask - Return true if this is a shuffle mask suitable for
1929/// a VMRGH* instruction with the specified unit size (1,2 or 4 bytes).
1930/// The ShuffleKind distinguishes between big-endian merges with two
1931/// different inputs (0), either-endian merges with two identical inputs (1),
1932/// and little-endian merges with two different inputs (2). For the latter,
1933/// the input operands are swapped (see PPCInstrAltivec.td).
1934bool PPC::isVMRGHShuffleMask(ShuffleVectorSDNode *N, unsigned UnitSize,
1935 unsigned ShuffleKind, SelectionDAG &DAG) {
1936 if (DAG.getDataLayout().isLittleEndian()) {
1937 if (ShuffleKind == 1) // unary
1938 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 8);
1939 else if (ShuffleKind == 2) // swapped
1940 return isVMerge(N, UnitSize, LHSStart: 8, RHSStart: 24);
1941 else
1942 return false;
1943 } else {
1944 if (ShuffleKind == 1) // unary
1945 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 0);
1946 else if (ShuffleKind == 0) // normal
1947 return isVMerge(N, UnitSize, LHSStart: 0, RHSStart: 16);
1948 else
1949 return false;
1950 }
1951}
1952
1953/**
1954 * Common function used to match vmrgew and vmrgow shuffles
1955 *
1956 * The indexOffset determines whether to look for even or odd words in
1957 * the shuffle mask. This is based on the of the endianness of the target
1958 * machine.
1959 * - Little Endian:
1960 * - Use offset of 0 to check for odd elements
1961 * - Use offset of 4 to check for even elements
1962 * - Big Endian:
1963 * - Use offset of 0 to check for even elements
1964 * - Use offset of 4 to check for odd elements
1965 * A detailed description of the vector element ordering for little endian and
1966 * big endian can be found at
1967 * http://www.ibm.com/developerworks/library/l-ibm-xl-c-cpp-compiler/index.html
1968 * Targeting your applications - what little endian and big endian IBM XL C/C++
1969 * compiler differences mean to you
1970 *
1971 * The mask to the shuffle vector instruction specifies the indices of the
1972 * elements from the two input vectors to place in the result. The elements are
1973 * numbered in array-access order, starting with the first vector. These vectors
1974 * are always of type v16i8, thus each vector will contain 16 elements of size
1975 * 8. More info on the shuffle vector can be found in the
1976 * http://llvm.org/docs/LangRef.html#shufflevector-instruction
1977 * Language Reference.
1978 *
1979 * The RHSStartValue indicates whether the same input vectors are used (unary)
1980 * or two different input vectors are used, based on the following:
1981 * - If the instruction uses the same vector for both inputs, the range of the
1982 * indices will be 0 to 15. In this case, the RHSStart value passed should
1983 * be 0.
1984 * - If the instruction has two different vectors then the range of the
1985 * indices will be 0 to 31. In this case, the RHSStart value passed should
1986 * be 16 (indices 0-15 specify elements in the first vector while indices 16
1987 * to 31 specify elements in the second vector).
1988 *
1989 * \param[in] N The shuffle vector SD Node to analyze
1990 * \param[in] IndexOffset Specifies whether to look for even or odd elements
1991 * \param[in] RHSStartValue Specifies the starting index for the righthand input
1992 * vector to the shuffle_vector instruction
1993 * \return true iff this shuffle vector represents an even or odd word merge
1994 */
1995static bool isVMerge(ShuffleVectorSDNode *N, unsigned IndexOffset,
1996 unsigned RHSStartValue) {
1997 if (N->getValueType(ResNo: 0) != MVT::v16i8)
1998 return false;
1999
2000 for (unsigned i = 0; i < 2; ++i)
2001 for (unsigned j = 0; j < 4; ++j)
2002 if (!isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j),
2003 Val: i*RHSStartValue+j+IndexOffset) ||
2004 !isConstantOrUndef(Op: N->getMaskElt(Idx: i*4+j+8),
2005 Val: i*RHSStartValue+j+IndexOffset+8))
2006 return false;
2007 return true;
2008}
2009
2010/**
2011 * Determine if the specified shuffle mask is suitable for the vmrgew or
2012 * vmrgow instructions.
2013 *
2014 * \param[in] N The shuffle vector SD Node to analyze
2015 * \param[in] CheckEven Check for an even merge (true) or an odd merge (false)
2016 * \param[in] ShuffleKind Identify the type of merge:
2017 * - 0 = big-endian merge with two different inputs;
2018 * - 1 = either-endian merge with two identical inputs;
2019 * - 2 = little-endian merge with two different inputs (inputs are swapped for
2020 * little-endian merges).
2021 * \param[in] DAG The current SelectionDAG
2022 * \return true iff this shuffle mask
2023 */
2024bool PPC::isVMRGEOShuffleMask(ShuffleVectorSDNode *N, bool CheckEven,
2025 unsigned ShuffleKind, SelectionDAG &DAG) {
2026 if (DAG.getDataLayout().isLittleEndian()) {
2027 unsigned indexOffset = CheckEven ? 4 : 0;
2028 if (ShuffleKind == 1) // Unary
2029 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2030 else if (ShuffleKind == 2) // swapped
2031 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2032 else
2033 return false;
2034 }
2035 else {
2036 unsigned indexOffset = CheckEven ? 0 : 4;
2037 if (ShuffleKind == 1) // Unary
2038 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 0);
2039 else if (ShuffleKind == 0) // Normal
2040 return isVMerge(N, IndexOffset: indexOffset, RHSStartValue: 16);
2041 else
2042 return false;
2043 }
2044 return false;
2045}
2046
2047/// isVSLDOIShuffleMask - If this is a vsldoi shuffle mask, return the shift
2048/// amount, otherwise return -1.
2049/// The ShuffleKind distinguishes between big-endian operations with two
2050/// different inputs (0), either-endian operations with two identical inputs
2051/// (1), and little-endian operations with two different inputs (2). For the
2052/// latter, the input operands are swapped (see PPCInstrAltivec.td).
2053int PPC::isVSLDOIShuffleMask(SDNode *N, unsigned ShuffleKind,
2054 SelectionDAG &DAG) {
2055 if (N->getValueType(ResNo: 0) != MVT::v16i8)
2056 return -1;
2057
2058 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2059
2060 // Find the first non-undef value in the shuffle mask.
2061 unsigned i;
2062 for (i = 0; i != 16 && SVOp->getMaskElt(Idx: i) < 0; ++i)
2063 /*search*/;
2064
2065 if (i == 16) return -1; // all undef.
2066
2067 // Otherwise, check to see if the rest of the elements are consecutively
2068 // numbered from this value.
2069 unsigned ShiftAmt = SVOp->getMaskElt(Idx: i);
2070 if (ShiftAmt < i) return -1;
2071
2072 ShiftAmt -= i;
2073 bool isLE = DAG.getDataLayout().isLittleEndian();
2074
2075 if ((ShuffleKind == 0 && !isLE) || (ShuffleKind == 2 && isLE)) {
2076 // Check the rest of the elements to see if they are consecutive.
2077 for (++i; i != 16; ++i)
2078 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: ShiftAmt+i))
2079 return -1;
2080 } else if (ShuffleKind == 1) {
2081 // Check the rest of the elements to see if they are consecutive.
2082 for (++i; i != 16; ++i)
2083 if (!isConstantOrUndef(Op: SVOp->getMaskElt(Idx: i), Val: (ShiftAmt+i) & 15))
2084 return -1;
2085 } else
2086 return -1;
2087
2088 if (isLE)
2089 ShiftAmt = 16 - ShiftAmt;
2090
2091 return ShiftAmt;
2092}
2093
2094/// isSplatShuffleMask - Return true if the specified VECTOR_SHUFFLE operand
2095/// specifies a splat of a single element that is suitable for input to
2096/// one of the splat operations (VSPLTB/VSPLTH/VSPLTW/XXSPLTW/LXVDSX/etc.).
2097bool PPC::isSplatShuffleMask(ShuffleVectorSDNode *N, unsigned EltSize) {
2098 EVT VT = N->getValueType(ResNo: 0);
2099 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2100 return EltSize == 8 && N->getMaskElt(Idx: 0) == N->getMaskElt(Idx: 1);
2101
2102 assert(VT == MVT::v16i8 && isPowerOf2_32(EltSize) &&
2103 EltSize <= 8 && "Can only handle 1,2,4,8 byte element sizes");
2104
2105 // The consecutive indices need to specify an element, not part of two
2106 // different elements. So abandon ship early if this isn't the case.
2107 if (N->getMaskElt(Idx: 0) % EltSize != 0)
2108 return false;
2109
2110 // This is a splat operation if each element of the permute is the same, and
2111 // if the value doesn't reference the second vector.
2112 unsigned ElementBase = N->getMaskElt(Idx: 0);
2113
2114 // FIXME: Handle UNDEF elements too!
2115 if (ElementBase >= 16)
2116 return false;
2117
2118 // Check that the indices are consecutive, in the case of a multi-byte element
2119 // splatted with a v16i8 mask.
2120 for (unsigned i = 1; i != EltSize; ++i)
2121 if (N->getMaskElt(Idx: i) < 0 || N->getMaskElt(Idx: i) != (int)(i+ElementBase))
2122 return false;
2123
2124 for (unsigned i = EltSize, e = 16; i != e; i += EltSize) {
2125 // An UNDEF element is a sequence of UNDEF bytes.
2126 if (N->getMaskElt(Idx: i) < 0) {
2127 for (unsigned j = 1; j != EltSize; ++j)
2128 if (N->getMaskElt(Idx: i + j) >= 0)
2129 return false;
2130 } else
2131 for (unsigned j = 0; j != EltSize; ++j)
2132 if (N->getMaskElt(Idx: i + j) != N->getMaskElt(Idx: j))
2133 return false;
2134 }
2135 return true;
2136}
2137
2138/// Check that the mask is shuffling N byte elements. Within each N byte
2139/// element of the mask, the indices could be either in increasing or
2140/// decreasing order as long as they are consecutive.
2141/// \param[in] N the shuffle vector SD Node to analyze
2142/// \param[in] Width the element width in bytes, could be 2/4/8/16 (HalfWord/
2143/// Word/DoubleWord/QuadWord).
2144/// \param[in] StepLen the delta indices number among the N byte element, if
2145/// the mask is in increasing/decreasing order then it is 1/-1.
2146/// \return true iff the mask is shuffling N byte elements.
2147static bool isNByteElemShuffleMask(ShuffleVectorSDNode *N, unsigned Width,
2148 int StepLen) {
2149 assert((Width == 2 || Width == 4 || Width == 8 || Width == 16) &&
2150 "Unexpected element width.");
2151 assert((StepLen == 1 || StepLen == -1) && "Unexpected element width.");
2152
2153 unsigned NumOfElem = 16 / Width;
2154 unsigned MaskVal[16]; // Width is never greater than 16
2155 for (unsigned i = 0; i < NumOfElem; ++i) {
2156 MaskVal[0] = N->getMaskElt(Idx: i * Width);
2157 if ((StepLen == 1) && (MaskVal[0] % Width)) {
2158 return false;
2159 } else if ((StepLen == -1) && ((MaskVal[0] + 1) % Width)) {
2160 return false;
2161 }
2162
2163 for (unsigned int j = 1; j < Width; ++j) {
2164 MaskVal[j] = N->getMaskElt(Idx: i * Width + j);
2165 if (MaskVal[j] != MaskVal[j-1] + StepLen) {
2166 return false;
2167 }
2168 }
2169 }
2170
2171 return true;
2172}
2173
2174bool PPC::isXXINSERTWMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2175 unsigned &InsertAtByte, bool &Swap, bool IsLE) {
2176 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2177 return false;
2178
2179 // Now we look at mask elements 0,4,8,12
2180 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2181 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2182 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2183 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2184 unsigned LittleEndianShifts[] = { 2, 1, 0, 3 };
2185 unsigned BigEndianShifts[] = { 3, 0, 1, 2 };
2186
2187 // Below, let H and L be arbitrary elements of the shuffle mask
2188 // where H is in the range [4,7] and L is in the range [0,3].
2189 // H, 1, 2, 3 or L, 5, 6, 7
2190 if ((M0 > 3 && M1 == 1 && M2 == 2 && M3 == 3) ||
2191 (M0 < 4 && M1 == 5 && M2 == 6 && M3 == 7)) {
2192 ShiftElts = IsLE ? LittleEndianShifts[M0 & 0x3] : BigEndianShifts[M0 & 0x3];
2193 InsertAtByte = IsLE ? 12 : 0;
2194 Swap = M0 < 4;
2195 return true;
2196 }
2197 // 0, H, 2, 3 or 4, L, 6, 7
2198 if ((M1 > 3 && M0 == 0 && M2 == 2 && M3 == 3) ||
2199 (M1 < 4 && M0 == 4 && M2 == 6 && M3 == 7)) {
2200 ShiftElts = IsLE ? LittleEndianShifts[M1 & 0x3] : BigEndianShifts[M1 & 0x3];
2201 InsertAtByte = IsLE ? 8 : 4;
2202 Swap = M1 < 4;
2203 return true;
2204 }
2205 // 0, 1, H, 3 or 4, 5, L, 7
2206 if ((M2 > 3 && M0 == 0 && M1 == 1 && M3 == 3) ||
2207 (M2 < 4 && M0 == 4 && M1 == 5 && M3 == 7)) {
2208 ShiftElts = IsLE ? LittleEndianShifts[M2 & 0x3] : BigEndianShifts[M2 & 0x3];
2209 InsertAtByte = IsLE ? 4 : 8;
2210 Swap = M2 < 4;
2211 return true;
2212 }
2213 // 0, 1, 2, H or 4, 5, 6, L
2214 if ((M3 > 3 && M0 == 0 && M1 == 1 && M2 == 2) ||
2215 (M3 < 4 && M0 == 4 && M1 == 5 && M2 == 6)) {
2216 ShiftElts = IsLE ? LittleEndianShifts[M3 & 0x3] : BigEndianShifts[M3 & 0x3];
2217 InsertAtByte = IsLE ? 0 : 12;
2218 Swap = M3 < 4;
2219 return true;
2220 }
2221
2222 // If both vector operands for the shuffle are the same vector, the mask will
2223 // contain only elements from the first one and the second one will be undef.
2224 if (N->getOperand(Num: 1).isUndef()) {
2225 ShiftElts = 0;
2226 Swap = true;
2227 unsigned XXINSERTWSrcElem = IsLE ? 2 : 1;
2228 if (M0 == XXINSERTWSrcElem && M1 == 1 && M2 == 2 && M3 == 3) {
2229 InsertAtByte = IsLE ? 12 : 0;
2230 return true;
2231 }
2232 if (M0 == 0 && M1 == XXINSERTWSrcElem && M2 == 2 && M3 == 3) {
2233 InsertAtByte = IsLE ? 8 : 4;
2234 return true;
2235 }
2236 if (M0 == 0 && M1 == 1 && M2 == XXINSERTWSrcElem && M3 == 3) {
2237 InsertAtByte = IsLE ? 4 : 8;
2238 return true;
2239 }
2240 if (M0 == 0 && M1 == 1 && M2 == 2 && M3 == XXINSERTWSrcElem) {
2241 InsertAtByte = IsLE ? 0 : 12;
2242 return true;
2243 }
2244 }
2245
2246 return false;
2247}
2248
2249bool PPC::isXXSLDWIShuffleMask(ShuffleVectorSDNode *N, unsigned &ShiftElts,
2250 bool &Swap, bool IsLE) {
2251 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2252 // Ensure each byte index of the word is consecutive.
2253 if (!isNByteElemShuffleMask(N, Width: 4, StepLen: 1))
2254 return false;
2255
2256 // Now we look at mask elements 0,4,8,12, which are the beginning of words.
2257 unsigned M0 = N->getMaskElt(Idx: 0) / 4;
2258 unsigned M1 = N->getMaskElt(Idx: 4) / 4;
2259 unsigned M2 = N->getMaskElt(Idx: 8) / 4;
2260 unsigned M3 = N->getMaskElt(Idx: 12) / 4;
2261
2262 // If both vector operands for the shuffle are the same vector, the mask will
2263 // contain only elements from the first one and the second one will be undef.
2264 if (N->getOperand(Num: 1).isUndef()) {
2265 assert(M0 < 4 && "Indexing into an undef vector?");
2266 if (M1 != (M0 + 1) % 4 || M2 != (M1 + 1) % 4 || M3 != (M2 + 1) % 4)
2267 return false;
2268
2269 ShiftElts = IsLE ? (4 - M0) % 4 : M0;
2270 Swap = false;
2271 return true;
2272 }
2273
2274 // Ensure each word index of the ShuffleVector Mask is consecutive.
2275 if (M1 != (M0 + 1) % 8 || M2 != (M1 + 1) % 8 || M3 != (M2 + 1) % 8)
2276 return false;
2277
2278 if (IsLE) {
2279 if (M0 == 0 || M0 == 7 || M0 == 6 || M0 == 5) {
2280 // Input vectors don't need to be swapped if the leading element
2281 // of the result is one of the 3 left elements of the second vector
2282 // (or if there is no shift to be done at all).
2283 Swap = false;
2284 ShiftElts = (8 - M0) % 8;
2285 } else if (M0 == 4 || M0 == 3 || M0 == 2 || M0 == 1) {
2286 // Input vectors need to be swapped if the leading element
2287 // of the result is one of the 3 left elements of the first vector
2288 // (or if we're shifting by 4 - thereby simply swapping the vectors).
2289 Swap = true;
2290 ShiftElts = (4 - M0) % 4;
2291 }
2292
2293 return true;
2294 } else { // BE
2295 if (M0 == 0 || M0 == 1 || M0 == 2 || M0 == 3) {
2296 // Input vectors don't need to be swapped if the leading element
2297 // of the result is one of the 4 elements of the first vector.
2298 Swap = false;
2299 ShiftElts = M0;
2300 } else if (M0 == 4 || M0 == 5 || M0 == 6 || M0 == 7) {
2301 // Input vectors need to be swapped if the leading element
2302 // of the result is one of the 4 elements of the right vector.
2303 Swap = true;
2304 ShiftElts = M0 - 4;
2305 }
2306
2307 return true;
2308 }
2309}
2310
2311bool static isXXBRShuffleMaskHelper(ShuffleVectorSDNode *N, int Width) {
2312 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2313
2314 if (!isNByteElemShuffleMask(N, Width, StepLen: -1))
2315 return false;
2316
2317 for (int i = 0; i < 16; i += Width)
2318 if (N->getMaskElt(Idx: i) != i + Width - 1)
2319 return false;
2320
2321 return true;
2322}
2323
2324bool PPC::isXXBRHShuffleMask(ShuffleVectorSDNode *N) {
2325 return isXXBRShuffleMaskHelper(N, Width: 2);
2326}
2327
2328bool PPC::isXXBRWShuffleMask(ShuffleVectorSDNode *N) {
2329 return isXXBRShuffleMaskHelper(N, Width: 4);
2330}
2331
2332bool PPC::isXXBRDShuffleMask(ShuffleVectorSDNode *N) {
2333 return isXXBRShuffleMaskHelper(N, Width: 8);
2334}
2335
2336bool PPC::isXXBRQShuffleMask(ShuffleVectorSDNode *N) {
2337 return isXXBRShuffleMaskHelper(N, Width: 16);
2338}
2339
2340/// Can node \p N be lowered to an XXPERMDI instruction? If so, set \p Swap
2341/// if the inputs to the instruction should be swapped and set \p DM to the
2342/// value for the immediate.
2343/// Specifically, set \p Swap to true only if \p N can be lowered to XXPERMDI
2344/// AND element 0 of the result comes from the first input (LE) or second input
2345/// (BE). Set \p DM to the calculated result (0-3) only if \p N can be lowered.
2346/// \return true iff the given mask of shuffle node \p N is a XXPERMDI shuffle
2347/// mask.
2348bool PPC::isXXPERMDIShuffleMask(ShuffleVectorSDNode *N, unsigned &DM,
2349 bool &Swap, bool IsLE) {
2350 assert(N->getValueType(0) == MVT::v16i8 && "Shuffle vector expects v16i8");
2351
2352 // Ensure each byte index of the double word is consecutive.
2353 if (!isNByteElemShuffleMask(N, Width: 8, StepLen: 1))
2354 return false;
2355
2356 unsigned M0 = N->getMaskElt(Idx: 0) / 8;
2357 unsigned M1 = N->getMaskElt(Idx: 8) / 8;
2358 assert(((M0 | M1) < 4) && "A mask element out of bounds?");
2359
2360 // If both vector operands for the shuffle are the same vector, the mask will
2361 // contain only elements from the first one and the second one will be undef.
2362 if (N->getOperand(Num: 1).isUndef()) {
2363 if ((M0 | M1) < 2) {
2364 DM = IsLE ? (((~M1) & 1) << 1) + ((~M0) & 1) : (M0 << 1) + (M1 & 1);
2365 Swap = false;
2366 return true;
2367 } else
2368 return false;
2369 }
2370
2371 if (IsLE) {
2372 if (M0 > 1 && M1 < 2) {
2373 Swap = false;
2374 } else if (M0 < 2 && M1 > 1) {
2375 M0 = (M0 + 2) % 4;
2376 M1 = (M1 + 2) % 4;
2377 Swap = true;
2378 } else
2379 return false;
2380
2381 // Note: if control flow comes here that means Swap is already set above
2382 DM = (((~M1) & 1) << 1) + ((~M0) & 1);
2383 return true;
2384 } else { // BE
2385 if (M0 < 2 && M1 > 1) {
2386 Swap = false;
2387 } else if (M0 > 1 && M1 < 2) {
2388 M0 = (M0 + 2) % 4;
2389 M1 = (M1 + 2) % 4;
2390 Swap = true;
2391 } else
2392 return false;
2393
2394 // Note: if control flow comes here that means Swap is already set above
2395 DM = (M0 << 1) + (M1 & 1);
2396 return true;
2397 }
2398}
2399
2400
2401/// getSplatIdxForPPCMnemonics - Return the splat index as a value that is
2402/// appropriate for PPC mnemonics (which have a big endian bias - namely
2403/// elements are counted from the left of the vector register).
2404unsigned PPC::getSplatIdxForPPCMnemonics(SDNode *N, unsigned EltSize,
2405 SelectionDAG &DAG) {
2406 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val: N);
2407 assert(isSplatShuffleMask(SVOp, EltSize));
2408 EVT VT = SVOp->getValueType(ResNo: 0);
2409
2410 if (VT == MVT::v2i64 || VT == MVT::v2f64)
2411 return DAG.getDataLayout().isLittleEndian() ? 1 - SVOp->getMaskElt(Idx: 0)
2412 : SVOp->getMaskElt(Idx: 0);
2413
2414 if (DAG.getDataLayout().isLittleEndian())
2415 return (16 / EltSize) - 1 - (SVOp->getMaskElt(Idx: 0) / EltSize);
2416 else
2417 return SVOp->getMaskElt(Idx: 0) / EltSize;
2418}
2419
2420/// get_VSPLTI_elt - If this is a build_vector of constants which can be formed
2421/// by using a vspltis[bhw] instruction of the specified element size, return
2422/// the constant being splatted. The ByteSize field indicates the number of
2423/// bytes of each element [124] -> [bhw].
2424SDValue PPC::get_VSPLTI_elt(SDNode *N, unsigned ByteSize, SelectionDAG &DAG) {
2425 SDValue OpVal;
2426
2427 // If ByteSize of the splat is bigger than the element size of the
2428 // build_vector, then we have a case where we are checking for a splat where
2429 // multiple elements of the buildvector are folded together into a single
2430 // logical element of the splat (e.g. "vsplish 1" to splat {0,1}*8).
2431 unsigned EltSize = 16/N->getNumOperands();
2432 if (EltSize < ByteSize) {
2433 unsigned Multiple = ByteSize/EltSize; // Number of BV entries per spltval.
2434 SDValue UniquedVals[4];
2435 assert(Multiple > 1 && Multiple <= 4 && "How can this happen?");
2436
2437 // See if all of the elements in the buildvector agree across.
2438 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2439 if (N->getOperand(Num: i).isUndef()) continue;
2440 // If the element isn't a constant, bail fully out.
2441 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: i))) return SDValue();
2442
2443 if (!UniquedVals[i&(Multiple-1)].getNode())
2444 UniquedVals[i&(Multiple-1)] = N->getOperand(Num: i);
2445 else if (UniquedVals[i&(Multiple-1)] != N->getOperand(Num: i))
2446 return SDValue(); // no match.
2447 }
2448
2449 // Okay, if we reached this point, UniquedVals[0..Multiple-1] contains
2450 // either constant or undef values that are identical for each chunk. See
2451 // if these chunks can form into a larger vspltis*.
2452
2453 // Check to see if all of the leading entries are either 0 or -1. If
2454 // neither, then this won't fit into the immediate field.
2455 bool LeadingZero = true;
2456 bool LeadingOnes = true;
2457 for (unsigned i = 0; i != Multiple-1; ++i) {
2458 if (!UniquedVals[i].getNode()) continue; // Must have been undefs.
2459
2460 LeadingZero &= isNullConstant(V: UniquedVals[i]);
2461 LeadingOnes &= isAllOnesConstant(V: UniquedVals[i]);
2462 }
2463 // Finally, check the least significant entry.
2464 if (LeadingZero) {
2465 if (!UniquedVals[Multiple-1].getNode())
2466 return DAG.getTargetConstant(Val: 0, DL: SDLoc(N), VT: MVT::i32); // 0,0,0,undef
2467 int Val = UniquedVals[Multiple - 1]->getAsZExtVal();
2468 if (Val < 16) // 0,0,0,4 -> vspltisw(4)
2469 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2470 }
2471 if (LeadingOnes) {
2472 if (!UniquedVals[Multiple-1].getNode())
2473 return DAG.getTargetConstant(Val: ~0U, DL: SDLoc(N), VT: MVT::i32); // -1,-1,-1,undef
2474 int Val =cast<ConstantSDNode>(Val&: UniquedVals[Multiple-1])->getSExtValue();
2475 if (Val >= -16) // -1,-1,-1,-2 -> vspltisw(-2)
2476 return DAG.getTargetConstant(Val, DL: SDLoc(N), VT: MVT::i32);
2477 }
2478
2479 return SDValue();
2480 }
2481
2482 // Check to see if this buildvec has a single non-undef value in its elements.
2483 for (unsigned i = 0, e = N->getNumOperands(); i != e; ++i) {
2484 if (N->getOperand(Num: i).isUndef()) continue;
2485 if (!OpVal.getNode())
2486 OpVal = N->getOperand(Num: i);
2487 else if (OpVal != N->getOperand(Num: i))
2488 return SDValue();
2489 }
2490
2491 if (!OpVal.getNode()) return SDValue(); // All UNDEF: use implicit def.
2492
2493 unsigned ValSizeInBytes = EltSize;
2494 uint64_t Value = 0;
2495 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: OpVal)) {
2496 Value = CN->getZExtValue();
2497 } else if (ConstantFPSDNode *CN = dyn_cast<ConstantFPSDNode>(Val&: OpVal)) {
2498 assert(CN->getValueType(0) == MVT::f32 && "Only one legal FP vector type!");
2499 Value = llvm::bit_cast<uint32_t>(from: CN->getValueAPF().convertToFloat());
2500 }
2501
2502 // If the splat value is larger than the element value, then we can never do
2503 // this splat. The only case that we could fit the replicated bits into our
2504 // immediate field for would be zero, and we prefer to use vxor for it.
2505 if (ValSizeInBytes < ByteSize) return SDValue();
2506
2507 // If the element value is larger than the splat value, check if it consists
2508 // of a repeated bit pattern of size ByteSize.
2509 if (!APInt(ValSizeInBytes * 8, Value).isSplat(SplatSizeInBits: ByteSize * 8))
2510 return SDValue();
2511
2512 // Properly sign extend the value.
2513 int MaskVal = SignExtend32(X: Value, B: ByteSize * 8);
2514
2515 // If this is zero, don't match, zero matches ISD::isBuildVectorAllZeros.
2516 if (MaskVal == 0) return SDValue();
2517
2518 // Finally, if this value fits in a 5 bit sext field, return it
2519 if (SignExtend32<5>(X: MaskVal) == MaskVal)
2520 return DAG.getSignedTargetConstant(Val: MaskVal, DL: SDLoc(N), VT: MVT::i32);
2521 return SDValue();
2522}
2523
2524//===----------------------------------------------------------------------===//
2525// Addressing Mode Selection
2526//===----------------------------------------------------------------------===//
2527
2528/// isIntS16Immediate - This method tests to see if the node is either a 32-bit
2529/// or 64-bit immediate, and if the value can be accurately represented as a
2530/// sign extension from a 16-bit value. If so, this returns true and the
2531/// immediate.
2532bool llvm::isIntS16Immediate(SDNode *N, int16_t &Imm) {
2533 if (!isa<ConstantSDNode>(Val: N))
2534 return false;
2535
2536 Imm = (int16_t)N->getAsZExtVal();
2537 if (N->getValueType(ResNo: 0) == MVT::i32)
2538 return Imm == (int32_t)N->getAsZExtVal();
2539 else
2540 return Imm == (int64_t)N->getAsZExtVal();
2541}
2542bool llvm::isIntS16Immediate(SDValue Op, int16_t &Imm) {
2543 return isIntS16Immediate(N: Op.getNode(), Imm);
2544}
2545
2546/// Used when computing address flags for selecting loads and stores.
2547/// If we have an OR, check if the LHS and RHS are provably disjoint.
2548/// An OR of two provably disjoint values is equivalent to an ADD.
2549/// Most PPC load/store instructions compute the effective address as a sum,
2550/// so doing this conversion is useful.
2551static bool provablyDisjointOr(SelectionDAG &DAG, const SDValue &N) {
2552 if (N.getOpcode() != ISD::OR)
2553 return false;
2554 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2555 if (!LHSKnown.Zero.getBoolValue())
2556 return false;
2557 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2558 return (~(LHSKnown.Zero | RHSKnown.Zero) == 0);
2559}
2560
2561/// SelectAddressEVXRegReg - Given the specified address, check to see if it can
2562/// be represented as an indexed [r+r] operation.
2563bool PPCTargetLowering::SelectAddressEVXRegReg(SDValue N, SDValue &Base,
2564 SDValue &Index,
2565 SelectionDAG &DAG) const {
2566 for (SDNode *U : N->users()) {
2567 if (MemSDNode *Memop = dyn_cast<MemSDNode>(Val: U)) {
2568 if (Memop->getMemoryVT() == MVT::f64) {
2569 Base = N.getOperand(i: 0);
2570 Index = N.getOperand(i: 1);
2571 return true;
2572 }
2573 }
2574 }
2575 return false;
2576}
2577
2578/// isIntS34Immediate - This method tests if value of node given can be
2579/// accurately represented as a sign extension from a 34-bit value. If so,
2580/// this returns true and the immediate.
2581bool llvm::isIntS34Immediate(SDNode *N, int64_t &Imm) {
2582 if (!isa<ConstantSDNode>(Val: N))
2583 return false;
2584
2585 Imm = cast<ConstantSDNode>(Val: N)->getSExtValue();
2586 return isInt<34>(x: Imm);
2587}
2588bool llvm::isIntS34Immediate(SDValue Op, int64_t &Imm) {
2589 return isIntS34Immediate(N: Op.getNode(), Imm);
2590}
2591
2592/// SelectAddressRegReg - Given the specified addressed, check to see if it
2593/// can be represented as an indexed [r+r] operation. Returns false if it
2594/// can be more efficiently represented as [r+imm]. If \p EncodingAlignment is
2595/// non-zero and N can be represented by a base register plus a signed 16-bit
2596/// displacement, make a more precise judgement by checking (displacement % \p
2597/// EncodingAlignment).
2598bool PPCTargetLowering::SelectAddressRegReg(
2599 SDValue N, SDValue &Base, SDValue &Index, SelectionDAG &DAG,
2600 MaybeAlign EncodingAlignment) const {
2601 // If we have a PC Relative target flag don't select as [reg+reg]. It will be
2602 // a [pc+imm].
2603 if (SelectAddressPCRel(N, Base))
2604 return false;
2605
2606 int16_t Imm = 0;
2607 if (N.getOpcode() == ISD::ADD) {
2608 // Is there any SPE load/store (f64), which can't handle 16bit offset?
2609 // SPE load/store can only handle 8-bit offsets.
2610 if (hasSPE() && SelectAddressEVXRegReg(N, Base, Index, DAG))
2611 return true;
2612 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2613 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2614 return false; // r+i
2615 if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo)
2616 return false; // r+i
2617
2618 Base = N.getOperand(i: 0);
2619 Index = N.getOperand(i: 1);
2620 return true;
2621 } else if (N.getOpcode() == ISD::OR) {
2622 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm) &&
2623 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm)))
2624 return false; // r+i can fold it if we can.
2625
2626 // If this is an or of disjoint bitfields, we can codegen this as an add
2627 // (for better address arithmetic) if the LHS and RHS of the OR are provably
2628 // disjoint.
2629 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2630
2631 if (LHSKnown.Zero.getBoolValue()) {
2632 KnownBits RHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 1));
2633 // If all of the bits are known zero on the LHS or RHS, the add won't
2634 // carry.
2635 if (~(LHSKnown.Zero | RHSKnown.Zero) == 0) {
2636 Base = N.getOperand(i: 0);
2637 Index = N.getOperand(i: 1);
2638 return true;
2639 }
2640 }
2641 }
2642
2643 return false;
2644}
2645
2646// If we happen to be doing an i64 load or store into a stack slot that has
2647// less than a 4-byte alignment, then the frame-index elimination may need to
2648// use an indexed load or store instruction (because the offset may not be a
2649// multiple of 4). The extra register needed to hold the offset comes from the
2650// register scavenger, and it is possible that the scavenger will need to use
2651// an emergency spill slot. As a result, we need to make sure that a spill slot
2652// is allocated when doing an i64 load/store into a less-than-4-byte-aligned
2653// stack slot.
2654static void fixupFuncForFI(SelectionDAG &DAG, int FrameIdx, EVT VT) {
2655 // FIXME: This does not handle the LWA case.
2656 if (VT != MVT::i64)
2657 return;
2658
2659 // NOTE: We'll exclude negative FIs here, which come from argument
2660 // lowering, because there are no known test cases triggering this problem
2661 // using packed structures (or similar). We can remove this exclusion if
2662 // we find such a test case. The reason why this is so test-case driven is
2663 // because this entire 'fixup' is only to prevent crashes (from the
2664 // register scavenger) on not-really-valid inputs. For example, if we have:
2665 // %a = alloca i1
2666 // %b = bitcast i1* %a to i64*
2667 // store i64* a, i64 b
2668 // then the store should really be marked as 'align 1', but is not. If it
2669 // were marked as 'align 1' then the indexed form would have been
2670 // instruction-selected initially, and the problem this 'fixup' is preventing
2671 // won't happen regardless.
2672 if (FrameIdx < 0)
2673 return;
2674
2675 MachineFunction &MF = DAG.getMachineFunction();
2676 MachineFrameInfo &MFI = MF.getFrameInfo();
2677
2678 if (MFI.getObjectAlign(ObjectIdx: FrameIdx) >= Align(4))
2679 return;
2680
2681 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
2682 FuncInfo->setHasNonRISpills();
2683}
2684
2685/// Returns true if the address N can be represented by a base register plus
2686/// a signed 16-bit displacement [r+imm], and if it is not better
2687/// represented as reg+reg. If \p EncodingAlignment is non-zero, only accept
2688/// displacements that are multiples of that value.
2689bool PPCTargetLowering::SelectAddressRegImm(
2690 SDValue N, SDValue &Disp, SDValue &Base, SelectionDAG &DAG,
2691 MaybeAlign EncodingAlignment) const {
2692 // FIXME dl should come from parent load or store, not from address
2693 SDLoc dl(N);
2694
2695 // If we have a PC Relative target flag don't select as [reg+imm]. It will be
2696 // a [pc+imm].
2697 if (SelectAddressPCRel(N, Base))
2698 return false;
2699
2700 // If this can be more profitably realized as r+r, fail.
2701 if (SelectAddressRegReg(N, Base&: Disp, Index&: Base, DAG, EncodingAlignment))
2702 return false;
2703
2704 if (N.getOpcode() == ISD::ADD) {
2705 int16_t imm = 0;
2706 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2707 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2708 Disp = DAG.getSignedTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2709 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2710 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2711 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2712 } else {
2713 Base = N.getOperand(i: 0);
2714 }
2715 return true; // [r+i]
2716 } else if (N.getOperand(i: 1).getOpcode() == PPCISD::Lo) {
2717 // Match LOAD (ADD (X, Lo(G))).
2718 assert(!N.getOperand(1).getConstantOperandVal(1) &&
2719 "Cannot handle constant offsets yet!");
2720 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
2721 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
2722 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
2723 Disp.getOpcode() == ISD::TargetConstantPool ||
2724 Disp.getOpcode() == ISD::TargetJumpTable);
2725 Base = N.getOperand(i: 0);
2726 return true; // [&g+r]
2727 }
2728 } else if (N.getOpcode() == ISD::OR) {
2729 int16_t imm = 0;
2730 if (isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) &&
2731 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: imm))) {
2732 // If this is an or of disjoint bitfields, we can codegen this as an add
2733 // (for better address arithmetic) if the LHS and RHS of the OR are
2734 // provably disjoint.
2735 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2736
2737 if ((LHSKnown.Zero.getZExtValue()|~(uint64_t)imm) == ~0ULL) {
2738 // If all of the bits are known zero on the LHS or RHS, the add won't
2739 // carry.
2740 if (FrameIndexSDNode *FI =
2741 dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0))) {
2742 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2743 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2744 } else {
2745 Base = N.getOperand(i: 0);
2746 }
2747 Disp = DAG.getTargetConstant(Val: imm, DL: dl, VT: N.getValueType());
2748 return true;
2749 }
2750 }
2751 } else if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
2752 // Loading from a constant address.
2753
2754 // If this address fits entirely in a 16-bit sext immediate field, codegen
2755 // this as "d, 0"
2756 int16_t Imm;
2757 if (isIntS16Immediate(N: CN, Imm) &&
2758 (!EncodingAlignment || isAligned(Lhs: *EncodingAlignment, SizeInBytes: Imm))) {
2759 Disp = DAG.getTargetConstant(Val: Imm, DL: dl, VT: CN->getValueType(ResNo: 0));
2760 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2761 VT: CN->getValueType(ResNo: 0));
2762 return true;
2763 }
2764
2765 // Handle 32-bit sext immediates with LIS + addr mode.
2766 if ((CN->getValueType(ResNo: 0) == MVT::i32 ||
2767 (int64_t)CN->getZExtValue() == (int)CN->getZExtValue()) &&
2768 (!EncodingAlignment ||
2769 isAligned(Lhs: *EncodingAlignment, SizeInBytes: CN->getZExtValue()))) {
2770 int Addr = (int)CN->getZExtValue();
2771
2772 // Otherwise, break this down into an LIS + disp.
2773 Disp = DAG.getTargetConstant(Val: (short)Addr, DL: dl, VT: MVT::i32);
2774
2775 Base = DAG.getTargetConstant(Val: (Addr - (signed short)Addr) >> 16, DL: dl,
2776 VT: MVT::i32);
2777 unsigned Opc = CN->getValueType(ResNo: 0) == MVT::i32 ? PPC::LIS : PPC::LIS8;
2778 Base = SDValue(DAG.getMachineNode(Opcode: Opc, dl, VT: CN->getValueType(ResNo: 0), Op1: Base), 0);
2779 return true;
2780 }
2781 }
2782
2783 Disp = DAG.getTargetConstant(Val: 0, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()));
2784 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
2785 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2786 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
2787 } else
2788 Base = N;
2789 return true; // [r+0]
2790}
2791
2792/// Similar to the 16-bit case but for instructions that take a 34-bit
2793/// displacement field (prefixed loads/stores).
2794bool PPCTargetLowering::SelectAddressRegImm34(SDValue N, SDValue &Disp,
2795 SDValue &Base,
2796 SelectionDAG &DAG) const {
2797 // Only on 64-bit targets.
2798 if (N.getValueType() != MVT::i64)
2799 return false;
2800
2801 SDLoc dl(N);
2802 int64_t Imm = 0;
2803
2804 if (N.getOpcode() == ISD::ADD) {
2805 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2806 return false;
2807 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2808 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2809 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2810 else
2811 Base = N.getOperand(i: 0);
2812 return true;
2813 }
2814
2815 if (N.getOpcode() == ISD::OR) {
2816 if (!isIntS34Immediate(Op: N.getOperand(i: 1), Imm))
2817 return false;
2818 // If this is an or of disjoint bitfields, we can codegen this as an add
2819 // (for better address arithmetic) if the LHS and RHS of the OR are
2820 // provably disjoint.
2821 KnownBits LHSKnown = DAG.computeKnownBits(Op: N.getOperand(i: 0));
2822 if ((LHSKnown.Zero.getZExtValue() | ~(uint64_t)Imm) != ~0ULL)
2823 return false;
2824 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
2825 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
2826 else
2827 Base = N.getOperand(i: 0);
2828 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2829 return true;
2830 }
2831
2832 if (isIntS34Immediate(Op: N, Imm)) { // If the address is a 34-bit const.
2833 Disp = DAG.getSignedTargetConstant(Val: Imm, DL: dl, VT: N.getValueType());
2834 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
2835 return true;
2836 }
2837
2838 return false;
2839}
2840
2841/// SelectAddressRegRegOnly - Given the specified addressed, force it to be
2842/// represented as an indexed [r+r] operation.
2843bool PPCTargetLowering::SelectAddressRegRegOnly(SDValue N, SDValue &Base,
2844 SDValue &Index,
2845 SelectionDAG &DAG) const {
2846 // Check to see if we can easily represent this as an [r+r] address. This
2847 // will fail if it thinks that the address is more profitably represented as
2848 // reg+imm, e.g. where imm = 0.
2849 if (SelectAddressRegReg(N, Base, Index, DAG))
2850 return true;
2851
2852 // If the address is the result of an add, we will utilize the fact that the
2853 // address calculation includes an implicit add. However, we can reduce
2854 // register pressure if we do not materialize a constant just for use as the
2855 // index register. We only get rid of the add if it is not an add of a
2856 // value and a 16-bit signed constant and both have a single use.
2857 int16_t imm = 0;
2858 if (N.getOpcode() == ISD::ADD &&
2859 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: imm) ||
2860 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
2861 Base = N.getOperand(i: 0);
2862 Index = N.getOperand(i: 1);
2863 return true;
2864 }
2865
2866 // Otherwise, do it the hard way, using R0 as the base register.
2867 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
2868 VT: N.getValueType());
2869 Index = N;
2870 return true;
2871}
2872
2873template <typename Ty> static bool isValidPCRelNode(SDValue N) {
2874 Ty *PCRelCand = dyn_cast<Ty>(N);
2875 return PCRelCand && (PPCInstrInfo::hasPCRelFlag(TF: PCRelCand->getTargetFlags()));
2876}
2877
2878/// Returns true if this address is a PC Relative address.
2879/// PC Relative addresses are marked with the flag PPCII::MO_PCREL_FLAG
2880/// or if the node opcode is PPCISD::MAT_PCREL_ADDR.
2881bool PPCTargetLowering::SelectAddressPCRel(SDValue N, SDValue &Base) const {
2882 // This is a materialize PC Relative node. Always select this as PC Relative.
2883 Base = N;
2884 if (N.getOpcode() == PPCISD::MAT_PCREL_ADDR)
2885 return true;
2886 if (isValidPCRelNode<ConstantPoolSDNode>(N) ||
2887 isValidPCRelNode<GlobalAddressSDNode>(N) ||
2888 isValidPCRelNode<JumpTableSDNode>(N) ||
2889 isValidPCRelNode<BlockAddressSDNode>(N))
2890 return true;
2891 return false;
2892}
2893
2894/// Returns true if we should use a direct load into vector instruction
2895/// (such as lxsd or lfd), instead of a load into gpr + direct move sequence.
2896static bool usePartialVectorLoads(SDNode *N, const PPCSubtarget& ST) {
2897
2898 // If there are any other uses other than scalar to vector, then we should
2899 // keep it as a scalar load -> direct move pattern to prevent multiple
2900 // loads.
2901 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N);
2902 if (!LD)
2903 return false;
2904
2905 EVT MemVT = LD->getMemoryVT();
2906 if (!MemVT.isSimple())
2907 return false;
2908 switch(MemVT.getSimpleVT().SimpleTy) {
2909 case MVT::i64:
2910 break;
2911 case MVT::i32:
2912 if (!ST.hasP8Vector())
2913 return false;
2914 break;
2915 case MVT::i16:
2916 case MVT::i8:
2917 if (!ST.hasP9Vector())
2918 return false;
2919 break;
2920 default:
2921 return false;
2922 }
2923
2924 SDValue LoadedVal(N, 0);
2925 if (!LoadedVal.hasOneUse())
2926 return false;
2927
2928 for (SDUse &Use : LD->uses())
2929 if (Use.getResNo() == 0 &&
2930 Use.getUser()->getOpcode() != ISD::SCALAR_TO_VECTOR &&
2931 Use.getUser()->getOpcode() != PPCISD::SCALAR_TO_VECTOR_PERMUTED)
2932 return false;
2933
2934 return true;
2935}
2936
2937/// getPreIndexedAddressParts - returns true by value, base pointer and
2938/// offset pointer and addressing mode by reference if the node's address
2939/// can be legally represented as pre-indexed load / store address.
2940bool PPCTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
2941 SDValue &Offset,
2942 ISD::MemIndexedMode &AM,
2943 SelectionDAG &DAG) const {
2944 if (!EnablePPCPreinc)
2945 return false;
2946
2947 bool isLoad = true;
2948 SDValue Ptr;
2949 EVT VT;
2950 Align Alignment;
2951 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
2952 Ptr = LD->getBasePtr();
2953 VT = LD->getMemoryVT();
2954 Alignment = LD->getAlign();
2955 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
2956 Ptr = ST->getBasePtr();
2957 VT = ST->getMemoryVT();
2958 Alignment = ST->getAlign();
2959 isLoad = false;
2960 } else
2961 return false;
2962
2963 // Do not generate pre-inc forms for specific loads that feed scalar_to_vector
2964 // instructions because we can fold these into a more efficient instruction
2965 // instead, (such as LXSD).
2966 if (isLoad && usePartialVectorLoads(N, ST: Subtarget)) {
2967 return false;
2968 }
2969
2970 // PowerPC doesn't have preinc load/store instructions for vectors
2971 if (VT.isVector())
2972 return false;
2973
2974 if (SelectAddressRegReg(N: Ptr, Base, Index&: Offset, DAG)) {
2975 // Common code will reject creating a pre-inc form if the base pointer
2976 // is a frame index, or if N is a store and the base pointer is either
2977 // the same as or a predecessor of the value being stored. Check for
2978 // those situations here, and try with swapped Base/Offset instead.
2979 bool Swap = false;
2980
2981 if (isa<FrameIndexSDNode>(Val: Base) || isa<RegisterSDNode>(Val: Base))
2982 Swap = true;
2983 else if (!isLoad) {
2984 SDValue Val = cast<StoreSDNode>(Val: N)->getValue();
2985 if (Val == Base || Base.getNode()->isPredecessorOf(N: Val.getNode()))
2986 Swap = true;
2987 }
2988
2989 if (Swap)
2990 std::swap(a&: Base, b&: Offset);
2991
2992 AM = ISD::PRE_INC;
2993 return true;
2994 }
2995
2996 // LDU/STU can only handle immediates that are a multiple of 4.
2997 if (VT != MVT::i64) {
2998 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: std::nullopt))
2999 return false;
3000 } else {
3001 // LDU/STU need an address with at least 4-byte alignment.
3002 if (Alignment < Align(4))
3003 return false;
3004
3005 if (!SelectAddressRegImm(N: Ptr, Disp&: Offset, Base, DAG, EncodingAlignment: Align(4)))
3006 return false;
3007 }
3008
3009 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
3010 // PPC64 doesn't have lwau, but it does have lwaux. Reject preinc load of
3011 // sext i32 to i64 when addr mode is r+i.
3012 if (LD->getValueType(ResNo: 0) == MVT::i64 && LD->getMemoryVT() == MVT::i32 &&
3013 LD->getExtensionType() == ISD::SEXTLOAD &&
3014 isa<ConstantSDNode>(Val: Offset))
3015 return false;
3016 }
3017
3018 AM = ISD::PRE_INC;
3019 return true;
3020}
3021
3022//===----------------------------------------------------------------------===//
3023// LowerOperation implementation
3024//===----------------------------------------------------------------------===//
3025
3026/// Return true if we should reference labels using a PICBase, set the HiOpFlags
3027/// and LoOpFlags to the target MO flags.
3028static void getLabelAccessInfo(bool IsPIC, const PPCSubtarget &Subtarget,
3029 unsigned &HiOpFlags, unsigned &LoOpFlags,
3030 const GlobalValue *GV = nullptr) {
3031 HiOpFlags = PPCII::MO_HA;
3032 LoOpFlags = PPCII::MO_LO;
3033
3034 // Don't use the pic base if not in PIC relocation model.
3035 if (IsPIC) {
3036 HiOpFlags = PPCII::MO_PIC_HA_FLAG;
3037 LoOpFlags = PPCII::MO_PIC_LO_FLAG;
3038 }
3039}
3040
3041static SDValue LowerLabelRef(SDValue HiPart, SDValue LoPart, bool isPIC,
3042 SelectionDAG &DAG) {
3043 SDLoc DL(HiPart);
3044 EVT PtrVT = HiPart.getValueType();
3045 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: PtrVT);
3046
3047 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL, VT: PtrVT, N1: HiPart, N2: Zero);
3048 SDValue Lo = DAG.getNode(Opcode: PPCISD::Lo, DL, VT: PtrVT, N1: LoPart, N2: Zero);
3049
3050 // With PIC, the first instruction is actually "GR+hi(&G)".
3051 if (isPIC)
3052 Hi = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT,
3053 N1: DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL, VT: PtrVT), N2: Hi);
3054
3055 // Generate non-pic code that has direct accesses to the constant pool.
3056 // The address of the global is just (hi(&g)+lo(&g)).
3057 return DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: Hi, N2: Lo);
3058}
3059
3060static void setUsesTOCBasePtr(MachineFunction &MF) {
3061 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3062 FuncInfo->setUsesTOCBasePtr();
3063}
3064
3065static void setUsesTOCBasePtr(SelectionDAG &DAG) {
3066 setUsesTOCBasePtr(DAG.getMachineFunction());
3067}
3068
3069SDValue PPCTargetLowering::getTOCEntry(SelectionDAG &DAG, const SDLoc &dl,
3070 SDValue GA) const {
3071 EVT VT = Subtarget.getScalarIntVT();
3072 SDValue Reg = Subtarget.isPPC64() ? DAG.getRegister(Reg: PPC::X2, VT)
3073 : Subtarget.isAIXABI()
3074 ? DAG.getRegister(Reg: PPC::R2, VT)
3075 : DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT);
3076 SDValue Ops[] = { GA, Reg };
3077 return DAG.getMemIntrinsicNode(
3078 Opcode: PPCISD::TOC_ENTRY, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops, MemVT: VT,
3079 PtrInfo: MachinePointerInfo::getGOT(MF&: DAG.getMachineFunction()), Alignment: std::nullopt,
3080 Flags: MachineMemOperand::MOLoad);
3081}
3082
3083SDValue PPCTargetLowering::LowerConstantPool(SDValue Op,
3084 SelectionDAG &DAG) const {
3085 EVT PtrVT = Op.getValueType();
3086 ConstantPoolSDNode *CP = cast<ConstantPoolSDNode>(Val&: Op);
3087 const Constant *C = CP->getConstVal();
3088
3089 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3090 // The actual address of the GlobalValue is stored in the TOC.
3091 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3092 if (Subtarget.isUsingPCRelativeCalls()) {
3093 SDLoc DL(CP);
3094 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3095 SDValue ConstPool = DAG.getTargetConstantPool(
3096 C, VT: Ty, Align: CP->getAlign(), Offset: CP->getOffset(), TargetFlags: PPCII::MO_PCREL_FLAG);
3097 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: ConstPool);
3098 }
3099 setUsesTOCBasePtr(DAG);
3100 SDValue GA = DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0);
3101 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3102 }
3103
3104 unsigned MOHiFlag, MOLoFlag;
3105 bool IsPIC = isPositionIndependent();
3106 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3107
3108 if (IsPIC && Subtarget.isSVR4ABI()) {
3109 SDValue GA =
3110 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: PPCII::MO_PIC_FLAG);
3111 return getTOCEntry(DAG, dl: SDLoc(CP), GA);
3112 }
3113
3114 SDValue CPIHi =
3115 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOHiFlag);
3116 SDValue CPILo =
3117 DAG.getTargetConstantPool(C, VT: PtrVT, Align: CP->getAlign(), Offset: 0, TargetFlags: MOLoFlag);
3118 return LowerLabelRef(HiPart: CPIHi, LoPart: CPILo, isPIC: IsPIC, DAG);
3119}
3120
3121// For 64-bit PowerPC, prefer the more compact relative encodings.
3122// This trades 32 bits per jump table entry for one or two instructions
3123// on the jump site.
3124unsigned PPCTargetLowering::getJumpTableEncoding() const {
3125 if (isJumpTableRelative())
3126 return MachineJumpTableInfo::EK_LabelDifference32;
3127
3128 return TargetLowering::getJumpTableEncoding();
3129}
3130
3131bool PPCTargetLowering::isJumpTableRelative() const {
3132 if (UseAbsoluteJumpTables)
3133 return false;
3134 if (Subtarget.isPPC64() || Subtarget.isAIXABI())
3135 return true;
3136 return TargetLowering::isJumpTableRelative();
3137}
3138
3139SDValue PPCTargetLowering::getPICJumpTableRelocBase(SDValue Table,
3140 SelectionDAG &DAG) const {
3141 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3142 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3143
3144 switch (getTargetMachine().getCodeModel()) {
3145 case CodeModel::Small:
3146 case CodeModel::Medium:
3147 return TargetLowering::getPICJumpTableRelocBase(Table, DAG);
3148 default:
3149 return DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: SDLoc(),
3150 VT: getPointerTy(DL: DAG.getDataLayout()));
3151 }
3152}
3153
3154const MCExpr *
3155PPCTargetLowering::getPICJumpTableRelocBaseExpr(const MachineFunction *MF,
3156 unsigned JTI,
3157 MCContext &Ctx) const {
3158 if (!Subtarget.isPPC64() || Subtarget.isAIXABI())
3159 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3160
3161 switch (getTargetMachine().getCodeModel()) {
3162 case CodeModel::Small:
3163 case CodeModel::Medium:
3164 return TargetLowering::getPICJumpTableRelocBaseExpr(MF, JTI, Ctx);
3165 default:
3166 return MCSymbolRefExpr::create(Symbol: MF->getPICBaseSymbol(), Ctx);
3167 }
3168}
3169
3170SDValue PPCTargetLowering::LowerJumpTable(SDValue Op, SelectionDAG &DAG) const {
3171 EVT PtrVT = Op.getValueType();
3172 JumpTableSDNode *JT = cast<JumpTableSDNode>(Val&: Op);
3173
3174 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3175 if (Subtarget.isUsingPCRelativeCalls()) {
3176 SDLoc DL(JT);
3177 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3178 SDValue GA =
3179 DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: Ty, TargetFlags: PPCII::MO_PCREL_FLAG);
3180 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3181 return MatAddr;
3182 }
3183
3184 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3185 // The actual address of the GlobalValue is stored in the TOC.
3186 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3187 setUsesTOCBasePtr(DAG);
3188 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT);
3189 return getTOCEntry(DAG, dl: SDLoc(JT), GA);
3190 }
3191
3192 unsigned MOHiFlag, MOLoFlag;
3193 bool IsPIC = isPositionIndependent();
3194 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3195
3196 if (IsPIC && Subtarget.isSVR4ABI()) {
3197 SDValue GA = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT,
3198 TargetFlags: PPCII::MO_PIC_FLAG);
3199 return getTOCEntry(DAG, dl: SDLoc(GA), GA);
3200 }
3201
3202 SDValue JTIHi = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOHiFlag);
3203 SDValue JTILo = DAG.getTargetJumpTable(JTI: JT->getIndex(), VT: PtrVT, TargetFlags: MOLoFlag);
3204 return LowerLabelRef(HiPart: JTIHi, LoPart: JTILo, isPIC: IsPIC, DAG);
3205}
3206
3207SDValue PPCTargetLowering::LowerBlockAddress(SDValue Op,
3208 SelectionDAG &DAG) const {
3209 EVT PtrVT = Op.getValueType();
3210 BlockAddressSDNode *BASDN = cast<BlockAddressSDNode>(Val&: Op);
3211 const BlockAddress *BA = BASDN->getBlockAddress();
3212
3213 // isUsingPCRelativeCalls() returns true when PCRelative is enabled
3214 if (Subtarget.isUsingPCRelativeCalls()) {
3215 SDLoc DL(BASDN);
3216 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3217 SDValue GA = DAG.getTargetBlockAddress(BA, VT: Ty, Offset: BASDN->getOffset(),
3218 TargetFlags: PPCII::MO_PCREL_FLAG);
3219 SDValue MatAddr = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3220 return MatAddr;
3221 }
3222
3223 // 64-bit SVR4 ABI and AIX ABI code are always position-independent.
3224 // The actual BlockAddress is stored in the TOC.
3225 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3226 setUsesTOCBasePtr(DAG);
3227 SDValue GA = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset());
3228 return getTOCEntry(DAG, dl: SDLoc(BASDN), GA);
3229 }
3230
3231 // 32-bit position-independent ELF stores the BlockAddress in the .got.
3232 if (Subtarget.is32BitELFABI() && isPositionIndependent())
3233 return getTOCEntry(
3234 DAG, dl: SDLoc(BASDN),
3235 GA: DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: BASDN->getOffset()));
3236
3237 unsigned MOHiFlag, MOLoFlag;
3238 bool IsPIC = isPositionIndependent();
3239 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag);
3240 SDValue TgtBAHi = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOHiFlag);
3241 SDValue TgtBALo = DAG.getTargetBlockAddress(BA, VT: PtrVT, Offset: 0, TargetFlags: MOLoFlag);
3242 return LowerLabelRef(HiPart: TgtBAHi, LoPart: TgtBALo, isPIC: IsPIC, DAG);
3243}
3244
3245SDValue PPCTargetLowering::LowerGlobalTLSAddress(SDValue Op,
3246 SelectionDAG &DAG) const {
3247 if (Subtarget.isAIXABI())
3248 return LowerGlobalTLSAddressAIX(Op, DAG);
3249
3250 return LowerGlobalTLSAddressLinux(Op, DAG);
3251}
3252
3253/// updateForAIXShLibTLSModelOpt - Helper to initialize TLS model opt settings,
3254/// and then apply the update.
3255static void updateForAIXShLibTLSModelOpt(TLSModel::Model &Model,
3256 SelectionDAG &DAG,
3257 const TargetMachine &TM) {
3258 // Initialize TLS model opt setting lazily:
3259 // (1) Use initial-exec for single TLS var references within current function.
3260 // (2) Use local-dynamic for multiple TLS var references within current
3261 // function.
3262 PPCFunctionInfo *FuncInfo =
3263 DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
3264 if (!FuncInfo->isAIXFuncTLSModelOptInitDone()) {
3265 SmallPtrSet<const GlobalValue *, 8> TLSGV;
3266 // Iterate over all instructions within current function, collect all TLS
3267 // global variables (global variables taken as the first parameter to
3268 // Intrinsic::threadlocal_address).
3269 const Function &Func = DAG.getMachineFunction().getFunction();
3270 for (const BasicBlock &BB : Func)
3271 for (const Instruction &I : BB)
3272 if (I.getOpcode() == Instruction::Call)
3273 if (const CallInst *CI = dyn_cast<const CallInst>(Val: &I))
3274 if (Function *CF = CI->getCalledFunction())
3275 if (CF->isDeclaration() &&
3276 CF->getIntrinsicID() == Intrinsic::threadlocal_address)
3277 if (const GlobalValue *GV =
3278 dyn_cast<GlobalValue>(Val: I.getOperand(i: 0))) {
3279 TLSModel::Model GVModel = TM.getTLSModel(GV);
3280 if (GVModel == TLSModel::LocalDynamic)
3281 TLSGV.insert(Ptr: GV);
3282 }
3283
3284 unsigned TLSGVCnt = TLSGV.size();
3285 LLVM_DEBUG(dbgs() << format("LocalDynamic TLSGV count:%d\n", TLSGVCnt));
3286 if (TLSGVCnt <= PPCAIXTLSModelOptUseIEForLDLimit)
3287 FuncInfo->setAIXFuncUseTLSIEForLD();
3288 FuncInfo->setAIXFuncTLSModelOptInitDone();
3289 }
3290
3291 if (FuncInfo->isAIXFuncUseTLSIEForLD()) {
3292 LLVM_DEBUG(
3293 dbgs() << DAG.getMachineFunction().getName()
3294 << " function is using the TLS-IE model for TLS-LD access.\n");
3295 Model = TLSModel::InitialExec;
3296 }
3297}
3298
3299SDValue PPCTargetLowering::LowerGlobalTLSAddressAIX(SDValue Op,
3300 SelectionDAG &DAG) const {
3301 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3302
3303 if (DAG.getTarget().useEmulatedTLS())
3304 report_fatal_error(reason: "Emulated TLS is not yet supported on AIX");
3305
3306 SDLoc dl(GA);
3307 const GlobalValue *GV = GA->getGlobal();
3308 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3309 bool Is64Bit = Subtarget.isPPC64();
3310 TLSModel::Model Model = getTargetMachine().getTLSModel(GV);
3311
3312 // Apply update to the TLS model.
3313 if (Subtarget.hasAIXShLibTLSModelOpt())
3314 updateForAIXShLibTLSModelOpt(Model, DAG, TM: getTargetMachine());
3315
3316 // TLS variables are accessed through TOC entries.
3317 // To support this, set the DAG to use the TOC base pointer.
3318 setUsesTOCBasePtr(DAG);
3319
3320 bool IsTLSLocalExecModel = Model == TLSModel::LocalExec;
3321
3322 if (IsTLSLocalExecModel || Model == TLSModel::InitialExec) {
3323 bool HasAIXSmallLocalExecTLS = Subtarget.hasAIXSmallLocalExecTLS();
3324 bool HasAIXSmallTLSGlobalAttr = false;
3325 SDValue VariableOffsetTGA =
3326 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TPREL_FLAG);
3327 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3328 SDValue TLSReg;
3329
3330 if (const GlobalVariable *GVar = dyn_cast<GlobalVariable>(Val: GV))
3331 if (GVar->hasAttribute(Kind: "aix-small-tls"))
3332 HasAIXSmallTLSGlobalAttr = true;
3333
3334 if (Is64Bit) {
3335 // For local-exec and initial-exec on AIX (64-bit), the sequence generated
3336 // involves a load of the variable offset (from the TOC), followed by an
3337 // add of the loaded variable offset to R13 (the thread pointer).
3338 // This code sequence looks like:
3339 // ld reg1,var[TC](2)
3340 // add reg2, reg1, r13 // r13 contains the thread pointer
3341 TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3342
3343 // With the -maix-small-local-exec-tls option, or with the "aix-small-tls"
3344 // global variable attribute, produce a faster access sequence for
3345 // local-exec TLS variables where the offset from the TLS base is encoded
3346 // as an immediate operand.
3347 //
3348 // We only utilize the faster local-exec access sequence when the TLS
3349 // variable has a size within the policy limit. We treat types that are
3350 // not sized or are empty as being over the policy size limit.
3351 if ((HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr) &&
3352 IsTLSLocalExecModel) {
3353 Type *GVType = GV->getValueType();
3354 if (GVType->isSized() && !GVType->isEmptyTy() &&
3355 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3356 AIXSmallTlsPolicySizeLimit)
3357 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA, N2: TLSReg);
3358 }
3359 } else {
3360 // For local-exec and initial-exec on AIX (32-bit), the sequence generated
3361 // involves loading the variable offset from the TOC, generating a call to
3362 // .__get_tpointer to get the thread pointer (which will be in R3), and
3363 // adding the two together:
3364 // lwz reg1,var[TC](2)
3365 // bla .__get_tpointer
3366 // add reg2, reg1, r3
3367 TLSReg = DAG.getNode(Opcode: PPCISD::GET_TPOINTER, DL: dl, VT: PtrVT);
3368
3369 // We do not implement the 32-bit version of the faster access sequence
3370 // for local-exec that is controlled by the -maix-small-local-exec-tls
3371 // option, or the "aix-small-tls" global variable attribute.
3372 if (HasAIXSmallLocalExecTLS || HasAIXSmallTLSGlobalAttr)
3373 report_fatal_error(reason: "The small-local-exec TLS access sequence is "
3374 "currently only supported on AIX (64-bit mode).");
3375 }
3376 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: VariableOffset);
3377 }
3378
3379 if (Model == TLSModel::LocalDynamic) {
3380 bool HasAIXSmallLocalDynamicTLS = Subtarget.hasAIXSmallLocalDynamicTLS();
3381
3382 // We do not implement the 32-bit version of the faster access sequence
3383 // for local-dynamic that is controlled by -maix-small-local-dynamic-tls.
3384 if (!Is64Bit && HasAIXSmallLocalDynamicTLS)
3385 report_fatal_error(reason: "The small-local-dynamic TLS access sequence is "
3386 "currently only supported on AIX (64-bit mode).");
3387
3388 // For local-dynamic on AIX, we need to generate one TOC entry for each
3389 // variable offset, and a single module-handle TOC entry for the entire
3390 // file.
3391
3392 SDValue VariableOffsetTGA =
3393 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLD_FLAG);
3394 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3395
3396 Module *M = DAG.getMachineFunction().getFunction().getParent();
3397 GlobalVariable *TLSGV =
3398 dyn_cast_or_null<GlobalVariable>(Val: M->getOrInsertGlobal(
3399 Name: StringRef("_$TLSML"), Ty: PointerType::getUnqual(C&: *DAG.getContext())));
3400 TLSGV->setThreadLocalMode(GlobalVariable::LocalDynamicTLSModel);
3401 assert(TLSGV && "Not able to create GV for _$TLSML.");
3402 SDValue ModuleHandleTGA =
3403 DAG.getTargetGlobalAddress(GV: TLSGV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSLDM_FLAG);
3404 SDValue ModuleHandleTOC = getTOCEntry(DAG, dl, GA: ModuleHandleTGA);
3405 SDValue ModuleHandle =
3406 DAG.getNode(Opcode: PPCISD::TLSLD_AIX, DL: dl, VT: PtrVT, Operand: ModuleHandleTOC);
3407
3408 // With the -maix-small-local-dynamic-tls option, produce a faster access
3409 // sequence for local-dynamic TLS variables where the offset from the
3410 // module-handle is encoded as an immediate operand.
3411 //
3412 // We only utilize the faster local-dynamic access sequence when the TLS
3413 // variable has a size within the policy limit. We treat types that are
3414 // not sized or are empty as being over the policy size limit.
3415 if (HasAIXSmallLocalDynamicTLS) {
3416 Type *GVType = GV->getValueType();
3417 if (GVType->isSized() && !GVType->isEmptyTy() &&
3418 GV->getDataLayout().getTypeAllocSize(Ty: GVType) <=
3419 AIXSmallTlsPolicySizeLimit)
3420 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: VariableOffsetTGA,
3421 N2: ModuleHandle);
3422 }
3423
3424 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: ModuleHandle, N2: VariableOffset);
3425 }
3426
3427 // If Local- or Initial-exec or Local-dynamic is not possible or specified,
3428 // all GlobalTLSAddress nodes are lowered using the general-dynamic model. We
3429 // need to generate two TOC entries, one for the variable offset, one for the
3430 // region handle. The global address for the TOC entry of the region handle is
3431 // created with the MO_TLSGDM_FLAG flag and the global address for the TOC
3432 // entry of the variable offset is created with MO_TLSGD_FLAG.
3433 SDValue VariableOffsetTGA =
3434 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGD_FLAG);
3435 SDValue RegionHandleTGA =
3436 DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: PPCII::MO_TLSGDM_FLAG);
3437 SDValue VariableOffset = getTOCEntry(DAG, dl, GA: VariableOffsetTGA);
3438 SDValue RegionHandle = getTOCEntry(DAG, dl, GA: RegionHandleTGA);
3439 return DAG.getNode(Opcode: PPCISD::TLSGD_AIX, DL: dl, VT: PtrVT, N1: VariableOffset,
3440 N2: RegionHandle);
3441}
3442
3443SDValue PPCTargetLowering::LowerGlobalTLSAddressLinux(SDValue Op,
3444 SelectionDAG &DAG) const {
3445 // FIXME: TLS addresses currently use medium model code sequences,
3446 // which is the most useful form. Eventually support for small and
3447 // large models could be added if users need it, at the cost of
3448 // additional complexity.
3449 GlobalAddressSDNode *GA = cast<GlobalAddressSDNode>(Val&: Op);
3450 if (DAG.getTarget().useEmulatedTLS())
3451 return LowerToTLSEmulatedModel(GA, DAG);
3452
3453 SDLoc dl(GA);
3454 const GlobalValue *GV = GA->getGlobal();
3455 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3456 bool is64bit = Subtarget.isPPC64();
3457 const Module *M = DAG.getMachineFunction().getFunction().getParent();
3458 PICLevel::Level picLevel = M->getPICLevel();
3459
3460 const TargetMachine &TM = getTargetMachine();
3461 TLSModel::Model Model = TM.getTLSModel(GV);
3462
3463 if (Model == TLSModel::LocalExec) {
3464 if (Subtarget.isUsingPCRelativeCalls()) {
3465 SDValue TLSReg = DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
3466 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3467 TargetFlags: PPCII::MO_TPREL_PCREL_FLAG);
3468 SDValue MatAddr =
3469 DAG.getNode(Opcode: PPCISD::TLS_LOCAL_EXEC_MAT_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3470 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TLSReg, N2: MatAddr);
3471 }
3472
3473 SDValue TGAHi = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3474 TargetFlags: PPCII::MO_TPREL_HA);
3475 SDValue TGALo = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3476 TargetFlags: PPCII::MO_TPREL_LO);
3477 SDValue TLSReg = is64bit ? DAG.getRegister(Reg: PPC::X13, VT: MVT::i64)
3478 : DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
3479
3480 SDValue Hi = DAG.getNode(Opcode: PPCISD::Hi, DL: dl, VT: PtrVT, N1: TGAHi, N2: TLSReg);
3481 return DAG.getNode(Opcode: PPCISD::Lo, DL: dl, VT: PtrVT, N1: TGALo, N2: Hi);
3482 }
3483
3484 if (Model == TLSModel::InitialExec) {
3485 bool IsPCRel = Subtarget.isUsingPCRelativeCalls();
3486 SDValue TGA = DAG.getTargetGlobalAddress(
3487 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_GOT_TPREL_PCREL_FLAG : 0);
3488 SDValue TGATLS = DAG.getTargetGlobalAddress(
3489 GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: IsPCRel ? PPCII::MO_TLS_PCREL_FLAG : PPCII::MO_TLS);
3490 SDValue TPOffset;
3491 if (IsPCRel) {
3492 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3493 TPOffset = DAG.getLoad(VT: MVT::i64, dl, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3494 PtrInfo: MachinePointerInfo());
3495 } else {
3496 SDValue GOTPtr;
3497 if (is64bit) {
3498 setUsesTOCBasePtr(DAG);
3499 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3500 GOTPtr =
3501 DAG.getNode(Opcode: PPCISD::ADDIS_GOT_TPREL_HA, DL: dl, VT: PtrVT, N1: GOTReg, N2: TGA);
3502 } else {
3503 if (!TM.isPositionIndependent())
3504 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_GOT, DL: dl, VT: PtrVT);
3505 else if (picLevel == PICLevel::SmallPIC)
3506 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3507 else
3508 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3509 }
3510 TPOffset = DAG.getNode(Opcode: PPCISD::LD_GOT_TPREL_L, DL: dl, VT: PtrVT, N1: TGA, N2: GOTPtr);
3511 }
3512 return DAG.getNode(Opcode: PPCISD::ADD_TLS, DL: dl, VT: PtrVT, N1: TPOffset, N2: TGATLS);
3513 }
3514
3515 if (Model == TLSModel::GeneralDynamic) {
3516 if (Subtarget.isUsingPCRelativeCalls()) {
3517 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3518 TargetFlags: PPCII::MO_GOT_TLSGD_PCREL_FLAG);
3519 return DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3520 }
3521
3522 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3523 SDValue GOTPtr;
3524 if (is64bit) {
3525 setUsesTOCBasePtr(DAG);
3526 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3527 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSGD_HA, DL: dl, VT: PtrVT,
3528 N1: GOTReg, N2: TGA);
3529 } else {
3530 if (picLevel == PICLevel::SmallPIC)
3531 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3532 else
3533 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3534 }
3535 return DAG.getNode(Opcode: PPCISD::ADDI_TLSGD_L_ADDR, DL: dl, VT: PtrVT,
3536 N1: GOTPtr, N2: TGA, N3: TGA);
3537 }
3538
3539 if (Model == TLSModel::LocalDynamic) {
3540 if (Subtarget.isUsingPCRelativeCalls()) {
3541 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0,
3542 TargetFlags: PPCII::MO_GOT_TLSLD_PCREL_FLAG);
3543 SDValue MatPCRel =
3544 DAG.getNode(Opcode: PPCISD::TLS_DYNAMIC_MAT_PCREL_ADDR, DL: dl, VT: PtrVT, Operand: TGA);
3545 return DAG.getNode(Opcode: PPCISD::PADDI_DTPREL, DL: dl, VT: PtrVT, N1: MatPCRel, N2: TGA);
3546 }
3547
3548 SDValue TGA = DAG.getTargetGlobalAddress(GV, DL: dl, VT: PtrVT, offset: 0, TargetFlags: 0);
3549 SDValue GOTPtr;
3550 if (is64bit) {
3551 setUsesTOCBasePtr(DAG);
3552 SDValue GOTReg = DAG.getRegister(Reg: PPC::X2, VT: MVT::i64);
3553 GOTPtr = DAG.getNode(Opcode: PPCISD::ADDIS_TLSLD_HA, DL: dl, VT: PtrVT,
3554 N1: GOTReg, N2: TGA);
3555 } else {
3556 if (picLevel == PICLevel::SmallPIC)
3557 GOTPtr = DAG.getNode(Opcode: PPCISD::GlobalBaseReg, DL: dl, VT: PtrVT);
3558 else
3559 GOTPtr = DAG.getNode(Opcode: PPCISD::PPC32_PICGOT, DL: dl, VT: PtrVT);
3560 }
3561 SDValue TLSAddr = DAG.getNode(Opcode: PPCISD::ADDI_TLSLD_L_ADDR, DL: dl,
3562 VT: PtrVT, N1: GOTPtr, N2: TGA, N3: TGA);
3563 SDValue DtvOffsetHi = DAG.getNode(Opcode: PPCISD::ADDIS_DTPREL_HA, DL: dl,
3564 VT: PtrVT, N1: TLSAddr, N2: TGA);
3565 return DAG.getNode(Opcode: PPCISD::ADDI_DTPREL_L, DL: dl, VT: PtrVT, N1: DtvOffsetHi, N2: TGA);
3566 }
3567
3568 llvm_unreachable("Unknown TLS model!");
3569}
3570
3571SDValue PPCTargetLowering::LowerGlobalAddress(SDValue Op,
3572 SelectionDAG &DAG) const {
3573 EVT PtrVT = Op.getValueType();
3574 GlobalAddressSDNode *GSDN = cast<GlobalAddressSDNode>(Val&: Op);
3575 SDLoc DL(GSDN);
3576 const GlobalValue *GV = GSDN->getGlobal();
3577
3578 // 64-bit SVR4 ABI & AIX ABI code is always position-independent.
3579 // The actual address of the GlobalValue is stored in the TOC.
3580 if (Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) {
3581 if (Subtarget.isUsingPCRelativeCalls()) {
3582 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
3583 if (isAccessedAsGotIndirect(N: Op)) {
3584 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3585 TargetFlags: PPCII::MO_GOT_PCREL_FLAG);
3586 SDValue MatPCRel = DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3587 SDValue Load = DAG.getLoad(VT: MVT::i64, dl: DL, Chain: DAG.getEntryNode(), Ptr: MatPCRel,
3588 PtrInfo: MachinePointerInfo());
3589 return Load;
3590 } else {
3591 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: GSDN->getOffset(),
3592 TargetFlags: PPCII::MO_PCREL_FLAG);
3593 return DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: Ty, Operand: GA);
3594 }
3595 }
3596 setUsesTOCBasePtr(DAG);
3597 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset());
3598 return getTOCEntry(DAG, dl: DL, GA);
3599 }
3600
3601 unsigned MOHiFlag, MOLoFlag;
3602 bool IsPIC = isPositionIndependent();
3603 getLabelAccessInfo(IsPIC, Subtarget, HiOpFlags&: MOHiFlag, LoOpFlags&: MOLoFlag, GV);
3604
3605 if (IsPIC && Subtarget.isSVR4ABI()) {
3606 SDValue GA = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT,
3607 offset: GSDN->getOffset(),
3608 TargetFlags: PPCII::MO_PIC_FLAG);
3609 return getTOCEntry(DAG, dl: DL, GA);
3610 }
3611
3612 SDValue GAHi =
3613 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOHiFlag);
3614 SDValue GALo =
3615 DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: GSDN->getOffset(), TargetFlags: MOLoFlag);
3616
3617 return LowerLabelRef(HiPart: GAHi, LoPart: GALo, isPIC: IsPIC, DAG);
3618}
3619
3620SDValue PPCTargetLowering::LowerSETCC(SDValue Op, SelectionDAG &DAG) const {
3621 bool IsStrict = Op->isStrictFPOpcode();
3622 const SDNodeFlags Flags = Op.getNode()->getFlags();
3623 ISD::CondCode CC = Op.getOperandCondCode(i: IsStrict ? 3 : 2);
3624 SDValue LHS = Op.getOperand(i: IsStrict ? 1 : 0);
3625 SDValue RHS = Op.getOperand(i: IsStrict ? 2 : 1);
3626 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
3627 EVT LHSVT = LHS.getValueType();
3628 SDLoc dl(Op);
3629
3630 // Soften the setcc with libcall if it is fp128 or it is SPE and fp32/fp64.
3631 if (LHSVT == MVT::f128 ||
3632 (Subtarget.hasSPE() && (LHSVT == MVT::f32 || LHSVT == MVT::f64) &&
3633 (!Flags.hasNoNaNs() || !Flags.hasNoInfs()))) {
3634 assert(!Subtarget.hasP9Vector() &&
3635 "SETCC for f128 is already legal under Power9!");
3636 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS, Chain,
3637 IsSignaling: Op->getOpcode() == ISD::STRICT_FSETCCS);
3638 if (RHS.getNode())
3639 LHS = DAG.getNode(Opcode: ISD::SETCC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS,
3640 N3: DAG.getCondCode(Cond: CC));
3641 if (IsStrict)
3642 return DAG.getMergeValues(Ops: {LHS, Chain}, dl);
3643 return LHS;
3644 } else if (LHSVT == MVT::f32 || LHSVT == MVT::f64) {
3645 return Op;
3646 }
3647
3648 assert(!IsStrict && "Don't know how to handle STRICT_FSETCC!");
3649
3650 if (Op.getValueType() == MVT::v2i64) {
3651 // When the operands themselves are v2i64 values, we need to do something
3652 // special because VSX has no underlying comparison operations for these.
3653 if (LHS.getValueType() == MVT::v2i64) {
3654 // Equality can be handled by casting to the legal type for Altivec
3655 // comparisons, everything else needs to be expanded.
3656 if (CC != ISD::SETEQ && CC != ISD::SETNE)
3657 return SDValue();
3658 SDValue SetCC32 = DAG.getSetCC(
3659 DL: dl, VT: MVT::v4i32, LHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: LHS),
3660 RHS: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: RHS), Cond: CC);
3661 int ShuffV[] = {1, 0, 3, 2};
3662 SDValue Shuff =
3663 DAG.getVectorShuffle(VT: MVT::v4i32, dl, N1: SetCC32, N2: SetCC32, Mask: ShuffV);
3664 return DAG.getBitcast(VT: MVT::v2i64,
3665 V: DAG.getNode(Opcode: CC == ISD::SETEQ ? ISD::AND : ISD::OR,
3666 DL: dl, VT: MVT::v4i32, N1: Shuff, N2: SetCC32));
3667 }
3668
3669 // We handle most of these in the usual way.
3670 return Op;
3671 }
3672
3673 // If we're comparing for equality to zero, expose the fact that this is
3674 // implemented as a ctlz/srl pair on ppc, so that the dag combiner can
3675 // fold the new nodes.
3676 if (SDValue V = lowerCmpEqZeroToCtlzSrl(Op, DAG))
3677 return V;
3678
3679 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: RHS)) {
3680 // Leave comparisons against 0 and -1 alone for now, since they're usually
3681 // optimized. FIXME: revisit this when we can custom lower all setcc
3682 // optimizations.
3683 if (C->isAllOnes() || C->isZero())
3684 return SDValue();
3685 }
3686
3687 // If we have an integer seteq/setne, turn it into a compare against zero
3688 // by xor'ing the rhs with the lhs, which is faster than setting a
3689 // condition register, reading it back out, and masking the correct bit. The
3690 // normal approach here uses sub to do this instead of xor. Using xor exposes
3691 // the result to other bit-twiddling opportunities.
3692 if (LHSVT.isInteger() && (CC == ISD::SETEQ || CC == ISD::SETNE)) {
3693 EVT VT = Op.getValueType();
3694 SDValue Sub = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: LHSVT, N1: LHS, N2: RHS);
3695 return DAG.getSetCC(DL: dl, VT, LHS: Sub, RHS: DAG.getConstant(Val: 0, DL: dl, VT: LHSVT), Cond: CC);
3696 }
3697 return SDValue();
3698}
3699
3700SDValue PPCTargetLowering::LowerBR_CC(SDValue Op, SelectionDAG &DAG) const {
3701 const SDNodeFlags Flags = Op->getFlags();
3702 SDValue Chain = Op.getOperand(i: 0);
3703 ISD::CondCode CC = Op.getOperandCondCode(i: 1);
3704 SDValue LHS = Op.getOperand(i: 2);
3705 SDValue RHS = Op.getOperand(i: 3);
3706 SDValue Dest = Op.getOperand(i: 4);
3707 EVT LHSVT = LHS.getValueType();
3708 SDLoc dl(Op);
3709
3710 if ((LHSVT == MVT::f32 || LHSVT == MVT::f64) && Flags.hasNoNaNs() &&
3711 Flags.hasNoInfs())
3712 return Op;
3713
3714 softenSetCCOperands(DAG, VT: LHSVT, NewLHS&: LHS, NewRHS&: RHS, CCCode&: CC, DL: dl, OldLHS: LHS, OldRHS: RHS);
3715
3716 // If softenSetCCOperands returned a scalar, we need to compare the result
3717 // against zero to select between true and false values.
3718 if (!RHS) {
3719 RHS = DAG.getConstant(Val: 0, DL: dl, VT: LHS.getValueType());
3720 CC = ISD::SETNE;
3721 }
3722
3723 return DAG.getNode(Opcode: ISD::BR_CC, DL: dl, VT: Op.getValueType(), N1: Chain,
3724 N2: DAG.getCondCode(Cond: CC), N3: LHS, N4: RHS, N5: Dest);
3725}
3726
3727SDValue PPCTargetLowering::LowerVAARG(SDValue Op, SelectionDAG &DAG) const {
3728 SDNode *Node = Op.getNode();
3729 EVT VT = Node->getValueType(ResNo: 0);
3730 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3731 SDValue InChain = Node->getOperand(Num: 0);
3732 SDValue VAListPtr = Node->getOperand(Num: 1);
3733 const Value *SV = cast<SrcValueSDNode>(Val: Node->getOperand(Num: 2))->getValue();
3734 SDLoc dl(Node);
3735
3736 assert(!Subtarget.isPPC64() && "LowerVAARG is PPC32 only");
3737
3738 // gpr_index
3739 SDValue GprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3740 Ptr: VAListPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3741 InChain = GprIndex.getValue(R: 1);
3742
3743 if (VT == MVT::i64) {
3744 // Check if GprIndex is even
3745 SDValue GprAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: GprIndex,
3746 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3747 SDValue CC64 = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: GprAnd,
3748 RHS: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32), Cond: ISD::SETNE);
3749 SDValue GprIndexPlusOne = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: GprIndex,
3750 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3751 // Align GprIndex to be even if it isn't
3752 GprIndex = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC64, N2: GprIndexPlusOne,
3753 N3: GprIndex);
3754 }
3755
3756 // fpr index is 1 byte after gpr
3757 SDValue FprPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3758 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
3759
3760 // fpr
3761 SDValue FprIndex = DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: MVT::i32, Chain: InChain,
3762 Ptr: FprPtr, PtrInfo: MachinePointerInfo(SV), MemVT: MVT::i8);
3763 InChain = FprIndex.getValue(R: 1);
3764
3765 SDValue RegSaveAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3766 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
3767
3768 SDValue OverflowAreaPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: VAListPtr,
3769 N2: DAG.getConstant(Val: 4, DL: dl, VT: MVT::i32));
3770
3771 // areas
3772 SDValue OverflowArea =
3773 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: OverflowAreaPtr, PtrInfo: MachinePointerInfo());
3774 InChain = OverflowArea.getValue(R: 1);
3775
3776 SDValue RegSaveArea =
3777 DAG.getLoad(VT: MVT::i32, dl, Chain: InChain, Ptr: RegSaveAreaPtr, PtrInfo: MachinePointerInfo());
3778 InChain = RegSaveArea.getValue(R: 1);
3779
3780 // select overflow_area if index > 8
3781 SDValue CC = DAG.getSetCC(DL: dl, VT: MVT::i32, LHS: VT.isInteger() ? GprIndex : FprIndex,
3782 RHS: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32), Cond: ISD::SETLT);
3783
3784 // adjustment constant gpr_index * 4/8
3785 SDValue RegConstant = DAG.getNode(Opcode: ISD::MUL, DL: dl, VT: MVT::i32,
3786 N1: VT.isInteger() ? GprIndex : FprIndex,
3787 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8, DL: dl,
3788 VT: MVT::i32));
3789
3790 // OurReg = RegSaveArea + RegConstant
3791 SDValue OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: RegSaveArea,
3792 N2: RegConstant);
3793
3794 // Floating types are 32 bytes into RegSaveArea
3795 if (VT.isFloatingPoint())
3796 OurReg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OurReg,
3797 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
3798
3799 // increase {f,g}pr_index by 1 (or 2 if VT is i64)
3800 SDValue IndexPlus1 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32,
3801 N1: VT.isInteger() ? GprIndex : FprIndex,
3802 N2: DAG.getConstant(Val: VT == MVT::i64 ? 2 : 1, DL: dl,
3803 VT: MVT::i32));
3804
3805 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: IndexPlus1,
3806 Ptr: VT.isInteger() ? VAListPtr : FprPtr,
3807 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
3808
3809 // determine if we should load from reg_save_area or overflow_area
3810 SDValue Result = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: PtrVT, N1: CC, N2: OurReg, N3: OverflowArea);
3811
3812 // increase overflow_area by 4/8 if gpr/fpr > 8
3813 SDValue OverflowAreaPlusN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: OverflowArea,
3814 N2: DAG.getConstant(Val: VT.isInteger() ? 4 : 8,
3815 DL: dl, VT: MVT::i32));
3816
3817 OverflowArea = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i32, N1: CC, N2: OverflowArea,
3818 N3: OverflowAreaPlusN);
3819
3820 InChain = DAG.getTruncStore(Chain: InChain, dl, Val: OverflowArea, Ptr: OverflowAreaPtr,
3821 PtrInfo: MachinePointerInfo(), SVT: MVT::i32);
3822
3823 return DAG.getLoad(VT, dl, Chain: InChain, Ptr: Result, PtrInfo: MachinePointerInfo());
3824}
3825
3826SDValue PPCTargetLowering::LowerVACOPY(SDValue Op, SelectionDAG &DAG) const {
3827 assert(!Subtarget.isPPC64() && "LowerVACOPY is PPC32 only");
3828
3829 // We have to copy the entire va_list struct:
3830 // 2*sizeof(char) + 2 Byte alignment + 2*sizeof(char*) = 12 Byte
3831 return DAG.getMemcpy(Chain: Op.getOperand(i: 0), dl: Op, Dst: Op.getOperand(i: 1), Src: Op.getOperand(i: 2),
3832 Size: DAG.getConstant(Val: 12, DL: SDLoc(Op), VT: MVT::i32), DstAlign: Align(8),
3833 SrcAlign: Align(8), isVol: false, AlwaysInline: true, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
3834 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
3835}
3836
3837SDValue PPCTargetLowering::LowerADJUST_TRAMPOLINE(SDValue Op,
3838 SelectionDAG &DAG) const {
3839 return Op.getOperand(i: 0);
3840}
3841
3842SDValue PPCTargetLowering::LowerINLINEASM(SDValue Op, SelectionDAG &DAG) const {
3843 MachineFunction &MF = DAG.getMachineFunction();
3844 PPCFunctionInfo &MFI = *MF.getInfo<PPCFunctionInfo>();
3845
3846 assert((Op.getOpcode() == ISD::INLINEASM ||
3847 Op.getOpcode() == ISD::INLINEASM_BR) &&
3848 "Expecting Inline ASM node.");
3849
3850 // If an LR store is already known to be required then there is not point in
3851 // checking this ASM as well.
3852 if (MFI.isLRStoreRequired())
3853 return Op;
3854
3855 // Inline ASM nodes have an optional last operand that is an incoming Flag of
3856 // type MVT::Glue. We want to ignore this last operand if that is the case.
3857 unsigned NumOps = Op.getNumOperands();
3858 if (Op.getOperand(i: NumOps - 1).getValueType() == MVT::Glue)
3859 --NumOps;
3860
3861 // Check all operands that may contain the LR.
3862 for (unsigned i = InlineAsm::Op_FirstOperand; i != NumOps;) {
3863 const InlineAsm::Flag Flags(Op.getConstantOperandVal(i));
3864 unsigned NumVals = Flags.getNumOperandRegisters();
3865 ++i; // Skip the ID value.
3866
3867 switch (Flags.getKind()) {
3868 default:
3869 llvm_unreachable("Bad flags!");
3870 case InlineAsm::Kind::RegUse:
3871 case InlineAsm::Kind::Imm:
3872 case InlineAsm::Kind::Mem:
3873 i += NumVals;
3874 break;
3875 case InlineAsm::Kind::Clobber:
3876 case InlineAsm::Kind::RegDef:
3877 case InlineAsm::Kind::RegDefEarlyClobber: {
3878 for (; NumVals; --NumVals, ++i) {
3879 Register Reg = cast<RegisterSDNode>(Val: Op.getOperand(i))->getReg();
3880 if (Reg != PPC::LR && Reg != PPC::LR8)
3881 continue;
3882 MFI.setLRStoreRequired();
3883 return Op;
3884 }
3885 break;
3886 }
3887 }
3888 }
3889
3890 return Op;
3891}
3892
3893SDValue PPCTargetLowering::LowerINIT_TRAMPOLINE(SDValue Op,
3894 SelectionDAG &DAG) const {
3895 SDValue Chain = Op.getOperand(i: 0);
3896 SDValue Trmp = Op.getOperand(i: 1); // trampoline
3897 SDValue FPtr = Op.getOperand(i: 2); // nested function
3898 SDValue Nest = Op.getOperand(i: 3); // 'nest' parameter value
3899 SDLoc dl(Op);
3900
3901 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
3902
3903 if (Subtarget.isAIXABI()) {
3904 // On AIX we create a trampoline descriptor by combining the
3905 // entry point and TOC from the global descriptor (FPtr) with the
3906 // nest argument as the environment pointer.
3907 uint64_t PointerSize = Subtarget.isPPC64() ? 8 : 4;
3908 MaybeAlign PointerAlign(PointerSize);
3909 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
3910 ? (MachineMemOperand::MODereferenceable |
3911 MachineMemOperand::MOInvariant)
3912 : MachineMemOperand::MONone;
3913
3914 uint64_t TOCPointerOffset = 1 * PointerSize;
3915 uint64_t EnvPointerOffset = 2 * PointerSize;
3916 SDValue SDTOCPtrOffset = DAG.getConstant(Val: TOCPointerOffset, DL: dl, VT: PtrVT);
3917 SDValue SDEnvPtrOffset = DAG.getConstant(Val: EnvPointerOffset, DL: dl, VT: PtrVT);
3918
3919 const Value *TrampolineAddr =
3920 cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
3921 const Function *Func =
3922 cast<Function>(Val: cast<SrcValueSDNode>(Val: Op.getOperand(i: 5))->getValue());
3923
3924 SDValue OutChains[3];
3925
3926 // Copy the entry point address from the global descriptor to the
3927 // trampoline buffer.
3928 SDValue LoadEntryPoint =
3929 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: FPtr, PtrInfo: MachinePointerInfo(Func, 0),
3930 Alignment: PointerAlign, MMOFlags);
3931 SDValue EPLoadChain = LoadEntryPoint.getValue(R: 1);
3932 OutChains[0] = DAG.getStore(Chain: EPLoadChain, dl, Val: LoadEntryPoint, Ptr: Trmp,
3933 PtrInfo: MachinePointerInfo(TrampolineAddr, 0));
3934
3935 // Copy the TOC pointer from the global descriptor to the trampoline
3936 // buffer.
3937 SDValue TOCFromDescriptorPtr =
3938 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FPtr, N2: SDTOCPtrOffset);
3939 SDValue TOCReg = DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: TOCFromDescriptorPtr,
3940 PtrInfo: MachinePointerInfo(Func, TOCPointerOffset),
3941 Alignment: PointerAlign, MMOFlags);
3942 SDValue TrampolineTOCPointer =
3943 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDTOCPtrOffset);
3944 SDValue TOCLoadChain = TOCReg.getValue(R: 1);
3945 OutChains[1] =
3946 DAG.getStore(Chain: TOCLoadChain, dl, Val: TOCReg, Ptr: TrampolineTOCPointer,
3947 PtrInfo: MachinePointerInfo(TrampolineAddr, TOCPointerOffset));
3948
3949 // Store the nest argument into the environment pointer in the trampoline
3950 // buffer.
3951 SDValue EnvPointer = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Trmp, N2: SDEnvPtrOffset);
3952 OutChains[2] =
3953 DAG.getStore(Chain, dl, Val: Nest, Ptr: EnvPointer,
3954 PtrInfo: MachinePointerInfo(TrampolineAddr, EnvPointerOffset));
3955
3956 SDValue TokenFactor =
3957 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
3958 return TokenFactor;
3959 }
3960
3961 bool isPPC64 = (PtrVT == MVT::i64);
3962 Type *IntPtrTy = DAG.getDataLayout().getIntPtrType(C&: *DAG.getContext());
3963
3964 TargetLowering::ArgListTy Args;
3965 Args.emplace_back(args&: Trmp, args&: IntPtrTy);
3966 // TrampSize == (isPPC64 ? 48 : 40);
3967 Args.emplace_back(
3968 args: DAG.getConstant(Val: isPPC64 ? 48 : 40, DL: dl, VT: Subtarget.getScalarIntVT()),
3969 args&: IntPtrTy);
3970 Args.emplace_back(args&: FPtr, args&: IntPtrTy);
3971 Args.emplace_back(args&: Nest, args&: IntPtrTy);
3972
3973 // Lower to a call to __trampoline_setup(Trmp, TrampSize, FPtr, ctx_reg)
3974 TargetLowering::CallLoweringInfo CLI(DAG);
3975 CLI.setDebugLoc(dl).setChain(Chain).setLibCallee(
3976 CC: CallingConv::C, ResultType: Type::getVoidTy(C&: *DAG.getContext()),
3977 Target: DAG.getExternalSymbol(Sym: "__trampoline_setup", VT: PtrVT), ArgsList: std::move(Args));
3978
3979 std::pair<SDValue, SDValue> CallResult = LowerCallTo(CLI);
3980 return CallResult.second;
3981}
3982
3983SDValue PPCTargetLowering::LowerVASTART(SDValue Op, SelectionDAG &DAG) const {
3984 MachineFunction &MF = DAG.getMachineFunction();
3985 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
3986 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
3987
3988 SDLoc dl(Op);
3989
3990 if (Subtarget.isPPC64() || Subtarget.isAIXABI()) {
3991 // vastart just stores the address of the VarArgsFrameIndex slot into the
3992 // memory location argument.
3993 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
3994 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
3995 return DAG.getStore(Chain: Op.getOperand(i: 0), dl, Val: FR, Ptr: Op.getOperand(i: 1),
3996 PtrInfo: MachinePointerInfo(SV));
3997 }
3998
3999 // For the 32-bit SVR4 ABI we follow the layout of the va_list struct.
4000 // We suppose the given va_list is already allocated.
4001 //
4002 // typedef struct {
4003 // char gpr; /* index into the array of 8 GPRs
4004 // * stored in the register save area
4005 // * gpr=0 corresponds to r3,
4006 // * gpr=1 to r4, etc.
4007 // */
4008 // char fpr; /* index into the array of 8 FPRs
4009 // * stored in the register save area
4010 // * fpr=0 corresponds to f1,
4011 // * fpr=1 to f2, etc.
4012 // */
4013 // char *overflow_arg_area;
4014 // /* location on stack that holds
4015 // * the next overflow argument
4016 // */
4017 // char *reg_save_area;
4018 // /* where r3:r10 and f1:f8 (if saved)
4019 // * are stored
4020 // */
4021 // } va_list[1];
4022
4023 SDValue ArgGPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumGPR(), DL: dl, VT: MVT::i32);
4024 SDValue ArgFPR = DAG.getConstant(Val: FuncInfo->getVarArgsNumFPR(), DL: dl, VT: MVT::i32);
4025 SDValue StackOffsetFI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsStackOffset(),
4026 VT: PtrVT);
4027 SDValue FR = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
4028 VT: PtrVT);
4029
4030 uint64_t FrameOffset = PtrVT.getSizeInBits()/8;
4031 SDValue ConstFrameOffset = DAG.getConstant(Val: FrameOffset, DL: dl, VT: PtrVT);
4032
4033 uint64_t StackOffset = PtrVT.getSizeInBits()/8 - 1;
4034 SDValue ConstStackOffset = DAG.getConstant(Val: StackOffset, DL: dl, VT: PtrVT);
4035
4036 uint64_t FPROffset = 1;
4037 SDValue ConstFPROffset = DAG.getConstant(Val: FPROffset, DL: dl, VT: PtrVT);
4038
4039 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
4040
4041 // Store first byte : number of int regs
4042 SDValue firstStore =
4043 DAG.getTruncStore(Chain: Op.getOperand(i: 0), dl, Val: ArgGPR, Ptr: Op.getOperand(i: 1),
4044 PtrInfo: MachinePointerInfo(SV), SVT: MVT::i8);
4045 uint64_t nextOffset = FPROffset;
4046 SDValue nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Op.getOperand(i: 1),
4047 N2: ConstFPROffset);
4048
4049 // Store second byte : number of float regs
4050 SDValue secondStore =
4051 DAG.getTruncStore(Chain: firstStore, dl, Val: ArgFPR, Ptr: nextPtr,
4052 PtrInfo: MachinePointerInfo(SV, nextOffset), SVT: MVT::i8);
4053 nextOffset += StackOffset;
4054 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstStackOffset);
4055
4056 // Store second word : arguments given on stack
4057 SDValue thirdStore = DAG.getStore(Chain: secondStore, dl, Val: StackOffsetFI, Ptr: nextPtr,
4058 PtrInfo: MachinePointerInfo(SV, nextOffset));
4059 nextOffset += FrameOffset;
4060 nextPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: nextPtr, N2: ConstFrameOffset);
4061
4062 // Store third word : arguments given in registers
4063 return DAG.getStore(Chain: thirdStore, dl, Val: FR, Ptr: nextPtr,
4064 PtrInfo: MachinePointerInfo(SV, nextOffset));
4065}
4066
4067/// FPR - The set of FP registers that should be allocated for arguments
4068/// on Darwin and AIX.
4069static const MCPhysReg FPR[] = {PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5,
4070 PPC::F6, PPC::F7, PPC::F8, PPC::F9, PPC::F10,
4071 PPC::F11, PPC::F12, PPC::F13};
4072
4073/// CalculateStackSlotSize - Calculates the size reserved for this argument on
4074/// the stack.
4075static unsigned CalculateStackSlotSize(EVT ArgVT, ISD::ArgFlagsTy Flags,
4076 unsigned PtrByteSize) {
4077 unsigned ArgSize = ArgVT.getStoreSize();
4078 if (Flags.isByVal())
4079 ArgSize = Flags.getByValSize();
4080
4081 // Round up to multiples of the pointer size, except for array members,
4082 // which are always packed.
4083 if (!Flags.isInConsecutiveRegs())
4084 ArgSize = ((ArgSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4085
4086 return ArgSize;
4087}
4088
4089/// CalculateStackSlotAlignment - Calculates the alignment of this argument
4090/// on the stack.
4091static Align CalculateStackSlotAlignment(EVT ArgVT, EVT OrigVT,
4092 ISD::ArgFlagsTy Flags,
4093 unsigned PtrByteSize) {
4094 Align Alignment(PtrByteSize);
4095
4096 // Altivec parameters are padded to a 16 byte boundary.
4097 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4098 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4099 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4100 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4101 Alignment = Align(16);
4102
4103 // ByVal parameters are aligned as requested.
4104 if (Flags.isByVal()) {
4105 auto BVAlign = Flags.getNonZeroByValAlign();
4106 if (BVAlign > PtrByteSize) {
4107 if (BVAlign.value() % PtrByteSize != 0)
4108 llvm_unreachable(
4109 "ByVal alignment is not a multiple of the pointer size");
4110
4111 Alignment = BVAlign;
4112 }
4113 }
4114
4115 // Array members are always packed to their original alignment.
4116 if (Flags.isInConsecutiveRegs()) {
4117 // If the array member was split into multiple registers, the first
4118 // needs to be aligned to the size of the full type. (Except for
4119 // ppcf128, which is only aligned as its f64 components.)
4120 if (Flags.isSplit() && OrigVT != MVT::ppcf128)
4121 Alignment = Align(OrigVT.getStoreSize());
4122 else
4123 Alignment = Align(ArgVT.getStoreSize());
4124 }
4125
4126 return Alignment;
4127}
4128
4129/// CalculateStackSlotUsed - Return whether this argument will use its
4130/// stack slot (instead of being passed in registers). ArgOffset,
4131/// AvailableFPRs, and AvailableVRs must hold the current argument
4132/// position, and will be updated to account for this argument.
4133static bool CalculateStackSlotUsed(EVT ArgVT, EVT OrigVT, ISD::ArgFlagsTy Flags,
4134 unsigned PtrByteSize, unsigned LinkageSize,
4135 unsigned ParamAreaSize, unsigned &ArgOffset,
4136 unsigned &AvailableFPRs,
4137 unsigned &AvailableVRs) {
4138 bool UseMemory = false;
4139
4140 // Respect alignment of argument on the stack.
4141 Align Alignment =
4142 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
4143 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4144 // If there's no space left in the argument save area, we must
4145 // use memory (this check also catches zero-sized arguments).
4146 if (ArgOffset >= LinkageSize + ParamAreaSize)
4147 UseMemory = true;
4148
4149 // Allocate argument on the stack.
4150 ArgOffset += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
4151 if (Flags.isInConsecutiveRegsLast())
4152 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4153 // If we overran the argument save area, we must use memory
4154 // (this check catches arguments passed partially in memory)
4155 if (ArgOffset > LinkageSize + ParamAreaSize)
4156 UseMemory = true;
4157
4158 // However, if the argument is actually passed in an FPR or a VR,
4159 // we don't use memory after all.
4160 if (!Flags.isByVal()) {
4161 if (ArgVT == MVT::f32 || ArgVT == MVT::f64)
4162 if (AvailableFPRs > 0) {
4163 --AvailableFPRs;
4164 return false;
4165 }
4166 if (ArgVT == MVT::v4f32 || ArgVT == MVT::v4i32 ||
4167 ArgVT == MVT::v8i16 || ArgVT == MVT::v16i8 ||
4168 ArgVT == MVT::v2f64 || ArgVT == MVT::v2i64 ||
4169 ArgVT == MVT::v1i128 || ArgVT == MVT::f128)
4170 if (AvailableVRs > 0) {
4171 --AvailableVRs;
4172 return false;
4173 }
4174 }
4175
4176 return UseMemory;
4177}
4178
4179/// EnsureStackAlignment - Round stack frame size up from NumBytes to
4180/// ensure minimum alignment required for target.
4181static unsigned EnsureStackAlignment(const PPCFrameLowering *Lowering,
4182 unsigned NumBytes) {
4183 return alignTo(Size: NumBytes, A: Lowering->getStackAlign());
4184}
4185
4186SDValue PPCTargetLowering::LowerFormalArguments(
4187 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4188 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4189 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4190 if (Subtarget.isAIXABI())
4191 return LowerFormalArguments_AIX(Chain, CallConv, isVarArg, Ins, dl, DAG,
4192 InVals);
4193 if (Subtarget.is64BitELFABI())
4194 return LowerFormalArguments_64SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4195 InVals);
4196 assert(Subtarget.is32BitELFABI());
4197 return LowerFormalArguments_32SVR4(Chain, CallConv, isVarArg, Ins, dl, DAG,
4198 InVals);
4199}
4200
4201SDValue PPCTargetLowering::LowerFormalArguments_32SVR4(
4202 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4203 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4204 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4205
4206 // 32-bit SVR4 ABI Stack Frame Layout:
4207 // +-----------------------------------+
4208 // +--> | Back chain |
4209 // | +-----------------------------------+
4210 // | | Floating-point register save area |
4211 // | +-----------------------------------+
4212 // | | General register save area |
4213 // | +-----------------------------------+
4214 // | | CR save word |
4215 // | +-----------------------------------+
4216 // | | VRSAVE save word |
4217 // | +-----------------------------------+
4218 // | | Alignment padding |
4219 // | +-----------------------------------+
4220 // | | Vector register save area |
4221 // | +-----------------------------------+
4222 // | | Local variable space |
4223 // | +-----------------------------------+
4224 // | | Parameter list area |
4225 // | +-----------------------------------+
4226 // | | LR save word |
4227 // | +-----------------------------------+
4228 // SP--> +--- | Back chain |
4229 // +-----------------------------------+
4230 //
4231 // Specifications:
4232 // System V Application Binary Interface PowerPC Processor Supplement
4233 // AltiVec Technology Programming Interface Manual
4234
4235 MachineFunction &MF = DAG.getMachineFunction();
4236 MachineFrameInfo &MFI = MF.getFrameInfo();
4237 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4238
4239 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4240 // Potential tail calls could cause overwriting of argument stack slots.
4241 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4242 (CallConv == CallingConv::Fast));
4243 const Align PtrAlign(4);
4244
4245 // Assign locations to all of the incoming arguments.
4246 SmallVector<CCValAssign, 16> ArgLocs;
4247 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), ArgLocs,
4248 *DAG.getContext());
4249
4250 // Reserve space for the linkage area on the stack.
4251 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4252 CCInfo.AllocateStack(Size: LinkageSize, Alignment: PtrAlign);
4253 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4);
4254
4255 for (unsigned i = 0, e = ArgLocs.size(); i != e; ++i) {
4256 CCValAssign &VA = ArgLocs[i];
4257
4258 // Arguments stored in registers.
4259 if (VA.isRegLoc()) {
4260 const TargetRegisterClass *RC;
4261 EVT ValVT = VA.getValVT();
4262
4263 switch (ValVT.getSimpleVT().SimpleTy) {
4264 default:
4265 llvm_unreachable("ValVT not supported by formal arguments Lowering");
4266 case MVT::i1:
4267 case MVT::i32:
4268 RC = &PPC::GPRCRegClass;
4269 break;
4270 case MVT::f32:
4271 if (Subtarget.hasP8Vector())
4272 RC = &PPC::VSSRCRegClass;
4273 else if (Subtarget.hasSPE())
4274 RC = &PPC::GPRCRegClass;
4275 else
4276 RC = &PPC::F4RCRegClass;
4277 break;
4278 case MVT::f64:
4279 if (Subtarget.hasVSX())
4280 RC = &PPC::VSFRCRegClass;
4281 else if (Subtarget.hasSPE())
4282 // SPE passes doubles in GPR pairs.
4283 RC = &PPC::GPRCRegClass;
4284 else
4285 RC = &PPC::F8RCRegClass;
4286 break;
4287 case MVT::v16i8:
4288 case MVT::v8i16:
4289 case MVT::v4i32:
4290 case MVT::v4f32:
4291 case MVT::v2f64:
4292 case MVT::v2i64:
4293 case MVT::f128:
4294 RC = &PPC::VRRCRegClass;
4295 break;
4296 }
4297
4298 SDValue ArgValue;
4299 // Transform the arguments stored in physical registers into
4300 // virtual ones.
4301 if (VA.getLocVT() == MVT::f64 && Subtarget.hasSPE()) {
4302 assert(i + 1 < e && "No second half of double precision argument");
4303 Register RegLo = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4304 Register RegHi = MF.addLiveIn(PReg: ArgLocs[++i].getLocReg(), RC);
4305 SDValue ArgValueLo = DAG.getCopyFromReg(Chain, dl, Reg: RegLo, VT: MVT::i32);
4306 SDValue ArgValueHi = DAG.getCopyFromReg(Chain, dl, Reg: RegHi, VT: MVT::i32);
4307 if (!Subtarget.isLittleEndian())
4308 std::swap (a&: ArgValueLo, b&: ArgValueHi);
4309 ArgValue = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: ArgValueLo,
4310 N2: ArgValueHi);
4311 } else {
4312 Register Reg = MF.addLiveIn(PReg: VA.getLocReg(), RC);
4313 ArgValue = DAG.getCopyFromReg(Chain, dl, Reg,
4314 VT: ValVT == MVT::i1 ? MVT::i32 : ValVT);
4315 if (ValVT == MVT::i1)
4316 ArgValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: ArgValue);
4317 }
4318
4319 InVals.push_back(Elt: ArgValue);
4320 } else {
4321 // Argument stored in memory.
4322 assert(VA.isMemLoc());
4323
4324 // Get the extended size of the argument type in stack
4325 unsigned ArgSize = VA.getLocVT().getStoreSize();
4326 // Get the actual size of the argument type
4327 unsigned ObjSize = VA.getValVT().getStoreSize();
4328 unsigned ArgOffset = VA.getLocMemOffset();
4329 // Stack objects in PPC32 are right justified.
4330 ArgOffset += ArgSize - ObjSize;
4331 int FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: isImmutable);
4332
4333 // Create load nodes to retrieve arguments from the stack.
4334 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4335 InVals.push_back(
4336 Elt: DAG.getLoad(VT: VA.getValVT(), dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo()));
4337 }
4338 }
4339
4340 // Assign locations to all of the incoming aggregate by value arguments.
4341 // Aggregates passed by value are stored in the local variable space of the
4342 // caller's stack frame, right above the parameter list area.
4343 SmallVector<CCValAssign, 16> ByValArgLocs;
4344 CCState CCByValInfo(CallConv, isVarArg, DAG.getMachineFunction(),
4345 ByValArgLocs, *DAG.getContext());
4346
4347 // Reserve stack space for the allocations in CCInfo.
4348 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
4349
4350 CCByValInfo.AnalyzeFormalArguments(Ins, Fn: CC_PPC32_SVR4_ByVal);
4351
4352 // Area that is at least reserved in the caller of this function.
4353 unsigned MinReservedArea = CCByValInfo.getStackSize();
4354 MinReservedArea = std::max(a: MinReservedArea, b: LinkageSize);
4355
4356 // Set the size that is at least reserved in caller of this function. Tail
4357 // call optimized function's reserved stack space needs to be aligned so that
4358 // taking the difference between two stack areas will result in an aligned
4359 // stack.
4360 MinReservedArea =
4361 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4362 FuncInfo->setMinReservedArea(MinReservedArea);
4363
4364 SmallVector<SDValue, 8> MemOps;
4365
4366 // If the function takes variable number of arguments, make a frame index for
4367 // the start of the first vararg value... for expansion of llvm.va_start.
4368 if (isVarArg) {
4369 static const MCPhysReg GPArgRegs[] = {
4370 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
4371 PPC::R7, PPC::R8, PPC::R9, PPC::R10,
4372 };
4373 const unsigned NumGPArgRegs = std::size(GPArgRegs);
4374
4375 static const MCPhysReg FPArgRegs[] = {
4376 PPC::F1, PPC::F2, PPC::F3, PPC::F4, PPC::F5, PPC::F6, PPC::F7,
4377 PPC::F8
4378 };
4379 unsigned NumFPArgRegs = std::size(FPArgRegs);
4380
4381 if (useSoftFloat() || hasSPE())
4382 NumFPArgRegs = 0;
4383
4384 FuncInfo->setVarArgsNumGPR(CCInfo.getFirstUnallocated(Regs: GPArgRegs));
4385 FuncInfo->setVarArgsNumFPR(CCInfo.getFirstUnallocated(Regs: FPArgRegs));
4386
4387 // Make room for NumGPArgRegs and NumFPArgRegs.
4388 int Depth = NumGPArgRegs * PtrVT.getSizeInBits()/8 +
4389 NumFPArgRegs * MVT(MVT::f64).getSizeInBits()/8;
4390
4391 FuncInfo->setVarArgsStackOffset(MFI.CreateFixedObject(
4392 Size: PtrVT.getSizeInBits() / 8, SPOffset: CCInfo.getStackSize(), IsImmutable: true));
4393
4394 FuncInfo->setVarArgsFrameIndex(
4395 MFI.CreateStackObject(Size: Depth, Alignment: Align(8), isSpillSlot: false));
4396 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4397
4398 // The fixed integer arguments of a variadic function are stored to the
4399 // VarArgsFrameIndex on the stack so that they may be loaded by
4400 // dereferencing the result of va_next.
4401 for (MCPhysReg GPArgReg : GPArgRegs) {
4402 // Get an existing live-in vreg, or add a new one.
4403 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: GPArgReg);
4404 if (!VReg)
4405 VReg = MF.addLiveIn(PReg: GPArgReg, RC: &PPC::GPRCRegClass);
4406
4407 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4408 SDValue Store =
4409 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4410 MemOps.push_back(Elt: Store);
4411 // Increment the address by four for the next argument to store
4412 SDValue PtrOff = DAG.getConstant(Val: PtrVT.getSizeInBits()/8, DL: dl, VT: PtrVT);
4413 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4414 }
4415
4416 // FIXME 32-bit SVR4: We only need to save FP argument registers if CR bit 6
4417 // is set.
4418 // The double arguments are stored to the VarArgsFrameIndex
4419 // on the stack.
4420 for (unsigned FPRIndex = 0; FPRIndex != NumFPArgRegs; ++FPRIndex) {
4421 // Get an existing live-in vreg, or add a new one.
4422 Register VReg = MF.getRegInfo().getLiveInVirtReg(PReg: FPArgRegs[FPRIndex]);
4423 if (!VReg)
4424 VReg = MF.addLiveIn(PReg: FPArgRegs[FPRIndex], RC: &PPC::F8RCRegClass);
4425
4426 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::f64);
4427 SDValue Store =
4428 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4429 MemOps.push_back(Elt: Store);
4430 // Increment the address by eight for the next argument to store
4431 SDValue PtrOff = DAG.getConstant(Val: MVT(MVT::f64).getSizeInBits()/8, DL: dl,
4432 VT: PtrVT);
4433 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4434 }
4435 }
4436
4437 if (!MemOps.empty())
4438 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4439
4440 return Chain;
4441}
4442
4443// PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4444// value to MVT::i64 and then truncate to the correct register size.
4445SDValue PPCTargetLowering::extendArgForPPC64(ISD::ArgFlagsTy Flags,
4446 EVT ObjectVT, SelectionDAG &DAG,
4447 SDValue ArgVal,
4448 const SDLoc &dl) const {
4449 if (Flags.isSExt())
4450 ArgVal = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: MVT::i64, N1: ArgVal,
4451 N2: DAG.getValueType(ObjectVT));
4452 else if (Flags.isZExt())
4453 ArgVal = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: MVT::i64, N1: ArgVal,
4454 N2: DAG.getValueType(ObjectVT));
4455
4456 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ObjectVT, Operand: ArgVal);
4457}
4458
4459SDValue PPCTargetLowering::LowerFormalArguments_64SVR4(
4460 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
4461 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
4462 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
4463 // TODO: add description of PPC stack frame format, or at least some docs.
4464 //
4465 bool isELFv2ABI = Subtarget.isELFv2ABI();
4466 bool isLittleEndian = Subtarget.isLittleEndian();
4467 MachineFunction &MF = DAG.getMachineFunction();
4468 MachineFrameInfo &MFI = MF.getFrameInfo();
4469 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
4470
4471 assert(!(CallConv == CallingConv::Fast && isVarArg) &&
4472 "fastcc not supported on varargs functions");
4473
4474 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
4475 // Potential tail calls could cause overwriting of argument stack slots.
4476 bool isImmutable = !(getTargetMachine().Options.GuaranteedTailCallOpt &&
4477 (CallConv == CallingConv::Fast));
4478 unsigned PtrByteSize = 8;
4479 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4480
4481 static const MCPhysReg GPR[] = {
4482 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4483 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4484 };
4485 static const MCPhysReg VR[] = {
4486 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4487 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4488 };
4489
4490 const unsigned Num_GPR_Regs = std::size(GPR);
4491 const unsigned Num_FPR_Regs = useSoftFloat() ? 0 : 13;
4492 const unsigned Num_VR_Regs = std::size(VR);
4493
4494 // Do a first pass over the arguments to determine whether the ABI
4495 // guarantees that our caller has allocated the parameter save area
4496 // on its stack frame. In the ELFv1 ABI, this is always the case;
4497 // in the ELFv2 ABI, it is true if this is a vararg function or if
4498 // any parameter is located in a stack slot.
4499
4500 bool HasParameterArea = !isELFv2ABI || isVarArg;
4501 unsigned ParamAreaSize = Num_GPR_Regs * PtrByteSize;
4502 unsigned NumBytes = LinkageSize;
4503 unsigned AvailableFPRs = Num_FPR_Regs;
4504 unsigned AvailableVRs = Num_VR_Regs;
4505 for (const ISD::InputArg &In : Ins) {
4506 if (In.Flags.isNest())
4507 continue;
4508
4509 if (CalculateStackSlotUsed(ArgVT: In.VT, OrigVT: In.ArgVT, Flags: In.Flags, PtrByteSize,
4510 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4511 AvailableFPRs, AvailableVRs))
4512 HasParameterArea = true;
4513 }
4514
4515 // Add DAG nodes to load the arguments or copy them out of registers. On
4516 // entry to a function on PPC, the arguments start after the linkage area,
4517 // although the first ones are often in registers.
4518
4519 unsigned ArgOffset = LinkageSize;
4520 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
4521 SmallVector<SDValue, 8> MemOps;
4522 Function::const_arg_iterator FuncArg = MF.getFunction().arg_begin();
4523 unsigned CurArgIdx = 0;
4524 for (unsigned ArgNo = 0, e = Ins.size(); ArgNo != e; ++ArgNo) {
4525 SDValue ArgVal;
4526 bool needsLoad = false;
4527 EVT ObjectVT = Ins[ArgNo].VT;
4528 EVT OrigVT = Ins[ArgNo].ArgVT;
4529 unsigned ObjSize = ObjectVT.getStoreSize();
4530 unsigned ArgSize = ObjSize;
4531 ISD::ArgFlagsTy Flags = Ins[ArgNo].Flags;
4532 if (Ins[ArgNo].isOrigArg()) {
4533 std::advance(i&: FuncArg, n: Ins[ArgNo].getOrigArgIndex() - CurArgIdx);
4534 CurArgIdx = Ins[ArgNo].getOrigArgIndex();
4535 }
4536 // We re-align the argument offset for each argument, except when using the
4537 // fast calling convention, when we need to make sure we do that only when
4538 // we'll actually use a stack slot.
4539 unsigned CurArgOffset;
4540 Align Alignment;
4541 auto ComputeArgOffset = [&]() {
4542 /* Respect alignment of argument on the stack. */
4543 Alignment =
4544 CalculateStackSlotAlignment(ArgVT: ObjectVT, OrigVT, Flags, PtrByteSize);
4545 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
4546 CurArgOffset = ArgOffset;
4547 };
4548
4549 if (CallConv != CallingConv::Fast) {
4550 ComputeArgOffset();
4551
4552 /* Compute GPR index associated with argument offset. */
4553 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4554 GPR_idx = std::min(a: GPR_idx, b: Num_GPR_Regs);
4555 }
4556
4557 // FIXME the codegen can be much improved in some cases.
4558 // We do not have to keep everything in memory.
4559 if (Flags.isByVal()) {
4560 assert(Ins[ArgNo].isOrigArg() && "Byval arguments cannot be implicit");
4561
4562 if (CallConv == CallingConv::Fast)
4563 ComputeArgOffset();
4564
4565 // ObjSize is the true size, ArgSize rounded up to multiple of registers.
4566 ObjSize = Flags.getByValSize();
4567 ArgSize = ((ObjSize + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4568 // Empty aggregate parameters do not take up registers. Examples:
4569 // struct { } a;
4570 // union { } b;
4571 // int c[0];
4572 // etc. However, we have to provide a place-holder in InVals, so
4573 // pretend we have an 8-byte item at the current address for that
4574 // purpose.
4575 if (!ObjSize) {
4576 int FI = MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: ArgOffset, IsImmutable: true);
4577 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4578 InVals.push_back(Elt: FIN);
4579 continue;
4580 }
4581
4582 // Create a stack object covering all stack doublewords occupied
4583 // by the argument. If the argument is (fully or partially) on
4584 // the stack, or if the argument is fully in registers but the
4585 // caller has allocated the parameter save anyway, we can refer
4586 // directly to the caller's stack frame. Otherwise, create a
4587 // local copy in our own frame.
4588 int FI;
4589 if (HasParameterArea ||
4590 ArgSize + ArgOffset > LinkageSize + Num_GPR_Regs * PtrByteSize)
4591 FI = MFI.CreateFixedObject(Size: ArgSize, SPOffset: ArgOffset, IsImmutable: false, isAliased: true);
4592 else
4593 FI = MFI.CreateStackObject(Size: ArgSize, Alignment, isSpillSlot: false);
4594 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4595
4596 // Handle aggregates smaller than 8 bytes.
4597 if (ObjSize < PtrByteSize) {
4598 // The value of the object is its address, which differs from the
4599 // address of the enclosing doubleword on big-endian systems.
4600 SDValue Arg = FIN;
4601 if (!isLittleEndian) {
4602 SDValue ArgOff = DAG.getConstant(Val: PtrByteSize - ObjSize, DL: dl, VT: PtrVT);
4603 Arg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: ArgOff.getValueType(), N1: Arg, N2: ArgOff);
4604 }
4605 InVals.push_back(Elt: Arg);
4606
4607 if (GPR_idx != Num_GPR_Regs) {
4608 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4609 FuncInfo->addLiveInAttr(VReg, Flags);
4610 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4611 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ObjSize * 8);
4612 SDValue Store =
4613 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Arg,
4614 PtrInfo: MachinePointerInfo(&*FuncArg), SVT: ObjType);
4615 MemOps.push_back(Elt: Store);
4616 }
4617 // Whether we copied from a register or not, advance the offset
4618 // into the parameter save area by a full doubleword.
4619 ArgOffset += PtrByteSize;
4620 continue;
4621 }
4622
4623 // The value of the object is its address, which is the address of
4624 // its first stack doubleword.
4625 InVals.push_back(Elt: FIN);
4626
4627 // Store whatever pieces of the object are in registers to memory.
4628 for (unsigned j = 0; j < ArgSize; j += PtrByteSize) {
4629 if (GPR_idx == Num_GPR_Regs)
4630 break;
4631
4632 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4633 FuncInfo->addLiveInAttr(VReg, Flags);
4634 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4635 SDValue Addr = FIN;
4636 if (j) {
4637 SDValue Off = DAG.getConstant(Val: j, DL: dl, VT: PtrVT);
4638 Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Off.getValueType(), N1: Addr, N2: Off);
4639 }
4640 unsigned StoreSizeInBits = std::min(a: PtrByteSize, b: (ObjSize - j)) * 8;
4641 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: StoreSizeInBits);
4642 SDValue Store =
4643 DAG.getTruncStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: Addr,
4644 PtrInfo: MachinePointerInfo(&*FuncArg, j), SVT: ObjType);
4645 MemOps.push_back(Elt: Store);
4646 ++GPR_idx;
4647 }
4648 ArgOffset += ArgSize;
4649 continue;
4650 }
4651
4652 switch (ObjectVT.getSimpleVT().SimpleTy) {
4653 default: llvm_unreachable("Unhandled argument type!");
4654 case MVT::i1:
4655 case MVT::i32:
4656 case MVT::i64:
4657 if (Flags.isNest()) {
4658 // The 'nest' parameter, if any, is passed in R11.
4659 Register VReg = MF.addLiveIn(PReg: PPC::X11, RC: &PPC::G8RCRegClass);
4660 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4661
4662 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4663 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4664
4665 break;
4666 }
4667
4668 // These can be scalar arguments or elements of an integer array type
4669 // passed directly. Clang may use those instead of "byval" aggregate
4670 // types to avoid forcing arguments to memory unnecessarily.
4671 if (GPR_idx != Num_GPR_Regs) {
4672 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4673 FuncInfo->addLiveInAttr(VReg, Flags);
4674 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4675
4676 if (ObjectVT == MVT::i32 || ObjectVT == MVT::i1)
4677 // PPC64 passes i8, i16, and i32 values in i64 registers. Promote
4678 // value to MVT::i64 and then truncate to the correct register size.
4679 ArgVal = extendArgForPPC64(Flags, ObjectVT, DAG, ArgVal, dl);
4680 } else {
4681 if (CallConv == CallingConv::Fast)
4682 ComputeArgOffset();
4683
4684 needsLoad = true;
4685 ArgSize = PtrByteSize;
4686 }
4687 if (CallConv != CallingConv::Fast || needsLoad)
4688 ArgOffset += 8;
4689 break;
4690
4691 case MVT::f32:
4692 case MVT::f64:
4693 // These can be scalar arguments or elements of a float array type
4694 // passed directly. The latter are used to implement ELFv2 homogenous
4695 // float aggregates.
4696 if (FPR_idx != Num_FPR_Regs) {
4697 unsigned VReg;
4698
4699 if (ObjectVT == MVT::f32)
4700 VReg = MF.addLiveIn(PReg: FPR[FPR_idx],
4701 RC: Subtarget.hasP8Vector()
4702 ? &PPC::VSSRCRegClass
4703 : &PPC::F4RCRegClass);
4704 else
4705 VReg = MF.addLiveIn(PReg: FPR[FPR_idx], RC: Subtarget.hasVSX()
4706 ? &PPC::VSFRCRegClass
4707 : &PPC::F8RCRegClass);
4708
4709 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4710 ++FPR_idx;
4711 } else if (GPR_idx != Num_GPR_Regs && CallConv != CallingConv::Fast) {
4712 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
4713 // once we support fp <-> gpr moves.
4714
4715 // This can only ever happen in the presence of f32 array types,
4716 // since otherwise we never run out of FPRs before running out
4717 // of GPRs.
4718 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx++], RC: &PPC::G8RCRegClass);
4719 FuncInfo->addLiveInAttr(VReg, Flags);
4720 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: MVT::i64);
4721
4722 if (ObjectVT == MVT::f32) {
4723 if ((ArgOffset % PtrByteSize) == (isLittleEndian ? 4 : 0))
4724 ArgVal = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgVal,
4725 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
4726 ArgVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: ArgVal);
4727 }
4728
4729 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: ObjectVT, Operand: ArgVal);
4730 } else {
4731 if (CallConv == CallingConv::Fast)
4732 ComputeArgOffset();
4733
4734 needsLoad = true;
4735 }
4736
4737 // When passing an array of floats, the array occupies consecutive
4738 // space in the argument area; only round up to the next doubleword
4739 // at the end of the array. Otherwise, each float takes 8 bytes.
4740 if (CallConv != CallingConv::Fast || needsLoad) {
4741 ArgSize = Flags.isInConsecutiveRegs() ? ObjSize : PtrByteSize;
4742 ArgOffset += ArgSize;
4743 if (Flags.isInConsecutiveRegsLast())
4744 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
4745 }
4746 break;
4747 case MVT::v4f32:
4748 case MVT::v4i32:
4749 case MVT::v8i16:
4750 case MVT::v16i8:
4751 case MVT::v2f64:
4752 case MVT::v2i64:
4753 case MVT::v1i128:
4754 case MVT::f128:
4755 // These can be scalar arguments or elements of a vector array type
4756 // passed directly. The latter are used to implement ELFv2 homogenous
4757 // vector aggregates.
4758 if (VR_idx != Num_VR_Regs) {
4759 Register VReg = MF.addLiveIn(PReg: VR[VR_idx], RC: &PPC::VRRCRegClass);
4760 ArgVal = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: ObjectVT);
4761 ++VR_idx;
4762 } else {
4763 if (CallConv == CallingConv::Fast)
4764 ComputeArgOffset();
4765 needsLoad = true;
4766 }
4767 if (CallConv != CallingConv::Fast || needsLoad)
4768 ArgOffset += 16;
4769 break;
4770 }
4771
4772 // We need to load the argument to a virtual register if we determined
4773 // above that we ran out of physical registers of the appropriate type.
4774 if (needsLoad) {
4775 if (ObjSize < ArgSize && !isLittleEndian)
4776 CurArgOffset += ArgSize - ObjSize;
4777 int FI = MFI.CreateFixedObject(Size: ObjSize, SPOffset: CurArgOffset, IsImmutable: isImmutable);
4778 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
4779 ArgVal = DAG.getLoad(VT: ObjectVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
4780 }
4781
4782 InVals.push_back(Elt: ArgVal);
4783 }
4784
4785 // Area that is at least reserved in the caller of this function.
4786 unsigned MinReservedArea;
4787 if (HasParameterArea)
4788 MinReservedArea = std::max(a: ArgOffset, b: LinkageSize + 8 * PtrByteSize);
4789 else
4790 MinReservedArea = LinkageSize;
4791
4792 // Set the size that is at least reserved in caller of this function. Tail
4793 // call optimized functions' reserved stack space needs to be aligned so that
4794 // taking the difference between two stack areas will result in an aligned
4795 // stack.
4796 MinReservedArea =
4797 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: MinReservedArea);
4798 FuncInfo->setMinReservedArea(MinReservedArea);
4799
4800 // If the function takes variable number of arguments, make a frame index for
4801 // the start of the first vararg value... for expansion of llvm.va_start.
4802 // On ELFv2ABI spec, it writes:
4803 // C programs that are intended to be *portable* across different compilers
4804 // and architectures must use the header file <stdarg.h> to deal with variable
4805 // argument lists.
4806 if (isVarArg && MFI.hasVAStart()) {
4807 int Depth = ArgOffset;
4808
4809 FuncInfo->setVarArgsFrameIndex(
4810 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: Depth, IsImmutable: true));
4811 SDValue FIN = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(), VT: PtrVT);
4812
4813 // If this function is vararg, store any remaining integer argument regs
4814 // to their spots on the stack so that they may be loaded by dereferencing
4815 // the result of va_next.
4816 for (GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
4817 GPR_idx < Num_GPR_Regs; ++GPR_idx) {
4818 Register VReg = MF.addLiveIn(PReg: GPR[GPR_idx], RC: &PPC::G8RCRegClass);
4819 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
4820 SDValue Store =
4821 DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MachinePointerInfo());
4822 MemOps.push_back(Elt: Store);
4823 // Increment the address by four for the next argument to store
4824 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
4825 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
4826 }
4827 }
4828
4829 if (!MemOps.empty())
4830 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
4831
4832 return Chain;
4833}
4834
4835/// CalculateTailCallSPDiff - Get the amount the stack pointer has to be
4836/// adjusted to accommodate the arguments for the tailcall.
4837static int CalculateTailCallSPDiff(SelectionDAG& DAG, bool isTailCall,
4838 unsigned ParamSize) {
4839
4840 if (!isTailCall) return 0;
4841
4842 PPCFunctionInfo *FI = DAG.getMachineFunction().getInfo<PPCFunctionInfo>();
4843 unsigned CallerMinReservedArea = FI->getMinReservedArea();
4844 int SPDiff = (int)CallerMinReservedArea - (int)ParamSize;
4845 // Remember only if the new adjustment is bigger.
4846 if (SPDiff < FI->getTailCallSPDelta())
4847 FI->setTailCallSPDelta(SPDiff);
4848
4849 return SPDiff;
4850}
4851
4852static bool isFunctionGlobalAddress(const GlobalValue *CalleeGV);
4853
4854static bool callsShareTOCBase(const Function *Caller,
4855 const GlobalValue *CalleeGV,
4856 const TargetMachine &TM) {
4857 // It does not make sense to call callsShareTOCBase() with a caller that
4858 // is PC Relative since PC Relative callers do not have a TOC.
4859#ifndef NDEBUG
4860 const PPCSubtarget *STICaller = &TM.getSubtarget<PPCSubtarget>(*Caller);
4861 assert(!STICaller->isUsingPCRelativeCalls() &&
4862 "PC Relative callers do not have a TOC and cannot share a TOC Base");
4863#endif
4864
4865 // Callee is either a GlobalAddress or an ExternalSymbol. ExternalSymbols
4866 // don't have enough information to determine if the caller and callee share
4867 // the same TOC base, so we have to pessimistically assume they don't for
4868 // correctness.
4869 if (!CalleeGV)
4870 return false;
4871
4872 // If the callee is preemptable, then the static linker will use a plt-stub
4873 // which saves the toc to the stack, and needs a nop after the call
4874 // instruction to convert to a toc-restore.
4875 if (!TM.shouldAssumeDSOLocal(GV: CalleeGV))
4876 return false;
4877
4878 // Functions with PC Relative enabled may clobber the TOC in the same DSO.
4879 // We may need a TOC restore in the situation where the caller requires a
4880 // valid TOC but the callee is PC Relative and does not.
4881 const Function *F = dyn_cast<Function>(Val: CalleeGV);
4882 const GlobalAlias *Alias = dyn_cast<GlobalAlias>(Val: CalleeGV);
4883
4884 // If we have an Alias we can try to get the function from there.
4885 if (Alias) {
4886 const GlobalObject *GlobalObj = Alias->getAliaseeObject();
4887 F = dyn_cast<Function>(Val: GlobalObj);
4888 }
4889
4890 // If we still have no valid function pointer we do not have enough
4891 // information to determine if the callee uses PC Relative calls so we must
4892 // assume that it does.
4893 if (!F)
4894 return false;
4895
4896 // If the callee uses PC Relative we cannot guarantee that the callee won't
4897 // clobber the TOC of the caller and so we must assume that the two
4898 // functions do not share a TOC base.
4899 const PPCSubtarget *STICallee = &TM.getSubtarget<PPCSubtarget>(F: *F);
4900 if (STICallee->isUsingPCRelativeCalls())
4901 return false;
4902
4903 // If the GV is not a strong definition then we need to assume it can be
4904 // replaced by another function at link time. The function that replaces
4905 // it may not share the same TOC as the caller since the callee may be
4906 // replaced by a PC Relative version of the same function.
4907 if (!CalleeGV->isStrongDefinitionForLinker())
4908 return false;
4909
4910 // The medium and large code models are expected to provide a sufficiently
4911 // large TOC to provide all data addressing needs of a module with a
4912 // single TOC.
4913 if (CodeModel::Medium == TM.getCodeModel() ||
4914 CodeModel::Large == TM.getCodeModel())
4915 return true;
4916
4917 // Any explicitly-specified sections and section prefixes must also match.
4918 // Also, if we're using -ffunction-sections, then each function is always in
4919 // a different section (the same is true for COMDAT functions).
4920 if (TM.getFunctionSections() || CalleeGV->hasComdat() ||
4921 Caller->hasComdat() || CalleeGV->getSection() != Caller->getSection())
4922 return false;
4923 if (const auto *F = dyn_cast<Function>(Val: CalleeGV)) {
4924 if (F->getSectionPrefix() != Caller->getSectionPrefix())
4925 return false;
4926 }
4927
4928 return true;
4929}
4930
4931static bool
4932needStackSlotPassParameters(const PPCSubtarget &Subtarget,
4933 const SmallVectorImpl<ISD::OutputArg> &Outs) {
4934 assert(Subtarget.is64BitELFABI());
4935
4936 const unsigned PtrByteSize = 8;
4937 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
4938
4939 static const MCPhysReg GPR[] = {
4940 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
4941 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
4942 };
4943 static const MCPhysReg VR[] = {
4944 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
4945 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
4946 };
4947
4948 const unsigned NumGPRs = std::size(GPR);
4949 const unsigned NumFPRs = 13;
4950 const unsigned NumVRs = std::size(VR);
4951 const unsigned ParamAreaSize = NumGPRs * PtrByteSize;
4952
4953 unsigned NumBytes = LinkageSize;
4954 unsigned AvailableFPRs = NumFPRs;
4955 unsigned AvailableVRs = NumVRs;
4956
4957 for (const ISD::OutputArg& Param : Outs) {
4958 if (Param.Flags.isNest()) continue;
4959
4960 if (CalculateStackSlotUsed(ArgVT: Param.VT, OrigVT: Param.ArgVT, Flags: Param.Flags, PtrByteSize,
4961 LinkageSize, ParamAreaSize, ArgOffset&: NumBytes,
4962 AvailableFPRs, AvailableVRs))
4963 return true;
4964 }
4965 return false;
4966}
4967
4968static bool hasSameArgumentList(const Function *CallerFn, const CallBase &CB) {
4969 if (CB.arg_size() != CallerFn->arg_size())
4970 return false;
4971
4972 auto CalleeArgIter = CB.arg_begin();
4973 auto CalleeArgEnd = CB.arg_end();
4974 Function::const_arg_iterator CallerArgIter = CallerFn->arg_begin();
4975
4976 for (; CalleeArgIter != CalleeArgEnd; ++CalleeArgIter, ++CallerArgIter) {
4977 const Value* CalleeArg = *CalleeArgIter;
4978 const Value* CallerArg = &(*CallerArgIter);
4979 if (CalleeArg == CallerArg)
4980 continue;
4981
4982 // e.g. @caller([4 x i64] %a, [4 x i64] %b) {
4983 // tail call @callee([4 x i64] undef, [4 x i64] %b)
4984 // }
4985 // 1st argument of callee is undef and has the same type as caller.
4986 if (CalleeArg->getType() == CallerArg->getType() &&
4987 isa<UndefValue>(Val: CalleeArg))
4988 continue;
4989
4990 return false;
4991 }
4992
4993 return true;
4994}
4995
4996// Returns true if TCO is possible between the callers and callees
4997// calling conventions.
4998static bool
4999areCallingConvEligibleForTCO_64SVR4(CallingConv::ID CallerCC,
5000 CallingConv::ID CalleeCC) {
5001 // Tail calls are possible with fastcc and ccc.
5002 auto isTailCallableCC = [] (CallingConv::ID CC){
5003 return CC == CallingConv::C || CC == CallingConv::Fast;
5004 };
5005 if (!isTailCallableCC(CallerCC) || !isTailCallableCC(CalleeCC))
5006 return false;
5007
5008 // We can safely tail call both fastcc and ccc callees from a c calling
5009 // convention caller. If the caller is fastcc, we may have less stack space
5010 // than a non-fastcc caller with the same signature so disable tail-calls in
5011 // that case.
5012 return CallerCC == CallingConv::C || CallerCC == CalleeCC;
5013}
5014
5015bool PPCTargetLowering::IsEligibleForTailCallOptimization_64SVR4(
5016 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5017 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5018 const SmallVectorImpl<ISD::OutputArg> &Outs,
5019 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5020 bool isCalleeExternalSymbol) const {
5021 bool TailCallOpt = getTargetMachine().Options.GuaranteedTailCallOpt;
5022
5023 if (!EnableSCO && !TailCallOpt)
5024 return false;
5025
5026 // Variadic argument functions are not supported.
5027 if (isVarArg) return false;
5028
5029 // Check that the calling conventions are compatible for tco.
5030 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC, CalleeCC))
5031 return false;
5032
5033 // Caller contains any byval parameter is not supported.
5034 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5035 return false;
5036
5037 // Callee contains any byval parameter is not supported, too.
5038 // Note: This is a quick work around, because in some cases, e.g.
5039 // caller's stack size > callee's stack size, we are still able to apply
5040 // sibling call optimization. For example, gcc is able to do SCO for caller1
5041 // in the following example, but not for caller2.
5042 // struct test {
5043 // long int a;
5044 // char ary[56];
5045 // } gTest;
5046 // __attribute__((noinline)) int callee(struct test v, struct test *b) {
5047 // b->a = v.a;
5048 // return 0;
5049 // }
5050 // void caller1(struct test a, struct test c, struct test *b) {
5051 // callee(gTest, b); }
5052 // void caller2(struct test *b) { callee(gTest, b); }
5053 if (any_of(Range: Outs, P: [](const ISD::OutputArg& OA) { return OA.Flags.isByVal(); }))
5054 return false;
5055
5056 // If callee and caller use different calling conventions, we cannot pass
5057 // parameters on stack since offsets for the parameter area may be different.
5058 if (CallerCC != CalleeCC && needStackSlotPassParameters(Subtarget, Outs))
5059 return false;
5060
5061 // All variants of 64-bit ELF ABIs without PC-Relative addressing require that
5062 // the caller and callee share the same TOC for TCO/SCO. If the caller and
5063 // callee potentially have different TOC bases then we cannot tail call since
5064 // we need to restore the TOC pointer after the call.
5065 // ref: https://bugzilla.mozilla.org/show_bug.cgi?id=973977
5066 // We cannot guarantee this for indirect calls or calls to external functions.
5067 // When PC-Relative addressing is used, the concept of the TOC is no longer
5068 // applicable so this check is not required.
5069 // Check first for indirect calls.
5070 if (!Subtarget.isUsingPCRelativeCalls() &&
5071 !isFunctionGlobalAddress(CalleeGV) && !isCalleeExternalSymbol)
5072 return false;
5073
5074 // Check if we share the TOC base.
5075 if (!Subtarget.isUsingPCRelativeCalls() &&
5076 !callsShareTOCBase(Caller: CallerFunc, CalleeGV, TM: getTargetMachine()))
5077 return false;
5078
5079 // TCO allows altering callee ABI, so we don't have to check further.
5080 if (CalleeCC == CallingConv::Fast && TailCallOpt)
5081 return true;
5082
5083 if (!EnableSCO)
5084 return false;
5085
5086 // If callee use the same argument list that caller is using, then we can
5087 // apply SCO on this case. If it is not, then we need to check if callee needs
5088 // stack for passing arguments.
5089 // PC Relative tail calls may not have a CallBase.
5090 // If there is no CallBase we cannot verify if we have the same argument
5091 // list so assume that we don't have the same argument list.
5092 if (CB && !hasSameArgumentList(CallerFn: CallerFunc, CB: *CB) &&
5093 needStackSlotPassParameters(Subtarget, Outs))
5094 return false;
5095 else if (!CB && needStackSlotPassParameters(Subtarget, Outs))
5096 return false;
5097
5098 return true;
5099}
5100
5101/// IsEligibleForTailCallOptimization - Check whether the call is eligible
5102/// for tail call optimization. Targets which want to do tail call
5103/// optimization should implement this function.
5104bool PPCTargetLowering::IsEligibleForTailCallOptimization(
5105 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5106 CallingConv::ID CallerCC, bool isVarArg,
5107 const SmallVectorImpl<ISD::InputArg> &Ins) const {
5108 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5109 return false;
5110
5111 // Variable argument functions are not supported.
5112 if (isVarArg)
5113 return false;
5114
5115 if (CalleeCC == CallingConv::Fast && CallerCC == CalleeCC) {
5116 // Functions containing by val parameters are not supported.
5117 if (any_of(Range: Ins, P: [](const ISD::InputArg &IA) { return IA.Flags.isByVal(); }))
5118 return false;
5119
5120 // Non-PIC/GOT tail calls are supported.
5121 if (getTargetMachine().getRelocationModel() != Reloc::PIC_)
5122 return true;
5123
5124 // At the moment we can only do local tail calls (in same module, hidden
5125 // or protected) if we are generating PIC.
5126 if (CalleeGV)
5127 return CalleeGV->hasHiddenVisibility() ||
5128 CalleeGV->hasProtectedVisibility();
5129 }
5130
5131 return false;
5132}
5133
5134/// isCallCompatibleAddress - Return the immediate to use if the specified
5135/// 32-bit value is representable in the immediate field of a BxA instruction.
5136static SDNode *isBLACompatibleAddress(SDValue Op, SelectionDAG &DAG) {
5137 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Op);
5138 if (!C) return nullptr;
5139
5140 int Addr = C->getZExtValue();
5141 if ((Addr & 3) != 0 || // Low 2 bits are implicitly zero.
5142 SignExtend32<26>(X: Addr) != Addr)
5143 return nullptr; // Top 6 bits have to be sext of immediate.
5144
5145 return DAG
5146 .getSignedConstant(
5147 Val: (int)C->getZExtValue() >> 2, DL: SDLoc(Op),
5148 VT: DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout()))
5149 .getNode();
5150}
5151
5152namespace {
5153
5154struct TailCallArgumentInfo {
5155 SDValue Arg;
5156 SDValue FrameIdxOp;
5157 int FrameIdx = 0;
5158
5159 TailCallArgumentInfo() = default;
5160};
5161
5162} // end anonymous namespace
5163
5164/// StoreTailCallArgumentsToStackSlot - Stores arguments to their stack slot.
5165static void StoreTailCallArgumentsToStackSlot(
5166 SelectionDAG &DAG, SDValue Chain,
5167 const SmallVectorImpl<TailCallArgumentInfo> &TailCallArgs,
5168 SmallVectorImpl<SDValue> &MemOpChains, const SDLoc &dl) {
5169 for (unsigned i = 0, e = TailCallArgs.size(); i != e; ++i) {
5170 SDValue Arg = TailCallArgs[i].Arg;
5171 SDValue FIN = TailCallArgs[i].FrameIdxOp;
5172 int FI = TailCallArgs[i].FrameIdx;
5173 // Store relative to framepointer.
5174 MemOpChains.push_back(Elt: DAG.getStore(
5175 Chain, dl, Val: Arg, Ptr: FIN,
5176 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI)));
5177 }
5178}
5179
5180/// EmitTailCallStoreFPAndRetAddr - Move the frame pointer and return address to
5181/// the appropriate stack slot for the tail call optimized function call.
5182static SDValue EmitTailCallStoreFPAndRetAddr(SelectionDAG &DAG, SDValue Chain,
5183 SDValue OldRetAddr, SDValue OldFP,
5184 int SPDiff, const SDLoc &dl) {
5185 if (SPDiff) {
5186 // Calculate the new stack slot for the return address.
5187 MachineFunction &MF = DAG.getMachineFunction();
5188 const PPCSubtarget &Subtarget = MF.getSubtarget<PPCSubtarget>();
5189 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
5190 int SlotSize = Subtarget.isPPC64() ? 8 : 4;
5191 int NewRetAddrLoc = SPDiff + FL->getReturnSaveOffset();
5192 int NewRetAddr = MF.getFrameInfo().CreateFixedObject(Size: SlotSize,
5193 SPOffset: NewRetAddrLoc, IsImmutable: true);
5194 SDValue NewRetAddrFrIdx =
5195 DAG.getFrameIndex(FI: NewRetAddr, VT: Subtarget.getScalarIntVT());
5196 Chain = DAG.getStore(Chain, dl, Val: OldRetAddr, Ptr: NewRetAddrFrIdx,
5197 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI: NewRetAddr));
5198 }
5199 return Chain;
5200}
5201
5202/// CalculateTailCallArgDest - Remember Argument for later processing. Calculate
5203/// the position of the argument.
5204static void CalculateTailCallArgDest(
5205 SelectionDAG &DAG, MachineFunction &MF, bool IsPPC64, SDValue Arg,
5206 int SPDiff, unsigned ArgOffset,
5207 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5208 int Offset = ArgOffset + SPDiff;
5209 uint32_t OpSize = (Arg.getValueSizeInBits() + 7) / 8;
5210 int FI = MF.getFrameInfo().CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
5211 EVT VT = IsPPC64 ? MVT::i64 : MVT::i32;
5212 SDValue FIN = DAG.getFrameIndex(FI, VT);
5213 TailCallArgumentInfo Info;
5214 Info.Arg = Arg;
5215 Info.FrameIdxOp = FIN;
5216 Info.FrameIdx = FI;
5217 TailCallArguments.push_back(Elt: Info);
5218}
5219
5220/// EmitTCFPAndRetAddrLoad - Emit load from frame pointer and return address
5221/// stack slot. Returns the chain as result and the loaded frame pointers in
5222/// LROpOut/FPOpout. Used when tail calling.
5223SDValue PPCTargetLowering::EmitTailCallLoadFPAndRetAddr(
5224 SelectionDAG &DAG, int SPDiff, SDValue Chain, SDValue &LROpOut,
5225 SDValue &FPOpOut, const SDLoc &dl) const {
5226 if (SPDiff) {
5227 // Load the LR and FP stack slot for later adjusting.
5228 LROpOut = getReturnAddrFrameIndex(DAG);
5229 LROpOut = DAG.getLoad(VT: Subtarget.getScalarIntVT(), dl, Chain, Ptr: LROpOut,
5230 PtrInfo: MachinePointerInfo());
5231 Chain = SDValue(LROpOut.getNode(), 1);
5232 }
5233 return Chain;
5234}
5235
5236/// CreateCopyOfByValArgument - Make a copy of an aggregate at address specified
5237/// by "Src" to address "Dst" of size "Size". Alignment information is
5238/// specified by the specific parameter attribute. The copy will be passed as
5239/// a byval function parameter.
5240/// Sometimes what we are copying is the end of a larger object, the part that
5241/// does not fit in registers.
5242static SDValue CreateCopyOfByValArgument(SDValue Src, SDValue Dst,
5243 SDValue Chain, ISD::ArgFlagsTy Flags,
5244 SelectionDAG &DAG, const SDLoc &dl) {
5245 SDValue SizeNode = DAG.getConstant(Val: Flags.getByValSize(), DL: dl, VT: MVT::i32);
5246 Align Alignment = Flags.getNonZeroByValAlign();
5247 return DAG.getMemcpy(
5248 Chain, dl, Dst, Src, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment, isVol: false, AlwaysInline: false,
5249 /*CI=*/nullptr, OverrideTailCall: std::nullopt, DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
5250}
5251
5252/// LowerMemOpCallTo - Store the argument to the stack or remember it in case of
5253/// tail calls.
5254static void LowerMemOpCallTo(
5255 SelectionDAG &DAG, MachineFunction &MF, SDValue Chain, SDValue Arg,
5256 SDValue PtrOff, int SPDiff, unsigned ArgOffset, bool isPPC64,
5257 bool isTailCall, bool isVector, SmallVectorImpl<SDValue> &MemOpChains,
5258 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments, const SDLoc &dl) {
5259 EVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5260 if (!isTailCall) {
5261 if (isVector) {
5262 SDValue StackPtr;
5263 if (isPPC64)
5264 StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
5265 else
5266 StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
5267 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr,
5268 N2: DAG.getConstant(Val: ArgOffset, DL: dl, VT: PtrVT));
5269 }
5270 MemOpChains.push_back(
5271 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
5272 // Calculate and remember argument location.
5273 } else
5274 CalculateTailCallArgDest(DAG, MF, IsPPC64: isPPC64, Arg, SPDiff, ArgOffset,
5275 TailCallArguments);
5276}
5277
5278static void
5279PrepareTailCall(SelectionDAG &DAG, SDValue &InGlue, SDValue &Chain,
5280 const SDLoc &dl, int SPDiff, unsigned NumBytes, SDValue LROp,
5281 SDValue FPOp,
5282 SmallVectorImpl<TailCallArgumentInfo> &TailCallArguments) {
5283 // Emit a sequence of copyto/copyfrom virtual registers for arguments that
5284 // might overwrite each other in case of tail call optimization.
5285 SmallVector<SDValue, 8> MemOpChains2;
5286 // Do not flag preceding copytoreg stuff together with the following stuff.
5287 InGlue = SDValue();
5288 StoreTailCallArgumentsToStackSlot(DAG, Chain, TailCallArgs: TailCallArguments,
5289 MemOpChains&: MemOpChains2, dl);
5290 if (!MemOpChains2.empty())
5291 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains2);
5292
5293 // Store the return address to the appropriate stack slot.
5294 Chain = EmitTailCallStoreFPAndRetAddr(DAG, Chain, OldRetAddr: LROp, OldFP: FPOp, SPDiff, dl);
5295
5296 // Emit callseq_end just before tailcall node.
5297 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue: InGlue, DL: dl);
5298 InGlue = Chain.getValue(R: 1);
5299}
5300
5301// Is this global address that of a function that can be called by name? (as
5302// opposed to something that must hold a descriptor for an indirect call).
5303static bool isFunctionGlobalAddress(const GlobalValue *GV) {
5304 if (GV) {
5305 if (GV->isThreadLocal())
5306 return false;
5307
5308 return GV->getValueType()->isFunctionTy();
5309 }
5310
5311 return false;
5312}
5313
5314SDValue PPCTargetLowering::LowerCallResult(
5315 SDValue Chain, SDValue InGlue, CallingConv::ID CallConv, bool isVarArg,
5316 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5317 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
5318 SmallVector<CCValAssign, 16> RVLocs;
5319 CCState CCRetInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
5320 *DAG.getContext());
5321
5322 CCRetInfo.AnalyzeCallResult(
5323 Ins, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
5324 ? RetCC_PPC_Cold
5325 : RetCC_PPC);
5326
5327 // Copy all of the result registers out of their specified physreg.
5328 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
5329 CCValAssign &VA = RVLocs[i];
5330 assert(VA.isRegLoc() && "Can only return in registers!");
5331
5332 SDValue Val;
5333
5334 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
5335 SDValue Lo = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5336 Glue: InGlue);
5337 Chain = Lo.getValue(R: 1);
5338 InGlue = Lo.getValue(R: 2);
5339 VA = RVLocs[++i]; // skip ahead to next loc
5340 SDValue Hi = DAG.getCopyFromReg(Chain, dl, Reg: VA.getLocReg(), VT: MVT::i32,
5341 Glue: InGlue);
5342 Chain = Hi.getValue(R: 1);
5343 InGlue = Hi.getValue(R: 2);
5344 if (!Subtarget.isLittleEndian())
5345 std::swap (a&: Lo, b&: Hi);
5346 Val = DAG.getNode(Opcode: PPCISD::BUILD_SPE64, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
5347 } else {
5348 Val = DAG.getCopyFromReg(Chain, dl,
5349 Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue: InGlue);
5350 Chain = Val.getValue(R: 1);
5351 InGlue = Val.getValue(R: 2);
5352 }
5353
5354 switch (VA.getLocInfo()) {
5355 default: llvm_unreachable("Unknown loc info!");
5356 case CCValAssign::Full: break;
5357 case CCValAssign::AExt:
5358 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5359 break;
5360 case CCValAssign::ZExt:
5361 Val = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: VA.getLocVT(), N1: Val,
5362 N2: DAG.getValueType(VA.getValVT()));
5363 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5364 break;
5365 case CCValAssign::SExt:
5366 Val = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: VA.getLocVT(), N1: Val,
5367 N2: DAG.getValueType(VA.getValVT()));
5368 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: VA.getValVT(), Operand: Val);
5369 break;
5370 }
5371
5372 InVals.push_back(Elt: Val);
5373 }
5374
5375 return Chain;
5376}
5377
5378static bool isIndirectCall(const SDValue &Callee, SelectionDAG &DAG,
5379 const PPCSubtarget &Subtarget, bool isPatchPoint) {
5380 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5381 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5382
5383 // PatchPoint calls are not indirect.
5384 if (isPatchPoint)
5385 return false;
5386
5387 if (isFunctionGlobalAddress(GV) || isa<ExternalSymbolSDNode>(Val: Callee))
5388 return false;
5389
5390 // Darwin, and 32-bit ELF can use a BLA. The descriptor based ABIs can not
5391 // becuase the immediate function pointer points to a descriptor instead of
5392 // a function entry point. The ELFv2 ABI cannot use a BLA because the function
5393 // pointer immediate points to the global entry point, while the BLA would
5394 // need to jump to the local entry point (see rL211174).
5395 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI() &&
5396 isBLACompatibleAddress(Op: Callee, DAG))
5397 return false;
5398
5399 return true;
5400}
5401
5402// AIX and 64-bit ELF ABIs w/o PCRel require a TOC save/restore around calls.
5403static inline bool isTOCSaveRestoreRequired(const PPCSubtarget &Subtarget) {
5404 return Subtarget.isAIXABI() ||
5405 (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls());
5406}
5407
5408static unsigned getCallOpcode(PPCTargetLowering::CallFlags CFlags,
5409 const Function &Caller, const SDValue &Callee,
5410 const PPCSubtarget &Subtarget,
5411 const TargetMachine &TM,
5412 bool IsStrictFPCall = false) {
5413 if (CFlags.IsTailCall)
5414 return PPCISD::TC_RETURN;
5415
5416 unsigned RetOpc = 0;
5417 // This is a call through a function pointer.
5418 if (CFlags.IsIndirect) {
5419 // AIX and the 64-bit ELF ABIs need to maintain the TOC pointer accross
5420 // indirect calls. The save of the caller's TOC pointer to the stack will be
5421 // inserted into the DAG as part of call lowering. The restore of the TOC
5422 // pointer is modeled by using a pseudo instruction for the call opcode that
5423 // represents the 2 instruction sequence of an indirect branch and link,
5424 // immediately followed by a load of the TOC pointer from the stack save
5425 // slot into gpr2. For 64-bit ELFv2 ABI with PCRel, do not restore the TOC
5426 // as it is not saved or used.
5427 if (Subtarget.usePointerGlueHelper())
5428 RetOpc = PPCISD::BL_LOAD_TOC;
5429 else
5430 RetOpc = isTOCSaveRestoreRequired(Subtarget) ? PPCISD::BCTRL_LOAD_TOC
5431 : PPCISD::BCTRL;
5432 } else if (Subtarget.isUsingPCRelativeCalls()) {
5433 assert(Subtarget.is64BitELFABI() && "PC Relative is only on ELF ABI.");
5434 RetOpc = PPCISD::CALL_NOTOC;
5435 } else if (Subtarget.isAIXABI() || Subtarget.is64BitELFABI()) {
5436 // The ABIs that maintain a TOC pointer accross calls need to have a nop
5437 // immediately following the call instruction if the caller and callee may
5438 // have different TOC bases. At link time if the linker determines the calls
5439 // may not share a TOC base, the call is redirected to a trampoline inserted
5440 // by the linker. The trampoline will (among other things) save the callers
5441 // TOC pointer at an ABI designated offset in the linkage area and the
5442 // linker will rewrite the nop to be a load of the TOC pointer from the
5443 // linkage area into gpr2.
5444 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5445 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5446 RetOpc =
5447 callsShareTOCBase(Caller: &Caller, CalleeGV: GV, TM) ? PPCISD::CALL : PPCISD::CALL_NOP;
5448 } else
5449 RetOpc = PPCISD::CALL;
5450 if (IsStrictFPCall) {
5451 switch (RetOpc) {
5452 default:
5453 llvm_unreachable("Unknown call opcode");
5454 case PPCISD::BCTRL_LOAD_TOC:
5455 RetOpc = PPCISD::BCTRL_LOAD_TOC_RM;
5456 break;
5457 case PPCISD::BCTRL:
5458 RetOpc = PPCISD::BCTRL_RM;
5459 break;
5460 case PPCISD::BL_LOAD_TOC:
5461 RetOpc = PPCISD::BL_LOAD_TOC_RM;
5462 break;
5463 case PPCISD::CALL_NOTOC:
5464 RetOpc = PPCISD::CALL_NOTOC_RM;
5465 break;
5466 case PPCISD::CALL:
5467 RetOpc = PPCISD::CALL_RM;
5468 break;
5469 case PPCISD::CALL_NOP:
5470 RetOpc = PPCISD::CALL_NOP_RM;
5471 break;
5472 }
5473 }
5474 return RetOpc;
5475}
5476
5477static SDValue transformCallee(const SDValue &Callee, SelectionDAG &DAG,
5478 const SDLoc &dl, const PPCSubtarget &Subtarget) {
5479 if (!Subtarget.usesFunctionDescriptors() && !Subtarget.isELFv2ABI())
5480 if (SDNode *Dest = isBLACompatibleAddress(Op: Callee, DAG))
5481 return SDValue(Dest, 0);
5482
5483 // Returns true if the callee is local, and false otherwise.
5484 auto isLocalCallee = [&]() {
5485 const GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5486 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5487
5488 return DAG.getTarget().shouldAssumeDSOLocal(GV) &&
5489 !isa_and_nonnull<GlobalIFunc>(Val: GV);
5490 };
5491
5492 // The PLT is only used in 32-bit ELF PIC mode. Attempting to use the PLT in
5493 // a static relocation model causes some versions of GNU LD (2.17.50, at
5494 // least) to force BSS-PLT, instead of secure-PLT, even if all objects are
5495 // built with secure-PLT.
5496 bool UsePlt =
5497 Subtarget.is32BitELFABI() && !isLocalCallee() &&
5498 Subtarget.getTargetMachine().getRelocationModel() == Reloc::PIC_;
5499
5500 const auto getAIXFuncEntryPointSymbolSDNode = [&](const GlobalValue *GV) {
5501 const TargetMachine &TM = Subtarget.getTargetMachine();
5502 const TargetLoweringObjectFile *TLOF = TM.getObjFileLowering();
5503 auto *S =
5504 static_cast<MCSymbolXCOFF *>(TLOF->getFunctionEntryPointSymbol(Func: GV, TM));
5505
5506 MVT PtrVT = DAG.getTargetLoweringInfo().getPointerTy(DL: DAG.getDataLayout());
5507 return DAG.getMCSymbol(Sym: S, VT: PtrVT);
5508 };
5509
5510 auto *G = dyn_cast<GlobalAddressSDNode>(Val: Callee);
5511 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5512 if (isFunctionGlobalAddress(GV)) {
5513 const GlobalValue *GV = cast<GlobalAddressSDNode>(Val: Callee)->getGlobal();
5514
5515 if (Subtarget.isAIXABI()) {
5516 return getAIXFuncEntryPointSymbolSDNode(GV);
5517 }
5518 return DAG.getTargetGlobalAddress(GV, DL: dl, VT: Callee.getValueType(), offset: 0,
5519 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5520 }
5521
5522 if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val: Callee)) {
5523 const char *SymName = S->getSymbol();
5524 if (Subtarget.isAIXABI()) {
5525 // If there exists a user-declared function whose name is the same as the
5526 // ExternalSymbol's, then we pick up the user-declared version.
5527 const Module *Mod = DAG.getMachineFunction().getFunction().getParent();
5528 if (const Function *F =
5529 dyn_cast_or_null<Function>(Val: Mod->getNamedValue(Name: SymName)))
5530 return getAIXFuncEntryPointSymbolSDNode(F);
5531
5532 // On AIX, direct function calls reference the symbol for the function's
5533 // entry point, which is named by prepending a "." before the function's
5534 // C-linkage name. A Qualname is returned here because an external
5535 // function entry point is a csect with XTY_ER property.
5536 const auto getExternalFunctionEntryPointSymbol = [&](StringRef SymName) {
5537 auto &Context = DAG.getMachineFunction().getContext();
5538 MCSectionXCOFF *Sec = Context.getXCOFFSection(
5539 Section: (Twine(".") + Twine(SymName)).str(), K: SectionKind::getMetadata(),
5540 CsectProp: XCOFF::CsectProperties(XCOFF::XMC_PR, XCOFF::XTY_ER));
5541 return Sec->getQualNameSymbol();
5542 };
5543
5544 SymName = getExternalFunctionEntryPointSymbol(SymName)->getName().data();
5545 }
5546 return DAG.getTargetExternalSymbol(Sym: SymName, VT: Callee.getValueType(),
5547 TargetFlags: UsePlt ? PPCII::MO_PLT : 0);
5548 }
5549
5550 // No transformation needed.
5551 assert(Callee.getNode() && "What no callee?");
5552 return Callee;
5553}
5554
5555static SDValue getOutputChainFromCallSeq(SDValue CallSeqStart) {
5556 assert(CallSeqStart.getOpcode() == ISD::CALLSEQ_START &&
5557 "Expected a CALLSEQ_STARTSDNode.");
5558
5559 // The last operand is the chain, except when the node has glue. If the node
5560 // has glue, then the last operand is the glue, and the chain is the second
5561 // last operand.
5562 SDValue LastValue = CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 1);
5563 if (LastValue.getValueType() != MVT::Glue)
5564 return LastValue;
5565
5566 return CallSeqStart.getValue(R: CallSeqStart->getNumValues() - 2);
5567}
5568
5569// Creates the node that moves a functions address into the count register
5570// to prepare for an indirect call instruction.
5571static void prepareIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5572 SDValue &Glue, SDValue &Chain,
5573 const SDLoc &dl) {
5574 SDValue MTCTROps[] = {Chain, Callee, Glue};
5575 EVT ReturnTypes[] = {MVT::Other, MVT::Glue};
5576 Chain = DAG.getNode(Opcode: PPCISD::MTCTR, DL: dl, ResultTys: ReturnTypes,
5577 Ops: ArrayRef(MTCTROps, Glue.getNode() ? 3 : 2));
5578 // The glue is the second value produced.
5579 Glue = Chain.getValue(R: 1);
5580}
5581
5582static void prepareDescriptorIndirectCall(SelectionDAG &DAG, SDValue &Callee,
5583 SDValue &Glue, SDValue &Chain,
5584 SDValue CallSeqStart,
5585 const CallBase *CB, const SDLoc &dl,
5586 bool hasNest,
5587 const PPCSubtarget &Subtarget) {
5588 // Function pointers in the 64-bit SVR4 ABI do not point to the function
5589 // entry point, but to the function descriptor (the function entry point
5590 // address is part of the function descriptor though).
5591 // The function descriptor is a three doubleword structure with the
5592 // following fields: function entry point, TOC base address and
5593 // environment pointer.
5594 // Thus for a call through a function pointer, the following actions need
5595 // to be performed:
5596 // 1. Save the TOC of the caller in the TOC save area of its stack
5597 // frame (this is done in LowerCall_Darwin() or LowerCall_64SVR4()).
5598 // 2. Load the address of the function entry point from the function
5599 // descriptor.
5600 // 3. Load the TOC of the callee from the function descriptor into r2.
5601 // 4. Load the environment pointer from the function descriptor into
5602 // r11.
5603 // 5. Branch to the function entry point address.
5604 // 6. On return of the callee, the TOC of the caller needs to be
5605 // restored (this is done in FinishCall()).
5606 //
5607 // The loads are scheduled at the beginning of the call sequence, and the
5608 // register copies are flagged together to ensure that no other
5609 // operations can be scheduled in between. E.g. without flagging the
5610 // copies together, a TOC access in the caller could be scheduled between
5611 // the assignment of the callee TOC and the branch to the callee, which leads
5612 // to incorrect code.
5613
5614 // Start by loading the function address from the descriptor.
5615 SDValue LDChain = getOutputChainFromCallSeq(CallSeqStart);
5616 auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors()
5617 ? (MachineMemOperand::MODereferenceable |
5618 MachineMemOperand::MOInvariant)
5619 : MachineMemOperand::MONone;
5620
5621 MachinePointerInfo MPI(CB ? CB->getCalledOperand() : nullptr);
5622
5623 // Registers used in building the DAG.
5624 const MCRegister EnvPtrReg = Subtarget.getEnvironmentPointerRegister();
5625 const MCRegister TOCReg = Subtarget.getTOCPointerRegister();
5626
5627 // Offsets of descriptor members.
5628 const unsigned TOCAnchorOffset = Subtarget.descriptorTOCAnchorOffset();
5629 const unsigned EnvPtrOffset = Subtarget.descriptorEnvironmentPointerOffset();
5630
5631 const MVT RegVT = Subtarget.getScalarIntVT();
5632 const Align Alignment = Subtarget.isPPC64() ? Align(8) : Align(4);
5633
5634 // One load for the functions entry point address.
5635 SDValue LoadFuncPtr = DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: Callee, PtrInfo: MPI,
5636 Alignment, MMOFlags);
5637
5638 // One for loading the TOC anchor for the module that contains the called
5639 // function.
5640 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCAnchorOffset, DL: dl);
5641 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: TOCOff);
5642 SDValue TOCPtr =
5643 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddTOC,
5644 PtrInfo: MPI.getWithOffset(O: TOCAnchorOffset), Alignment, MMOFlags);
5645
5646 // One for loading the environment pointer.
5647 SDValue PtrOff = DAG.getIntPtrConstant(Val: EnvPtrOffset, DL: dl);
5648 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: Callee, N2: PtrOff);
5649 SDValue LoadEnvPtr =
5650 DAG.getLoad(VT: RegVT, dl, Chain: LDChain, Ptr: AddPtr,
5651 PtrInfo: MPI.getWithOffset(O: EnvPtrOffset), Alignment, MMOFlags);
5652
5653
5654 // Then copy the newly loaded TOC anchor to the TOC pointer.
5655 SDValue TOCVal = DAG.getCopyToReg(Chain, dl, Reg: TOCReg, N: TOCPtr, Glue);
5656 Chain = TOCVal.getValue(R: 0);
5657 Glue = TOCVal.getValue(R: 1);
5658
5659 // If the function call has an explicit 'nest' parameter, it takes the
5660 // place of the environment pointer.
5661 assert((!hasNest || !Subtarget.isAIXABI()) &&
5662 "Nest parameter is not supported on AIX.");
5663 if (!hasNest) {
5664 SDValue EnvVal = DAG.getCopyToReg(Chain, dl, Reg: EnvPtrReg, N: LoadEnvPtr, Glue);
5665 Chain = EnvVal.getValue(R: 0);
5666 Glue = EnvVal.getValue(R: 1);
5667 }
5668
5669 // The rest of the indirect call sequence is the same as the non-descriptor
5670 // DAG.
5671 prepareIndirectCall(DAG, Callee&: LoadFuncPtr, Glue, Chain, dl);
5672}
5673
5674static void prepareOutOfLineGlueCall(SelectionDAG &DAG, SDValue &Callee,
5675 SDValue &Glue, SDValue &Chain,
5676 SDValue CallSeqStart, const CallBase *CB,
5677 const SDLoc &dl, bool hasNest,
5678 const PPCSubtarget &Subtarget) {
5679 // On AIX there is a feature ("out of line glue code") which uses a special
5680 // trampoline function ._ptrgl to do the indirect call. If this option is
5681 // enabled we instead simply load the address of the descriptor into gpr11,
5682 // with the arguments in the 'normal' registers and branch to the ._ptrgl
5683 // stub.
5684 const MCRegister PtrGlueReg = Subtarget.getGlueCodeDescriptorRegister();
5685 SDValue MoveToPhysicalReg =
5686 DAG.getCopyToReg(Chain, dl, Reg: PtrGlueReg, N: Callee, Glue);
5687 Chain = MoveToPhysicalReg.getValue(R: 0);
5688 Glue = MoveToPhysicalReg.getValue(R: 1);
5689}
5690
5691static void
5692buildCallOperands(SmallVectorImpl<SDValue> &Ops,
5693 PPCTargetLowering::CallFlags CFlags, const SDLoc &dl,
5694 SelectionDAG &DAG,
5695 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass,
5696 SDValue Glue, SDValue Chain, SDValue &Callee, int SPDiff,
5697 const PPCSubtarget &Subtarget) {
5698 const bool IsPPC64 = Subtarget.isPPC64();
5699 // MVT for a general purpose register.
5700 const MVT RegVT = Subtarget.getScalarIntVT();
5701
5702 // First operand is always the chain.
5703 Ops.push_back(Elt: Chain);
5704
5705 // If it's a direct call pass the callee as the second operand.
5706 if (!CFlags.IsIndirect)
5707 Ops.push_back(Elt: Callee);
5708 else if (Subtarget.usePointerGlueHelper()) {
5709 Ops.push_back(Elt: Callee);
5710 // Add the register used to pass the descriptor address.
5711 Ops.push_back(
5712 Elt: DAG.getRegister(Reg: Subtarget.getGlueCodeDescriptorRegister(), VT: RegVT));
5713 } else {
5714 assert(!CFlags.IsPatchPoint && "Patch point calls are not indirect.");
5715
5716 // For the TOC based ABIs, we have saved the TOC pointer to the linkage area
5717 // on the stack (this would have been done in `LowerCall_64SVR4` or
5718 // `LowerCall_AIX`). The call instruction is a pseudo instruction that
5719 // represents both the indirect branch and a load that restores the TOC
5720 // pointer from the linkage area. The operand for the TOC restore is an add
5721 // of the TOC save offset to the stack pointer. This must be the second
5722 // operand: after the chain input but before any other variadic arguments.
5723 // For 64-bit ELFv2 ABI with PCRel, do not restore the TOC as it is not
5724 // saved or used.
5725 if (isTOCSaveRestoreRequired(Subtarget)) {
5726 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
5727
5728 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: RegVT);
5729 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
5730 SDValue TOCOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
5731 SDValue AddTOC = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RegVT, N1: StackPtr, N2: TOCOff);
5732 Ops.push_back(Elt: AddTOC);
5733 }
5734
5735 // Add the register used for the environment pointer.
5736 if (Subtarget.usesFunctionDescriptors() && !CFlags.HasNest)
5737 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getEnvironmentPointerRegister(),
5738 VT: RegVT));
5739
5740
5741 // Add CTR register as callee so a bctr can be emitted later.
5742 if (CFlags.IsTailCall)
5743 Ops.push_back(Elt: DAG.getRegister(Reg: IsPPC64 ? PPC::CTR8 : PPC::CTR, VT: RegVT));
5744 }
5745
5746 // If this is a tail call add stack pointer delta.
5747 if (CFlags.IsTailCall)
5748 Ops.push_back(Elt: DAG.getConstant(Val: SPDiff, DL: dl, VT: MVT::i32));
5749
5750 // Add argument registers to the end of the list so that they are known live
5751 // into the call.
5752 for (const auto &[Reg, N] : RegsToPass)
5753 Ops.push_back(Elt: DAG.getRegister(Reg, VT: N.getValueType()));
5754
5755 // We cannot add R2/X2 as an operand here for PATCHPOINT, because there is
5756 // no way to mark dependencies as implicit here.
5757 // We will add the R2/X2 dependency in EmitInstrWithCustomInserter.
5758 if ((Subtarget.is64BitELFABI() || Subtarget.isAIXABI()) &&
5759 !CFlags.IsPatchPoint && !Subtarget.isUsingPCRelativeCalls())
5760 Ops.push_back(Elt: DAG.getRegister(Reg: Subtarget.getTOCPointerRegister(), VT: RegVT));
5761
5762 // Add implicit use of CR bit 6 for 32-bit SVR4 vararg calls
5763 if (CFlags.IsVarArg && Subtarget.is32BitELFABI())
5764 Ops.push_back(Elt: DAG.getRegister(Reg: PPC::CR1EQ, VT: MVT::i32));
5765
5766 // Add a register mask operand representing the call-preserved registers.
5767 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
5768 const uint32_t *Mask =
5769 TRI->getCallPreservedMask(MF: DAG.getMachineFunction(), CFlags.CallConv);
5770 assert(Mask && "Missing call preserved mask for calling convention");
5771 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
5772
5773 // If the glue is valid, it is the last operand.
5774 if (Glue.getNode())
5775 Ops.push_back(Elt: Glue);
5776}
5777
5778SDValue PPCTargetLowering::FinishCall(
5779 CallFlags CFlags, const SDLoc &dl, SelectionDAG &DAG,
5780 SmallVector<std::pair<unsigned, SDValue>, 8> &RegsToPass, SDValue Glue,
5781 SDValue Chain, SDValue CallSeqStart, SDValue &Callee, int SPDiff,
5782 unsigned NumBytes, const SmallVectorImpl<ISD::InputArg> &Ins,
5783 SmallVectorImpl<SDValue> &InVals, const CallBase *CB) const {
5784
5785 if ((Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls()) ||
5786 Subtarget.isAIXABI())
5787 setUsesTOCBasePtr(DAG);
5788
5789 unsigned CallOpc =
5790 getCallOpcode(CFlags, Caller: DAG.getMachineFunction().getFunction(), Callee,
5791 Subtarget, TM: DAG.getTarget(), IsStrictFPCall: CB ? CB->isStrictFP() : false);
5792
5793 if (!CFlags.IsIndirect)
5794 Callee = transformCallee(Callee, DAG, dl, Subtarget);
5795 else if (Subtarget.usesFunctionDescriptors()) {
5796 if (Subtarget.usePointerGlueHelper()) {
5797 prepareOutOfLineGlueCall(DAG, Callee, Glue, Chain, CallSeqStart, CB, dl,
5798 hasNest: CFlags.HasNest, Subtarget);
5799 SDValue PtrGlueCallee =
5800 DAG.getExternalSymbol(Sym: "_ptrgl", VT: getPointerTy(DL: DAG.getDataLayout()));
5801 Callee = transformCallee(Callee: PtrGlueCallee, DAG, dl, Subtarget);
5802 } else {
5803 prepareDescriptorIndirectCall(DAG, Callee, Glue, Chain, CallSeqStart, CB,
5804 dl, hasNest: CFlags.HasNest, Subtarget);
5805 }
5806 } else {
5807 prepareIndirectCall(DAG, Callee, Glue, Chain, dl);
5808 }
5809
5810 // Build the operand list for the call instruction.
5811 SmallVector<SDValue, 8> Ops;
5812 buildCallOperands(Ops, CFlags, dl, DAG, RegsToPass, Glue, Chain, Callee,
5813 SPDiff, Subtarget);
5814
5815 // Emit tail call.
5816 if (CFlags.IsTailCall) {
5817 // Indirect tail call when using PC Relative calls do not have the same
5818 // constraints.
5819 assert(((Callee.getOpcode() == ISD::Register &&
5820 cast<RegisterSDNode>(Callee)->getReg() == PPC::CTR) ||
5821 Callee.getOpcode() == ISD::TargetExternalSymbol ||
5822 Callee.getOpcode() == ISD::TargetGlobalAddress ||
5823 isa<ConstantSDNode>(Callee) ||
5824 (CFlags.IsIndirect && Subtarget.isUsingPCRelativeCalls())) &&
5825 "Expecting a global address, external symbol, absolute value, "
5826 "register or an indirect tail call when PC Relative calls are "
5827 "used.");
5828 // PC Relative calls also use TC_RETURN as the way to mark tail calls.
5829 assert(CallOpc == PPCISD::TC_RETURN &&
5830 "Unexpected call opcode for a tail call.");
5831 DAG.getMachineFunction().getFrameInfo().setHasTailCall();
5832 SDValue Ret = DAG.getNode(Opcode: CallOpc, DL: dl, VT: MVT::Other, Ops);
5833 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CFlags.NoMerge);
5834 return Ret;
5835 }
5836
5837 std::array<EVT, 2> ReturnTypes = {._M_elems: {MVT::Other, MVT::Glue}};
5838 Chain = DAG.getNode(Opcode: CallOpc, DL: dl, ResultTys: ReturnTypes, Ops);
5839 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CFlags.NoMerge);
5840 Glue = Chain.getValue(R: 1);
5841
5842 // When performing tail call optimization the callee pops its arguments off
5843 // the stack. Account for this here so these bytes can be pushed back on in
5844 // PPCFrameLowering::eliminateCallFramePseudoInstr.
5845 int BytesCalleePops = (CFlags.CallConv == CallingConv::Fast &&
5846 getTargetMachine().Options.GuaranteedTailCallOpt)
5847 ? NumBytes
5848 : 0;
5849
5850 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: BytesCalleePops, Glue, DL: dl);
5851 Glue = Chain.getValue(R: 1);
5852
5853 return LowerCallResult(Chain, InGlue: Glue, CallConv: CFlags.CallConv, isVarArg: CFlags.IsVarArg, Ins, dl,
5854 DAG, InVals);
5855}
5856
5857bool PPCTargetLowering::supportsTailCallFor(const CallBase *CB) const {
5858 CallingConv::ID CalleeCC = CB->getCallingConv();
5859 const Function *CallerFunc = CB->getCaller();
5860 CallingConv::ID CallerCC = CallerFunc->getCallingConv();
5861 const Function *CalleeFunc = CB->getCalledFunction();
5862 if (!CalleeFunc)
5863 return false;
5864 const GlobalValue *CalleeGV = dyn_cast<GlobalValue>(Val: CalleeFunc);
5865
5866 SmallVector<ISD::OutputArg, 2> Outs;
5867 SmallVector<ISD::InputArg, 2> Ins;
5868
5869 GetReturnInfo(CC: CalleeCC, ReturnType: CalleeFunc->getReturnType(),
5870 attr: CalleeFunc->getAttributes(), Outs, TLI: *this,
5871 DL: CalleeFunc->getDataLayout());
5872
5873 return isEligibleForTCO(CalleeGV, CalleeCC, CallerCC, CB,
5874 isVarArg: CalleeFunc->isVarArg(), Outs, Ins, CallerFunc,
5875 isCalleeExternalSymbol: false /*isCalleeExternalSymbol*/);
5876}
5877
5878bool PPCTargetLowering::isEligibleForTCO(
5879 const GlobalValue *CalleeGV, CallingConv::ID CalleeCC,
5880 CallingConv::ID CallerCC, const CallBase *CB, bool isVarArg,
5881 const SmallVectorImpl<ISD::OutputArg> &Outs,
5882 const SmallVectorImpl<ISD::InputArg> &Ins, const Function *CallerFunc,
5883 bool isCalleeExternalSymbol) const {
5884 if (Subtarget.useLongCalls() && !(CB && CB->isMustTailCall()))
5885 return false;
5886
5887 if (Subtarget.isSVR4ABI() && Subtarget.isPPC64())
5888 return IsEligibleForTailCallOptimization_64SVR4(
5889 CalleeGV, CalleeCC, CallerCC, CB, isVarArg, Outs, Ins, CallerFunc,
5890 isCalleeExternalSymbol);
5891 else
5892 return IsEligibleForTailCallOptimization(CalleeGV, CalleeCC, CallerCC,
5893 isVarArg, Ins);
5894}
5895
5896SDValue
5897PPCTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI,
5898 SmallVectorImpl<SDValue> &InVals) const {
5899 SelectionDAG &DAG = CLI.DAG;
5900 SDLoc &dl = CLI.DL;
5901 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
5902 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
5903 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
5904 SDValue Chain = CLI.Chain;
5905 SDValue Callee = CLI.Callee;
5906 bool &isTailCall = CLI.IsTailCall;
5907 CallingConv::ID CallConv = CLI.CallConv;
5908 bool isVarArg = CLI.IsVarArg;
5909 bool isPatchPoint = CLI.IsPatchPoint;
5910 const CallBase *CB = CLI.CB;
5911
5912 if (isTailCall) {
5913 MachineFunction &MF = DAG.getMachineFunction();
5914 CallingConv::ID CallerCC = MF.getFunction().getCallingConv();
5915 auto *G = dyn_cast<GlobalAddressSDNode>(Val&: Callee);
5916 const GlobalValue *GV = G ? G->getGlobal() : nullptr;
5917 bool IsCalleeExternalSymbol = isa<ExternalSymbolSDNode>(Val: Callee);
5918
5919 isTailCall =
5920 isEligibleForTCO(CalleeGV: GV, CalleeCC: CallConv, CallerCC, CB, isVarArg, Outs, Ins,
5921 CallerFunc: &(MF.getFunction()), isCalleeExternalSymbol: IsCalleeExternalSymbol);
5922 if (isTailCall) {
5923 ++NumTailCalls;
5924 if (!getTargetMachine().Options.GuaranteedTailCallOpt)
5925 ++NumSiblingCalls;
5926
5927 // PC Relative calls no longer guarantee that the callee is a Global
5928 // Address Node. The callee could be an indirect tail call in which
5929 // case the SDValue for the callee could be a load (to load the address
5930 // of a function pointer) or it may be a register copy (to move the
5931 // address of the callee from a function parameter into a virtual
5932 // register). It may also be an ExternalSymbolSDNode (ex memcopy).
5933 assert((Subtarget.isUsingPCRelativeCalls() ||
5934 isa<GlobalAddressSDNode>(Callee)) &&
5935 "Callee should be an llvm::Function object.");
5936
5937 LLVM_DEBUG(dbgs() << "TCO caller: " << DAG.getMachineFunction().getName()
5938 << "\nTCO callee: ");
5939 LLVM_DEBUG(Callee.dump());
5940 }
5941 }
5942
5943 if (!isTailCall && CB && CB->isMustTailCall())
5944 report_fatal_error(reason: "failed to perform tail call elimination on a call "
5945 "site marked musttail");
5946
5947 // When long calls (i.e. indirect calls) are always used, calls are always
5948 // made via function pointer. If we have a function name, first translate it
5949 // into a pointer.
5950 if (Subtarget.useLongCalls() && isa<GlobalAddressSDNode>(Val: Callee) &&
5951 !isTailCall)
5952 Callee = LowerGlobalAddress(Op: Callee, DAG);
5953
5954 CallFlags CFlags(
5955 CallConv, isTailCall, isVarArg, isPatchPoint,
5956 isIndirectCall(Callee, DAG, Subtarget, isPatchPoint),
5957 // hasNest
5958 Subtarget.is64BitELFABI() &&
5959 any_of(Range&: Outs, P: [](ISD::OutputArg Arg) { return Arg.Flags.isNest(); }),
5960 CLI.NoMerge);
5961
5962 if (Subtarget.isAIXABI())
5963 return LowerCall_AIX(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5964 InVals, CB);
5965
5966 assert(Subtarget.isSVR4ABI());
5967 if (Subtarget.isPPC64())
5968 return LowerCall_64SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5969 InVals, CB);
5970 return LowerCall_32SVR4(Chain, Callee, CFlags, Outs, OutVals, Ins, dl, DAG,
5971 InVals, CB);
5972}
5973
5974SDValue PPCTargetLowering::LowerCall_32SVR4(
5975 SDValue Chain, SDValue Callee, CallFlags CFlags,
5976 const SmallVectorImpl<ISD::OutputArg> &Outs,
5977 const SmallVectorImpl<SDValue> &OutVals,
5978 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
5979 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
5980 const CallBase *CB) const {
5981 // See PPCTargetLowering::LowerFormalArguments_32SVR4() for a description
5982 // of the 32-bit SVR4 ABI stack frame layout.
5983
5984 const CallingConv::ID CallConv = CFlags.CallConv;
5985 const bool IsVarArg = CFlags.IsVarArg;
5986 const bool IsTailCall = CFlags.IsTailCall;
5987
5988 assert((CallConv == CallingConv::C ||
5989 CallConv == CallingConv::Cold ||
5990 CallConv == CallingConv::Fast) && "Unknown calling convention!");
5991
5992 const Align PtrAlign(4);
5993
5994 MachineFunction &MF = DAG.getMachineFunction();
5995
5996 // Mark this function as potentially containing a function that contains a
5997 // tail call. As a consequence the frame pointer will be used for dynamicalloc
5998 // and restoring the callers stack pointer in this functions epilog. This is
5999 // done because by tail calling the called function might overwrite the value
6000 // in this function's (MF) stack pointer stack slot 0(SP).
6001 if (getTargetMachine().Options.GuaranteedTailCallOpt &&
6002 CallConv == CallingConv::Fast)
6003 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
6004
6005 // Count how many bytes are to be pushed on the stack, including the linkage
6006 // area, parameter list area and the part of the local variable space which
6007 // contains copies of aggregates which are passed by value.
6008
6009 // Assign locations to all of the outgoing arguments.
6010 SmallVector<CCValAssign, 16> ArgLocs;
6011 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
6012
6013 // Reserve space for the linkage area on the stack.
6014 CCInfo.AllocateStack(Size: Subtarget.getFrameLowering()->getLinkageSize(),
6015 Alignment: PtrAlign);
6016
6017 if (IsVarArg) {
6018 // Handle fixed and variable vector arguments differently.
6019 // Fixed vector arguments go into registers as long as registers are
6020 // available. Variable vector arguments always go into memory.
6021 unsigned NumArgs = Outs.size();
6022
6023 for (unsigned i = 0; i != NumArgs; ++i) {
6024 MVT ArgVT = Outs[i].VT;
6025 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
6026 bool Result;
6027
6028 if (!ArgFlags.isVarArg()) {
6029 Result = CC_PPC32_SVR4(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full, ArgFlags,
6030 OrigTy: Outs[i].OrigTy, State&: CCInfo);
6031 } else {
6032 Result = CC_PPC32_SVR4_VarArg(ValNo: i, ValVT: ArgVT, LocVT: ArgVT, LocInfo: CCValAssign::Full,
6033 ArgFlags, OrigTy: Outs[i].OrigTy, State&: CCInfo);
6034 }
6035
6036 if (Result) {
6037#ifndef NDEBUG
6038 errs() << "Call operand #" << i << " has unhandled type "
6039 << ArgVT << "\n";
6040#endif
6041 llvm_unreachable(nullptr);
6042 }
6043 }
6044 } else {
6045 // All arguments are treated the same.
6046 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4);
6047 }
6048
6049 // Assign locations to all of the outgoing aggregate by value arguments.
6050 SmallVector<CCValAssign, 16> ByValArgLocs;
6051 CCState CCByValInfo(CallConv, IsVarArg, MF, ByValArgLocs, *DAG.getContext());
6052
6053 // Reserve stack space for the allocations in CCInfo.
6054 CCByValInfo.AllocateStack(Size: CCInfo.getStackSize(), Alignment: PtrAlign);
6055
6056 CCByValInfo.AnalyzeCallOperands(Outs, Fn: CC_PPC32_SVR4_ByVal);
6057
6058 // Size of the linkage area, parameter list area and the part of the local
6059 // space variable where copies of aggregates which are passed by value are
6060 // stored.
6061 unsigned NumBytes = CCByValInfo.getStackSize();
6062
6063 // Calculate by how many bytes the stack has to be adjusted in case of tail
6064 // call optimization.
6065 int SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: IsTailCall, ParamSize: NumBytes);
6066
6067 // Adjust the stack pointer for the new arguments...
6068 // These operations are automatically eliminated by the prolog/epilog pass
6069 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6070 SDValue CallSeqStart = Chain;
6071
6072 // Load the return address and frame pointer so it can be moved somewhere else
6073 // later.
6074 SDValue LROp, FPOp;
6075 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6076
6077 // Set up a copy of the stack pointer for use loading and storing any
6078 // arguments that may not fit in the registers available for argument
6079 // passing.
6080 SDValue StackPtr = DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
6081
6082 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6083 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6084 SmallVector<SDValue, 8> MemOpChains;
6085
6086 bool seenFloatArg = false;
6087 // Walk the register/memloc assignments, inserting copies/loads.
6088 // i - Tracks the index into the list of registers allocated for the call
6089 // RealArgIdx - Tracks the index into the list of actual function arguments
6090 // j - Tracks the index into the list of byval arguments
6091 for (unsigned i = 0, RealArgIdx = 0, j = 0, e = ArgLocs.size();
6092 i != e;
6093 ++i, ++RealArgIdx) {
6094 CCValAssign &VA = ArgLocs[i];
6095 SDValue Arg = OutVals[RealArgIdx];
6096 ISD::ArgFlagsTy Flags = Outs[RealArgIdx].Flags;
6097
6098 if (Flags.isByVal()) {
6099 // Argument is an aggregate which is passed by value, thus we need to
6100 // create a copy of it in the local variable space of the current stack
6101 // frame (which is the stack frame of the caller) and pass the address of
6102 // this copy to the callee.
6103 assert((j < ByValArgLocs.size()) && "Index out of bounds!");
6104 CCValAssign &ByValVA = ByValArgLocs[j++];
6105 assert((VA.getValNo() == ByValVA.getValNo()) && "ValNo mismatch!");
6106
6107 // Memory reserved in the local variable space of the callers stack frame.
6108 unsigned LocMemOffset = ByValVA.getLocMemOffset();
6109
6110 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6111 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6112 N1: StackPtr, N2: PtrOff);
6113
6114 // Create a copy of the argument in the local area of the current
6115 // stack frame.
6116 SDValue MemcpyCall =
6117 CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6118 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6119 Flags, DAG, dl);
6120
6121 // This must go outside the CALLSEQ_START..END.
6122 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: NumBytes, OutSize: 0,
6123 DL: SDLoc(MemcpyCall));
6124 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6125 To: NewCallSeqStart.getNode());
6126 Chain = CallSeqStart = NewCallSeqStart;
6127
6128 // Pass the address of the aggregate copy on the stack either in a
6129 // physical register or in the parameter list area of the current stack
6130 // frame to the callee.
6131 Arg = PtrOff;
6132 }
6133
6134 // When useCRBits() is true, there can be i1 arguments.
6135 // It is because getRegisterType(MVT::i1) => MVT::i1,
6136 // and for other integer types getRegisterType() => MVT::i32.
6137 // Extend i1 and ensure callee will get i32.
6138 if (Arg.getValueType() == MVT::i1)
6139 Arg = DAG.getNode(Opcode: Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND,
6140 DL: dl, VT: MVT::i32, Operand: Arg);
6141
6142 if (VA.isRegLoc()) {
6143 seenFloatArg |= VA.getLocVT().isFloatingPoint();
6144 // Put argument in a physical register.
6145 if (Subtarget.hasSPE() && Arg.getValueType() == MVT::f64) {
6146 bool IsLE = Subtarget.isLittleEndian();
6147 SDValue SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6148 N2: DAG.getIntPtrConstant(Val: IsLE ? 0 : 1, DL: dl));
6149 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y: SVal.getValue(R: 0)));
6150 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
6151 N2: DAG.getIntPtrConstant(Val: IsLE ? 1 : 0, DL: dl));
6152 RegsToPass.push_back(Elt: std::make_pair(x: ArgLocs[++i].getLocReg(),
6153 y: SVal.getValue(R: 0)));
6154 } else
6155 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
6156 } else {
6157 // Put argument in the parameter list area of the current stack frame.
6158 assert(VA.isMemLoc());
6159 unsigned LocMemOffset = VA.getLocMemOffset();
6160
6161 if (!IsTailCall) {
6162 SDValue PtrOff = DAG.getIntPtrConstant(Val: LocMemOffset, DL: dl);
6163 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()),
6164 N1: StackPtr, N2: PtrOff);
6165
6166 MemOpChains.push_back(
6167 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo()));
6168 } else {
6169 // Calculate and remember argument location.
6170 CalculateTailCallArgDest(DAG, MF, IsPPC64: false, Arg, SPDiff, ArgOffset: LocMemOffset,
6171 TailCallArguments);
6172 }
6173 }
6174 }
6175
6176 if (!MemOpChains.empty())
6177 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6178
6179 // Build a sequence of copy-to-reg nodes chained together with token chain
6180 // and flag operands which copy the outgoing args into the appropriate regs.
6181 SDValue InGlue;
6182 for (const auto &[Reg, N] : RegsToPass) {
6183 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6184 InGlue = Chain.getValue(R: 1);
6185 }
6186
6187 // Set CR bit 6 to true if this is a vararg call with floating args passed in
6188 // registers.
6189 if (IsVarArg) {
6190 SDVTList VTs = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
6191 SDValue Ops[] = { Chain, InGlue };
6192
6193 Chain = DAG.getNode(Opcode: seenFloatArg ? PPCISD::CR6SET : PPCISD::CR6UNSET, DL: dl,
6194 VTList: VTs, Ops: ArrayRef(Ops, InGlue.getNode() ? 2 : 1));
6195
6196 InGlue = Chain.getValue(R: 1);
6197 }
6198
6199 if (IsTailCall)
6200 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6201 TailCallArguments);
6202
6203 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6204 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6205}
6206
6207// Copy an argument into memory, being careful to do this outside the
6208// call sequence for the call to which the argument belongs.
6209SDValue PPCTargetLowering::createMemcpyOutsideCallSeq(
6210 SDValue Arg, SDValue PtrOff, SDValue CallSeqStart, ISD::ArgFlagsTy Flags,
6211 SelectionDAG &DAG, const SDLoc &dl) const {
6212 SDValue MemcpyCall = CreateCopyOfByValArgument(Src: Arg, Dst: PtrOff,
6213 Chain: CallSeqStart.getNode()->getOperand(Num: 0),
6214 Flags, DAG, dl);
6215 // The MEMCPY must go outside the CALLSEQ_START..END.
6216 int64_t FrameSize = CallSeqStart.getConstantOperandVal(i: 1);
6217 SDValue NewCallSeqStart = DAG.getCALLSEQ_START(Chain: MemcpyCall, InSize: FrameSize, OutSize: 0,
6218 DL: SDLoc(MemcpyCall));
6219 DAG.ReplaceAllUsesWith(From: CallSeqStart.getNode(),
6220 To: NewCallSeqStart.getNode());
6221 return NewCallSeqStart;
6222}
6223
6224SDValue PPCTargetLowering::LowerCall_64SVR4(
6225 SDValue Chain, SDValue Callee, CallFlags CFlags,
6226 const SmallVectorImpl<ISD::OutputArg> &Outs,
6227 const SmallVectorImpl<SDValue> &OutVals,
6228 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
6229 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
6230 const CallBase *CB) const {
6231 bool isELFv2ABI = Subtarget.isELFv2ABI();
6232 bool isLittleEndian = Subtarget.isLittleEndian();
6233 unsigned NumOps = Outs.size();
6234 bool IsSibCall = false;
6235 bool IsFastCall = CFlags.CallConv == CallingConv::Fast;
6236
6237 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
6238 unsigned PtrByteSize = 8;
6239
6240 MachineFunction &MF = DAG.getMachineFunction();
6241
6242 if (CFlags.IsTailCall && !getTargetMachine().Options.GuaranteedTailCallOpt)
6243 IsSibCall = true;
6244
6245 // Mark this function as potentially containing a function that contains a
6246 // tail call. As a consequence the frame pointer will be used for dynamicalloc
6247 // and restoring the callers stack pointer in this functions epilog. This is
6248 // done because by tail calling the called function might overwrite the value
6249 // in this function's (MF) stack pointer stack slot 0(SP).
6250 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6251 MF.getInfo<PPCFunctionInfo>()->setHasFastCall();
6252
6253 assert(!(IsFastCall && CFlags.IsVarArg) &&
6254 "fastcc not supported on varargs functions");
6255
6256 // Count how many bytes are to be pushed on the stack, including the linkage
6257 // area, and parameter passing area. On ELFv1, the linkage area is 48 bytes
6258 // reserved space for [SP][CR][LR][2 x unused][TOC]; on ELFv2, the linkage
6259 // area is 32 bytes reserved space for [SP][CR][LR][TOC].
6260 unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
6261 unsigned NumBytes = LinkageSize;
6262 unsigned GPR_idx = 0, FPR_idx = 0, VR_idx = 0;
6263
6264 static const MCPhysReg GPR[] = {
6265 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6266 PPC::X7, PPC::X8, PPC::X9, PPC::X10,
6267 };
6268 static const MCPhysReg VR[] = {
6269 PPC::V2, PPC::V3, PPC::V4, PPC::V5, PPC::V6, PPC::V7, PPC::V8,
6270 PPC::V9, PPC::V10, PPC::V11, PPC::V12, PPC::V13
6271 };
6272
6273 const unsigned NumGPRs = std::size(GPR);
6274 const unsigned NumFPRs = useSoftFloat() ? 0 : 13;
6275 const unsigned NumVRs = std::size(VR);
6276
6277 // On ELFv2, we can avoid allocating the parameter area if all the arguments
6278 // can be passed to the callee in registers.
6279 // For the fast calling convention, there is another check below.
6280 // Note: We should keep consistent with LowerFormalArguments_64SVR4()
6281 bool HasParameterArea = !isELFv2ABI || CFlags.IsVarArg || IsFastCall;
6282 if (!HasParameterArea) {
6283 unsigned ParamAreaSize = NumGPRs * PtrByteSize;
6284 unsigned AvailableFPRs = NumFPRs;
6285 unsigned AvailableVRs = NumVRs;
6286 unsigned NumBytesTmp = NumBytes;
6287 for (unsigned i = 0; i != NumOps; ++i) {
6288 if (Outs[i].Flags.isNest()) continue;
6289 if (CalculateStackSlotUsed(ArgVT: Outs[i].VT, OrigVT: Outs[i].ArgVT, Flags: Outs[i].Flags,
6290 PtrByteSize, LinkageSize, ParamAreaSize,
6291 ArgOffset&: NumBytesTmp, AvailableFPRs, AvailableVRs))
6292 HasParameterArea = true;
6293 }
6294 }
6295
6296 // When using the fast calling convention, we don't provide backing for
6297 // arguments that will be in registers.
6298 unsigned NumGPRsUsed = 0, NumFPRsUsed = 0, NumVRsUsed = 0;
6299
6300 // Avoid allocating parameter area for fastcc functions if all the arguments
6301 // can be passed in the registers.
6302 if (IsFastCall)
6303 HasParameterArea = false;
6304
6305 // Add up all the space actually used.
6306 for (unsigned i = 0; i != NumOps; ++i) {
6307 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6308 EVT ArgVT = Outs[i].VT;
6309 EVT OrigVT = Outs[i].ArgVT;
6310
6311 if (Flags.isNest())
6312 continue;
6313
6314 if (IsFastCall) {
6315 if (Flags.isByVal()) {
6316 NumGPRsUsed += (Flags.getByValSize()+7)/8;
6317 if (NumGPRsUsed > NumGPRs)
6318 HasParameterArea = true;
6319 } else {
6320 switch (ArgVT.getSimpleVT().SimpleTy) {
6321 default: llvm_unreachable("Unexpected ValueType for argument!");
6322 case MVT::i1:
6323 case MVT::i32:
6324 case MVT::i64:
6325 if (++NumGPRsUsed <= NumGPRs)
6326 continue;
6327 break;
6328 case MVT::v4i32:
6329 case MVT::v8i16:
6330 case MVT::v16i8:
6331 case MVT::v2f64:
6332 case MVT::v2i64:
6333 case MVT::v1i128:
6334 case MVT::f128:
6335 if (++NumVRsUsed <= NumVRs)
6336 continue;
6337 break;
6338 case MVT::v4f32:
6339 if (++NumVRsUsed <= NumVRs)
6340 continue;
6341 break;
6342 case MVT::f32:
6343 case MVT::f64:
6344 if (++NumFPRsUsed <= NumFPRs)
6345 continue;
6346 break;
6347 }
6348 HasParameterArea = true;
6349 }
6350 }
6351
6352 /* Respect alignment of argument on the stack. */
6353 auto Alignement =
6354 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6355 NumBytes = alignTo(Size: NumBytes, A: Alignement);
6356
6357 NumBytes += CalculateStackSlotSize(ArgVT, Flags, PtrByteSize);
6358 if (Flags.isInConsecutiveRegsLast())
6359 NumBytes = ((NumBytes + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6360 }
6361
6362 unsigned NumBytesActuallyUsed = NumBytes;
6363
6364 // In the old ELFv1 ABI,
6365 // the prolog code of the callee may store up to 8 GPR argument registers to
6366 // the stack, allowing va_start to index over them in memory if its varargs.
6367 // Because we cannot tell if this is needed on the caller side, we have to
6368 // conservatively assume that it is needed. As such, make sure we have at
6369 // least enough stack space for the caller to store the 8 GPRs.
6370 // In the ELFv2 ABI, we allocate the parameter area iff a callee
6371 // really requires memory operands, e.g. a vararg function.
6372 if (HasParameterArea)
6373 NumBytes = std::max(a: NumBytes, b: LinkageSize + 8 * PtrByteSize);
6374 else
6375 NumBytes = LinkageSize;
6376
6377 // Tail call needs the stack to be aligned.
6378 if (getTargetMachine().Options.GuaranteedTailCallOpt && IsFastCall)
6379 NumBytes = EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes);
6380
6381 int SPDiff = 0;
6382
6383 // Calculate by how many bytes the stack has to be adjusted in case of tail
6384 // call optimization.
6385 if (!IsSibCall)
6386 SPDiff = CalculateTailCallSPDiff(DAG, isTailCall: CFlags.IsTailCall, ParamSize: NumBytes);
6387
6388 // To protect arguments on the stack from being clobbered in a tail call,
6389 // force all the loads to happen before doing any other lowering.
6390 if (CFlags.IsTailCall)
6391 Chain = DAG.getStackArgumentTokenFactor(Chain);
6392
6393 // Adjust the stack pointer for the new arguments...
6394 // These operations are automatically eliminated by the prolog/epilog pass
6395 if (!IsSibCall)
6396 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
6397 SDValue CallSeqStart = Chain;
6398
6399 // Load the return address and frame pointer so it can be move somewhere else
6400 // later.
6401 SDValue LROp, FPOp;
6402 Chain = EmitTailCallLoadFPAndRetAddr(DAG, SPDiff, Chain, LROpOut&: LROp, FPOpOut&: FPOp, dl);
6403
6404 // Set up a copy of the stack pointer for use loading and storing any
6405 // arguments that may not fit in the registers available for argument
6406 // passing.
6407 SDValue StackPtr = DAG.getRegister(Reg: PPC::X1, VT: MVT::i64);
6408
6409 // Figure out which arguments are going to go in registers, and which in
6410 // memory. Also, if this is a vararg function, floating point operations
6411 // must be stored to our stack, and loaded into integer regs as well, if
6412 // any integer regs are available for argument passing.
6413 unsigned ArgOffset = LinkageSize;
6414
6415 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
6416 SmallVector<TailCallArgumentInfo, 8> TailCallArguments;
6417
6418 SmallVector<SDValue, 8> MemOpChains;
6419 for (unsigned i = 0; i != NumOps; ++i) {
6420 SDValue Arg = OutVals[i];
6421 ISD::ArgFlagsTy Flags = Outs[i].Flags;
6422 EVT ArgVT = Outs[i].VT;
6423 EVT OrigVT = Outs[i].ArgVT;
6424
6425 // PtrOff will be used to store the current argument to the stack if a
6426 // register cannot be found for it.
6427 SDValue PtrOff;
6428
6429 // We re-align the argument offset for each argument, except when using the
6430 // fast calling convention, when we need to make sure we do that only when
6431 // we'll actually use a stack slot.
6432 auto ComputePtrOff = [&]() {
6433 /* Respect alignment of argument on the stack. */
6434 auto Alignment =
6435 CalculateStackSlotAlignment(ArgVT, OrigVT, Flags, PtrByteSize);
6436 ArgOffset = alignTo(Size: ArgOffset, A: Alignment);
6437
6438 PtrOff = DAG.getConstant(Val: ArgOffset, DL: dl, VT: StackPtr.getValueType());
6439
6440 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6441 };
6442
6443 if (!IsFastCall) {
6444 ComputePtrOff();
6445
6446 /* Compute GPR index associated with argument offset. */
6447 GPR_idx = (ArgOffset - LinkageSize) / PtrByteSize;
6448 GPR_idx = std::min(a: GPR_idx, b: NumGPRs);
6449 }
6450
6451 // Promote integers to 64-bit values.
6452 if (Arg.getValueType() == MVT::i32 || Arg.getValueType() == MVT::i1) {
6453 // FIXME: Should this use ANY_EXTEND if neither sext nor zext?
6454 unsigned ExtOp = Flags.isSExt() ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
6455 Arg = DAG.getNode(Opcode: ExtOp, DL: dl, VT: MVT::i64, Operand: Arg);
6456 }
6457
6458 // FIXME memcpy is used way more than necessary. Correctness first.
6459 // Note: "by value" is code for passing a structure by value, not
6460 // basic types.
6461 if (Flags.isByVal()) {
6462 // Note: Size includes alignment padding, so
6463 // struct x { short a; char b; }
6464 // will have Size = 4. With #pragma pack(1), it will have Size = 3.
6465 // These are the proper values we need for right-justifying the
6466 // aggregate in a parameter register.
6467 unsigned Size = Flags.getByValSize();
6468
6469 // An empty aggregate parameter takes up no storage and no
6470 // registers.
6471 if (Size == 0)
6472 continue;
6473
6474 if (IsFastCall)
6475 ComputePtrOff();
6476
6477 // All aggregates smaller than 8 bytes must be passed right-justified.
6478 if (Size==1 || Size==2 || Size==4) {
6479 EVT VT = (Size==1) ? MVT::i8 : ((Size==2) ? MVT::i16 : MVT::i32);
6480 if (GPR_idx != NumGPRs) {
6481 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: Arg,
6482 PtrInfo: MachinePointerInfo(), MemVT: VT);
6483 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6484 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6485
6486 ArgOffset += PtrByteSize;
6487 continue;
6488 }
6489 }
6490
6491 if (GPR_idx == NumGPRs && Size < 8) {
6492 SDValue AddPtr = PtrOff;
6493 if (!isLittleEndian) {
6494 SDValue Const = DAG.getConstant(Val: PtrByteSize - Size, DL: dl,
6495 VT: PtrOff.getValueType());
6496 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6497 }
6498 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6499 CallSeqStart,
6500 Flags, DAG, dl);
6501 ArgOffset += PtrByteSize;
6502 continue;
6503 }
6504 // Copy the object to parameter save area if it can not be entirely passed
6505 // by registers.
6506 // FIXME: we only need to copy the parts which need to be passed in
6507 // parameter save area. For the parts passed by registers, we don't need
6508 // to copy them to the stack although we need to allocate space for them
6509 // in parameter save area.
6510 if ((NumGPRs - GPR_idx) * PtrByteSize < Size)
6511 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff,
6512 CallSeqStart,
6513 Flags, DAG, dl);
6514
6515 // When a register is available, pass a small aggregate right-justified.
6516 if (Size < 8 && GPR_idx != NumGPRs) {
6517 // The easiest way to get this right-justified in a register
6518 // is to copy the structure into the rightmost portion of a
6519 // local variable slot, then load the whole slot into the
6520 // register.
6521 // FIXME: The memcpy seems to produce pretty awful code for
6522 // small aggregates, particularly for packed ones.
6523 // FIXME: It would be preferable to use the slot in the
6524 // parameter save area instead of a new local variable.
6525 SDValue AddPtr = PtrOff;
6526 if (!isLittleEndian) {
6527 SDValue Const = DAG.getConstant(Val: 8 - Size, DL: dl, VT: PtrOff.getValueType());
6528 AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: Const);
6529 }
6530 Chain = CallSeqStart = createMemcpyOutsideCallSeq(Arg, PtrOff: AddPtr,
6531 CallSeqStart,
6532 Flags, DAG, dl);
6533
6534 // Load the slot into the register.
6535 SDValue Load =
6536 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6537 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6538 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6539
6540 // Done with this argument.
6541 ArgOffset += PtrByteSize;
6542 continue;
6543 }
6544
6545 // For aggregates larger than PtrByteSize, copy the pieces of the
6546 // object that fit into registers from the parameter save area.
6547 for (unsigned j=0; j<Size; j+=PtrByteSize) {
6548 SDValue Const = DAG.getConstant(Val: j, DL: dl, VT: PtrOff.getValueType());
6549 SDValue AddArg = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: Arg, N2: Const);
6550 if (GPR_idx != NumGPRs) {
6551 unsigned LoadSizeInBits = std::min(a: PtrByteSize, b: (Size - j)) * 8;
6552 EVT ObjType = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: LoadSizeInBits);
6553 SDValue Load = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: PtrVT, Chain, Ptr: AddArg,
6554 PtrInfo: MachinePointerInfo(), MemVT: ObjType);
6555
6556 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6557 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6558 ArgOffset += PtrByteSize;
6559 } else {
6560 ArgOffset += ((Size - j + PtrByteSize-1)/PtrByteSize)*PtrByteSize;
6561 break;
6562 }
6563 }
6564 continue;
6565 }
6566
6567 switch (Arg.getSimpleValueType().SimpleTy) {
6568 default: llvm_unreachable("Unexpected ValueType for argument!");
6569 case MVT::i1:
6570 case MVT::i32:
6571 case MVT::i64:
6572 if (Flags.isNest()) {
6573 // The 'nest' parameter, if any, is passed in R11.
6574 RegsToPass.push_back(Elt: std::make_pair(x: PPC::X11, y&: Arg));
6575 break;
6576 }
6577
6578 // These can be scalar arguments or elements of an integer array type
6579 // passed directly. Clang may use those instead of "byval" aggregate
6580 // types to avoid forcing arguments to memory unnecessarily.
6581 if (GPR_idx != NumGPRs) {
6582 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Arg));
6583 } else {
6584 if (IsFastCall)
6585 ComputePtrOff();
6586
6587 assert(HasParameterArea &&
6588 "Parameter area must exist to pass an argument in memory.");
6589 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6590 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6591 TailCallArguments, dl);
6592 if (IsFastCall)
6593 ArgOffset += PtrByteSize;
6594 }
6595 if (!IsFastCall)
6596 ArgOffset += PtrByteSize;
6597 break;
6598 case MVT::f32:
6599 case MVT::f64: {
6600 // These can be scalar arguments or elements of a float array type
6601 // passed directly. The latter are used to implement ELFv2 homogenous
6602 // float aggregates.
6603
6604 // Named arguments go into FPRs first, and once they overflow, the
6605 // remaining arguments go into GPRs and then the parameter save area.
6606 // Unnamed arguments for vararg functions always go to GPRs and
6607 // then the parameter save area. For now, put all arguments to vararg
6608 // routines always in both locations (FPR *and* GPR or stack slot).
6609 bool NeedGPROrStack = CFlags.IsVarArg || FPR_idx == NumFPRs;
6610 bool NeededLoad = false;
6611
6612 // First load the argument into the next available FPR.
6613 if (FPR_idx != NumFPRs)
6614 RegsToPass.push_back(Elt: std::make_pair(x: FPR[FPR_idx++], y&: Arg));
6615
6616 // Next, load the argument into GPR or stack slot if needed.
6617 if (!NeedGPROrStack)
6618 ;
6619 else if (GPR_idx != NumGPRs && !IsFastCall) {
6620 // FIXME: We may want to re-enable this for CallingConv::Fast on the P8
6621 // once we support fp <-> gpr moves.
6622
6623 // In the non-vararg case, this can only ever happen in the
6624 // presence of f32 array types, since otherwise we never run
6625 // out of FPRs before running out of GPRs.
6626 SDValue ArgVal;
6627
6628 // Double values are always passed in a single GPR.
6629 if (Arg.getValueType() != MVT::f32) {
6630 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: Arg);
6631
6632 // Non-array float values are extended and passed in a GPR.
6633 } else if (!Flags.isInConsecutiveRegs()) {
6634 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6635 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6636
6637 // If we have an array of floats, we collect every odd element
6638 // together with its predecessor into one GPR.
6639 } else if (ArgOffset % PtrByteSize != 0) {
6640 SDValue Lo, Hi;
6641 Lo = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: OutVals[i - 1]);
6642 Hi = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6643 if (!isLittleEndian)
6644 std::swap(a&: Lo, b&: Hi);
6645 ArgVal = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
6646
6647 // The final element, if even, goes into the first half of a GPR.
6648 } else if (Flags.isInConsecutiveRegsLast()) {
6649 ArgVal = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i32, Operand: Arg);
6650 ArgVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: ArgVal);
6651 if (!isLittleEndian)
6652 ArgVal = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: ArgVal,
6653 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i32));
6654
6655 // Non-final even elements are skipped; they will be handled
6656 // together the with subsequent argument on the next go-around.
6657 } else
6658 ArgVal = SDValue();
6659
6660 if (ArgVal.getNode())
6661 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: ArgVal));
6662 } else {
6663 if (IsFastCall)
6664 ComputePtrOff();
6665
6666 // Single-precision floating-point values are mapped to the
6667 // second (rightmost) word of the stack doubleword.
6668 if (Arg.getValueType() == MVT::f32 &&
6669 !isLittleEndian && !Flags.isInConsecutiveRegs()) {
6670 SDValue ConstFour = DAG.getConstant(Val: 4, DL: dl, VT: PtrOff.getValueType());
6671 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff, N2: ConstFour);
6672 }
6673
6674 assert(HasParameterArea &&
6675 "Parameter area must exist to pass an argument in memory.");
6676 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6677 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: false, MemOpChains,
6678 TailCallArguments, dl);
6679
6680 NeededLoad = true;
6681 }
6682 // When passing an array of floats, the array occupies consecutive
6683 // space in the argument area; only round up to the next doubleword
6684 // at the end of the array. Otherwise, each float takes 8 bytes.
6685 if (!IsFastCall || NeededLoad) {
6686 ArgOffset += (Arg.getValueType() == MVT::f32 &&
6687 Flags.isInConsecutiveRegs()) ? 4 : 8;
6688 if (Flags.isInConsecutiveRegsLast())
6689 ArgOffset = ((ArgOffset + PtrByteSize - 1)/PtrByteSize) * PtrByteSize;
6690 }
6691 break;
6692 }
6693 case MVT::v4f32:
6694 case MVT::v4i32:
6695 case MVT::v8i16:
6696 case MVT::v16i8:
6697 case MVT::v2f64:
6698 case MVT::v2i64:
6699 case MVT::v1i128:
6700 case MVT::f128:
6701 // These can be scalar arguments or elements of a vector array type
6702 // passed directly. The latter are used to implement ELFv2 homogenous
6703 // vector aggregates.
6704
6705 // For a varargs call, named arguments go into VRs or on the stack as
6706 // usual; unnamed arguments always go to the stack or the corresponding
6707 // GPRs when within range. For now, we always put the value in both
6708 // locations (or even all three).
6709 if (CFlags.IsVarArg) {
6710 assert(HasParameterArea &&
6711 "Parameter area must exist if we have a varargs call.");
6712 // We could elide this store in the case where the object fits
6713 // entirely in R registers. Maybe later.
6714 SDValue Store =
6715 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6716 MemOpChains.push_back(Elt: Store);
6717 if (VR_idx != NumVRs) {
6718 SDValue Load =
6719 DAG.getLoad(VT: MVT::v4f32, dl, Chain: Store, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
6720 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6721 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Load));
6722 }
6723 ArgOffset += 16;
6724 for (unsigned i=0; i<16; i+=PtrByteSize) {
6725 if (GPR_idx == NumGPRs)
6726 break;
6727 SDValue Ix = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
6728 N2: DAG.getConstant(Val: i, DL: dl, VT: PtrVT));
6729 SDValue Load =
6730 DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Ix, PtrInfo: MachinePointerInfo());
6731 MemOpChains.push_back(Elt: Load.getValue(R: 1));
6732 RegsToPass.push_back(Elt: std::make_pair(x: GPR[GPR_idx++], y&: Load));
6733 }
6734 break;
6735 }
6736
6737 // Non-varargs Altivec params go into VRs or on the stack.
6738 if (VR_idx != NumVRs) {
6739 RegsToPass.push_back(Elt: std::make_pair(x: VR[VR_idx++], y&: Arg));
6740 } else {
6741 if (IsFastCall)
6742 ComputePtrOff();
6743
6744 assert(HasParameterArea &&
6745 "Parameter area must exist to pass an argument in memory.");
6746 LowerMemOpCallTo(DAG, MF, Chain, Arg, PtrOff, SPDiff, ArgOffset,
6747 isPPC64: true, isTailCall: CFlags.IsTailCall, isVector: true, MemOpChains,
6748 TailCallArguments, dl);
6749 if (IsFastCall)
6750 ArgOffset += 16;
6751 }
6752
6753 if (!IsFastCall)
6754 ArgOffset += 16;
6755 break;
6756 }
6757 }
6758
6759 assert((!HasParameterArea || NumBytesActuallyUsed == ArgOffset) &&
6760 "mismatch in size of parameter area");
6761 (void)NumBytesActuallyUsed;
6762
6763 if (!MemOpChains.empty())
6764 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
6765
6766 // Check if this is an indirect call (MTCTR/BCTRL).
6767 // See prepareDescriptorIndirectCall and buildCallOperands for more
6768 // information about calls through function pointers in the 64-bit SVR4 ABI.
6769 if (CFlags.IsIndirect) {
6770 // For 64-bit ELFv2 ABI with PCRel, do not save the TOC of the
6771 // caller in the TOC save area.
6772 if (isTOCSaveRestoreRequired(Subtarget)) {
6773 assert(!CFlags.IsTailCall && "Indirect tails calls not supported");
6774 // Load r2 into a virtual register and store it to the TOC save area.
6775 setUsesTOCBasePtr(DAG);
6776 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: PPC::X2, VT: MVT::i64);
6777 // TOC save area offset.
6778 unsigned TOCSaveOffset = Subtarget.getFrameLowering()->getTOCSaveOffset();
6779 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
6780 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
6781 Chain = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
6782 PtrInfo: MachinePointerInfo::getStack(
6783 MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
6784 }
6785 // In the ELFv2 ABI, R12 must contain the address of an indirect callee.
6786 // This does not mean the MTCTR instruction must use R12; it's easier
6787 // to model this as an extra parameter, so do that.
6788 if (isELFv2ABI && !CFlags.IsPatchPoint)
6789 RegsToPass.push_back(Elt: std::make_pair(x: (unsigned)PPC::X12, y&: Callee));
6790 }
6791
6792 // Build a sequence of copy-to-reg nodes chained together with token chain
6793 // and flag operands which copy the outgoing args into the appropriate regs.
6794 SDValue InGlue;
6795 for (const auto &[Reg, N] : RegsToPass) {
6796 Chain = DAG.getCopyToReg(Chain, dl, Reg, N, Glue: InGlue);
6797 InGlue = Chain.getValue(R: 1);
6798 }
6799
6800 if (CFlags.IsTailCall && !IsSibCall)
6801 PrepareTailCall(DAG, InGlue, Chain, dl, SPDiff, NumBytes, LROp, FPOp,
6802 TailCallArguments);
6803
6804 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
6805 Callee, SPDiff, NumBytes, Ins, InVals, CB);
6806}
6807
6808// Returns true when the shadow of a general purpose argument register
6809// in the parameter save area is aligned to at least 'RequiredAlign'.
6810static bool isGPRShadowAligned(MCPhysReg Reg, Align RequiredAlign) {
6811 assert(RequiredAlign.value() <= 16 &&
6812 "Required alignment greater than stack alignment.");
6813 switch (Reg) {
6814 default:
6815 report_fatal_error(reason: "called on invalid register.");
6816 case PPC::R5:
6817 case PPC::R9:
6818 case PPC::X3:
6819 case PPC::X5:
6820 case PPC::X7:
6821 case PPC::X9:
6822 // These registers are 16 byte aligned which is the most strict aligment
6823 // we can support.
6824 return true;
6825 case PPC::R3:
6826 case PPC::R7:
6827 case PPC::X4:
6828 case PPC::X6:
6829 case PPC::X8:
6830 case PPC::X10:
6831 // The shadow of these registers in the PSA is 8 byte aligned.
6832 return RequiredAlign <= 8;
6833 case PPC::R4:
6834 case PPC::R6:
6835 case PPC::R8:
6836 case PPC::R10:
6837 return RequiredAlign <= 4;
6838 }
6839}
6840
6841static bool CC_AIX(unsigned ValNo, MVT ValVT, MVT LocVT,
6842 CCValAssign::LocInfo LocInfo, ISD::ArgFlagsTy ArgFlags,
6843 Type *OrigTy, CCState &State) {
6844 const PPCSubtarget &Subtarget = static_cast<const PPCSubtarget &>(
6845 State.getMachineFunction().getSubtarget());
6846 const bool IsPPC64 = Subtarget.isPPC64();
6847 const unsigned PtrSize = IsPPC64 ? 8 : 4;
6848 const Align PtrAlign(PtrSize);
6849 const Align StackAlign(16);
6850 const MVT RegVT = Subtarget.getScalarIntVT();
6851
6852 if (ValVT == MVT::f128)
6853 report_fatal_error(reason: "f128 is unimplemented on AIX.");
6854
6855 static const MCPhysReg GPR_32[] = {// 32-bit registers.
6856 PPC::R3, PPC::R4, PPC::R5, PPC::R6,
6857 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
6858 static const MCPhysReg GPR_64[] = {// 64-bit registers.
6859 PPC::X3, PPC::X4, PPC::X5, PPC::X6,
6860 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
6861
6862 static const MCPhysReg VR[] = {// Vector registers.
6863 PPC::V2, PPC::V3, PPC::V4, PPC::V5,
6864 PPC::V6, PPC::V7, PPC::V8, PPC::V9,
6865 PPC::V10, PPC::V11, PPC::V12, PPC::V13};
6866
6867 const ArrayRef<MCPhysReg> GPRs = IsPPC64 ? GPR_64 : GPR_32;
6868
6869 if (ArgFlags.isNest()) {
6870 MCRegister EnvReg = State.AllocateReg(Reg: IsPPC64 ? PPC::X11 : PPC::R11);
6871 if (!EnvReg)
6872 report_fatal_error(reason: "More then one nest argument.");
6873 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: EnvReg, LocVT: RegVT, HTP: LocInfo));
6874 return false;
6875 }
6876
6877 if (ArgFlags.isByVal()) {
6878 const Align ByValAlign(ArgFlags.getNonZeroByValAlign());
6879 if (ByValAlign > StackAlign)
6880 report_fatal_error(reason: "Pass-by-value arguments with alignment greater than "
6881 "16 are not supported.");
6882
6883 const unsigned ByValSize = ArgFlags.getByValSize();
6884 const Align ObjAlign = ByValAlign > PtrAlign ? ByValAlign : PtrAlign;
6885
6886 // An empty aggregate parameter takes up no storage and no registers,
6887 // but needs a MemLoc for a stack slot for the formal arguments side.
6888 if (ByValSize == 0) {
6889 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6890 Offset: State.getStackSize(), LocVT: RegVT, HTP: LocInfo));
6891 return false;
6892 }
6893
6894 // Shadow allocate any registers that are not properly aligned.
6895 unsigned NextReg = State.getFirstUnallocated(Regs: GPRs);
6896 while (NextReg != GPRs.size() &&
6897 !isGPRShadowAligned(Reg: GPRs[NextReg], RequiredAlign: ObjAlign)) {
6898 // Shadow allocate next registers since its aligment is not strict enough.
6899 MCRegister Reg = State.AllocateReg(Regs: GPRs);
6900 // Allocate the stack space shadowed by said register.
6901 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6902 assert(Reg && "Alocating register unexpectedly failed.");
6903 (void)Reg;
6904 NextReg = State.getFirstUnallocated(Regs: GPRs);
6905 }
6906
6907 const unsigned StackSize = alignTo(Size: ByValSize, A: ObjAlign);
6908 unsigned Offset = State.AllocateStack(Size: StackSize, Alignment: ObjAlign);
6909 for (const unsigned E = Offset + StackSize; Offset < E; Offset += PtrSize) {
6910 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6911 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6912 else {
6913 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6914 Offset, LocVT: MVT::INVALID_SIMPLE_VALUE_TYPE,
6915 HTP: LocInfo));
6916 break;
6917 }
6918 }
6919 return false;
6920 }
6921
6922 // Arguments always reserve parameter save area.
6923 switch (ValVT.SimpleTy) {
6924 default:
6925 report_fatal_error(reason: "Unhandled value type for argument.");
6926 case MVT::i64:
6927 // i64 arguments should have been split to i32 for PPC32.
6928 assert(IsPPC64 && "PPC32 should have split i64 values.");
6929 [[fallthrough]];
6930 case MVT::i1:
6931 case MVT::i32: {
6932 const unsigned Offset = State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
6933 // AIX integer arguments are always passed in register width.
6934 if (ValVT.getFixedSizeInBits() < RegVT.getFixedSizeInBits())
6935 LocInfo = ArgFlags.isSExt() ? CCValAssign::LocInfo::SExt
6936 : CCValAssign::LocInfo::ZExt;
6937 if (MCRegister Reg = State.AllocateReg(Regs: GPRs))
6938 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6939 else
6940 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT: RegVT, HTP: LocInfo));
6941
6942 return false;
6943 }
6944 case MVT::f32:
6945 case MVT::f64: {
6946 // Parameter save area (PSA) is reserved even if the float passes in fpr.
6947 const unsigned StoreSize = LocVT.getStoreSize();
6948 // Floats are always 4-byte aligned in the PSA on AIX.
6949 // This includes f64 in 64-bit mode for ABI compatibility.
6950 const unsigned Offset =
6951 State.AllocateStack(Size: IsPPC64 ? 8 : StoreSize, Alignment: Align(4));
6952 MCRegister FReg = State.AllocateReg(Regs: FPR);
6953 if (FReg)
6954 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: FReg, LocVT, HTP: LocInfo));
6955
6956 // Reserve and initialize GPRs or initialize the PSA as required.
6957 for (unsigned I = 0; I < StoreSize; I += PtrSize) {
6958 if (MCRegister Reg = State.AllocateReg(Regs: GPRs)) {
6959 assert(FReg && "An FPR should be available when a GPR is reserved.");
6960 if (State.isVarArg()) {
6961 // Successfully reserved GPRs are only initialized for vararg calls.
6962 // Custom handling is required for:
6963 // f64 in PPC32 needs to be split into 2 GPRs.
6964 // f32 in PPC64 needs to occupy only lower 32 bits of 64-bit GPR.
6965 State.addLoc(
6966 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
6967 }
6968 } else {
6969 // If there are insufficient GPRs, the PSA needs to be initialized.
6970 // Initialization occurs even if an FPR was initialized for
6971 // compatibility with the AIX XL compiler. The full memory for the
6972 // argument will be initialized even if a prior word is saved in GPR.
6973 // A custom memLoc is used when the argument also passes in FPR so
6974 // that the callee handling can skip over it easily.
6975 State.addLoc(
6976 V: FReg ? CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT,
6977 HTP: LocInfo)
6978 : CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
6979 break;
6980 }
6981 }
6982
6983 return false;
6984 }
6985 case MVT::v4f32:
6986 case MVT::v4i32:
6987 case MVT::v8i16:
6988 case MVT::v16i8:
6989 case MVT::v2i64:
6990 case MVT::v2f64:
6991 case MVT::v1i128: {
6992 const unsigned VecSize = 16;
6993 const Align VecAlign(VecSize);
6994
6995 if (!State.isVarArg()) {
6996 // If there are vector registers remaining we don't consume any stack
6997 // space.
6998 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
6999 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
7000 return false;
7001 }
7002 // Vectors passed on the stack do not shadow GPRs or FPRs even though they
7003 // might be allocated in the portion of the PSA that is shadowed by the
7004 // GPRs.
7005 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7006 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7007 return false;
7008 }
7009
7010 unsigned NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7011 // Burn any underaligned registers and their shadowed stack space until
7012 // we reach the required alignment.
7013 while (NextRegIndex != GPRs.size() &&
7014 !isGPRShadowAligned(Reg: GPRs[NextRegIndex], RequiredAlign: VecAlign)) {
7015 // Shadow allocate register and its stack shadow.
7016 MCRegister Reg = State.AllocateReg(Regs: GPRs);
7017 State.AllocateStack(Size: PtrSize, Alignment: PtrAlign);
7018 assert(Reg && "Allocating register unexpectedly failed.");
7019 (void)Reg;
7020 NextRegIndex = State.getFirstUnallocated(Regs: GPRs);
7021 }
7022
7023 // Vectors that are passed as fixed arguments are handled differently.
7024 // They are passed in VRs if any are available (unlike arguments passed
7025 // through ellipses) and shadow GPRs (unlike arguments to non-vaarg
7026 // functions)
7027 if (!ArgFlags.isVarArg()) {
7028 if (MCRegister VReg = State.AllocateReg(Regs: VR)) {
7029 State.addLoc(V: CCValAssign::getReg(ValNo, ValVT, Reg: VReg, LocVT, HTP: LocInfo));
7030 // Shadow allocate GPRs and stack space even though we pass in a VR.
7031 for (unsigned I = 0; I != VecSize; I += PtrSize)
7032 State.AllocateReg(Regs: GPRs);
7033 State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7034 return false;
7035 }
7036 // No vector registers remain so pass on the stack.
7037 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7038 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7039 return false;
7040 }
7041
7042 // If all GPRS are consumed then we pass the argument fully on the stack.
7043 if (NextRegIndex == GPRs.size()) {
7044 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7045 State.addLoc(V: CCValAssign::getMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7046 return false;
7047 }
7048
7049 // Corner case for 32-bit codegen. We have 2 registers to pass the first
7050 // half of the argument, and then need to pass the remaining half on the
7051 // stack.
7052 if (GPRs[NextRegIndex] == PPC::R9) {
7053 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7054 State.addLoc(
7055 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7056
7057 const MCRegister FirstReg = State.AllocateReg(Reg: PPC::R9);
7058 const MCRegister SecondReg = State.AllocateReg(Reg: PPC::R10);
7059 assert(FirstReg && SecondReg &&
7060 "Allocating R9 or R10 unexpectedly failed.");
7061 State.addLoc(
7062 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: FirstReg, LocVT: RegVT, HTP: LocInfo));
7063 State.addLoc(
7064 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg: SecondReg, LocVT: RegVT, HTP: LocInfo));
7065 return false;
7066 }
7067
7068 // We have enough GPRs to fully pass the vector argument, and we have
7069 // already consumed any underaligned registers. Start with the custom
7070 // MemLoc and then the custom RegLocs.
7071 const unsigned Offset = State.AllocateStack(Size: VecSize, Alignment: VecAlign);
7072 State.addLoc(
7073 V: CCValAssign::getCustomMem(ValNo, ValVT, Offset, LocVT, HTP: LocInfo));
7074 for (unsigned I = 0; I != VecSize; I += PtrSize) {
7075 const MCRegister Reg = State.AllocateReg(Regs: GPRs);
7076 assert(Reg && "Failed to allocated register for vararg vector argument");
7077 State.addLoc(
7078 V: CCValAssign::getCustomReg(ValNo, ValVT, Reg, LocVT: RegVT, HTP: LocInfo));
7079 }
7080 return false;
7081 }
7082 }
7083 return true;
7084}
7085
7086// So far, this function is only used by LowerFormalArguments_AIX()
7087static const TargetRegisterClass *getRegClassForSVT(MVT::SimpleValueType SVT,
7088 bool IsPPC64,
7089 bool HasP8Vector,
7090 bool HasVSX) {
7091 assert((IsPPC64 || SVT != MVT::i64) &&
7092 "i64 should have been split for 32-bit codegen.");
7093
7094 switch (SVT) {
7095 default:
7096 report_fatal_error(reason: "Unexpected value type for formal argument");
7097 case MVT::i1:
7098 case MVT::i32:
7099 case MVT::i64:
7100 return IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7101 case MVT::f32:
7102 return HasP8Vector ? &PPC::VSSRCRegClass : &PPC::F4RCRegClass;
7103 case MVT::f64:
7104 return HasVSX ? &PPC::VSFRCRegClass : &PPC::F8RCRegClass;
7105 case MVT::v4f32:
7106 case MVT::v4i32:
7107 case MVT::v8i16:
7108 case MVT::v16i8:
7109 case MVT::v2i64:
7110 case MVT::v2f64:
7111 case MVT::v1i128:
7112 return &PPC::VRRCRegClass;
7113 }
7114}
7115
7116static SDValue truncateScalarIntegerArg(ISD::ArgFlagsTy Flags, EVT ValVT,
7117 SelectionDAG &DAG, SDValue ArgValue,
7118 MVT LocVT, const SDLoc &dl) {
7119 assert(ValVT.isScalarInteger() && LocVT.isScalarInteger());
7120 assert(ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits());
7121
7122 if (Flags.isSExt())
7123 ArgValue = DAG.getNode(Opcode: ISD::AssertSext, DL: dl, VT: LocVT, N1: ArgValue,
7124 N2: DAG.getValueType(ValVT));
7125 else if (Flags.isZExt())
7126 ArgValue = DAG.getNode(Opcode: ISD::AssertZext, DL: dl, VT: LocVT, N1: ArgValue,
7127 N2: DAG.getValueType(ValVT));
7128
7129 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ValVT, Operand: ArgValue);
7130}
7131
7132static unsigned mapArgRegToOffsetAIX(unsigned Reg, const PPCFrameLowering *FL) {
7133 const unsigned LASize = FL->getLinkageSize();
7134
7135 if (PPC::GPRCRegClass.contains(Reg)) {
7136 assert(Reg >= PPC::R3 && Reg <= PPC::R10 &&
7137 "Reg must be a valid argument register!");
7138 return LASize + 4 * (Reg - PPC::R3);
7139 }
7140
7141 if (PPC::G8RCRegClass.contains(Reg)) {
7142 assert(Reg >= PPC::X3 && Reg <= PPC::X10 &&
7143 "Reg must be a valid argument register!");
7144 return LASize + 8 * (Reg - PPC::X3);
7145 }
7146
7147 llvm_unreachable("Only general purpose registers expected.");
7148}
7149
7150// AIX ABI Stack Frame Layout:
7151//
7152// Low Memory +--------------------------------------------+
7153// SP +---> | Back chain | ---+
7154// | +--------------------------------------------+ |
7155// | | Saved Condition Register | |
7156// | +--------------------------------------------+ |
7157// | | Saved Linkage Register | |
7158// | +--------------------------------------------+ | Linkage Area
7159// | | Reserved for compilers | |
7160// | +--------------------------------------------+ |
7161// | | Reserved for binders | |
7162// | +--------------------------------------------+ |
7163// | | Saved TOC pointer | ---+
7164// | +--------------------------------------------+
7165// | | Parameter save area |
7166// | +--------------------------------------------+
7167// | | Alloca space |
7168// | +--------------------------------------------+
7169// | | Local variable space |
7170// | +--------------------------------------------+
7171// | | Float/int conversion temporary |
7172// | +--------------------------------------------+
7173// | | Save area for AltiVec registers |
7174// | +--------------------------------------------+
7175// | | AltiVec alignment padding |
7176// | +--------------------------------------------+
7177// | | Save area for VRSAVE register |
7178// | +--------------------------------------------+
7179// | | Save area for General Purpose registers |
7180// | +--------------------------------------------+
7181// | | Save area for Floating Point registers |
7182// | +--------------------------------------------+
7183// +---- | Back chain |
7184// High Memory +--------------------------------------------+
7185//
7186// Specifications:
7187// AIX 7.2 Assembler Language Reference
7188// Subroutine linkage convention
7189
7190SDValue PPCTargetLowering::LowerFormalArguments_AIX(
7191 SDValue Chain, CallingConv::ID CallConv, bool isVarArg,
7192 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7193 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
7194
7195 assert((CallConv == CallingConv::C || CallConv == CallingConv::Cold ||
7196 CallConv == CallingConv::Fast) &&
7197 "Unexpected calling convention!");
7198
7199 if (getTargetMachine().Options.GuaranteedTailCallOpt)
7200 report_fatal_error(reason: "Tail call support is unimplemented on AIX.");
7201
7202 if (useSoftFloat())
7203 report_fatal_error(reason: "Soft float support is unimplemented on AIX.");
7204
7205 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7206
7207 const bool IsPPC64 = Subtarget.isPPC64();
7208 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7209
7210 // Assign locations to all of the incoming arguments.
7211 SmallVector<CCValAssign, 16> ArgLocs;
7212 MachineFunction &MF = DAG.getMachineFunction();
7213 MachineFrameInfo &MFI = MF.getFrameInfo();
7214 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
7215 CCState CCInfo(CallConv, isVarArg, MF, ArgLocs, *DAG.getContext());
7216
7217 const EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
7218 // Reserve space for the linkage area on the stack.
7219 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7220 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7221 uint64_t SaveStackPos = CCInfo.getStackSize();
7222 bool SaveParams = MF.getFunction().hasFnAttribute(Kind: "save-reg-params");
7223 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_AIX);
7224
7225 SmallVector<SDValue, 8> MemOps;
7226
7227 for (size_t I = 0, End = ArgLocs.size(); I != End; /* No increment here */) {
7228 CCValAssign &VA = ArgLocs[I++];
7229 MVT LocVT = VA.getLocVT();
7230 MVT ValVT = VA.getValVT();
7231 ISD::ArgFlagsTy Flags = Ins[VA.getValNo()].Flags;
7232
7233 EVT ArgVT = Ins[VA.getValNo()].ArgVT;
7234 bool ArgSignExt = Ins[VA.getValNo()].Flags.isSExt();
7235 // For compatibility with the AIX XL compiler, the float args in the
7236 // parameter save area are initialized even if the argument is available
7237 // in register. The caller is required to initialize both the register
7238 // and memory, however, the callee can choose to expect it in either.
7239 // The memloc is dismissed here because the argument is retrieved from
7240 // the register.
7241 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isFloatingPoint())
7242 continue;
7243
7244 if (SaveParams && VA.isRegLoc() && !Flags.isByVal() && !VA.needsCustom()) {
7245 const TargetRegisterClass *RegClass = getRegClassForSVT(
7246 SVT: LocVT.SimpleTy, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(), HasVSX: Subtarget.hasVSX());
7247 // On PPC64, debugger assumes extended 8-byte values are stored from GPR.
7248 MVT SaveVT = RegClass == &PPC::G8RCRegClass ? MVT::i64 : LocVT;
7249 const Register VReg = MF.addLiveIn(PReg: VA.getLocReg(), RC: RegClass);
7250 SDValue Parm = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: SaveVT);
7251 int FI = MFI.CreateFixedObject(Size: SaveVT.getStoreSize(), SPOffset: SaveStackPos, IsImmutable: true);
7252 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7253 SDValue StoreReg = DAG.getStore(Chain, dl, Val: Parm, Ptr: FIN,
7254 PtrInfo: MachinePointerInfo(), Alignment: Align(PtrByteSize));
7255 SaveStackPos = alignTo(Value: SaveStackPos + SaveVT.getStoreSize(), Align: PtrByteSize);
7256 MemOps.push_back(Elt: StoreReg);
7257 }
7258
7259 if (SaveParams && (VA.isMemLoc() || Flags.isByVal()) && !VA.needsCustom()) {
7260 unsigned StoreSize =
7261 Flags.isByVal() ? Flags.getByValSize() : LocVT.getStoreSize();
7262 SaveStackPos = alignTo(Value: SaveStackPos + StoreSize, Align: PtrByteSize);
7263 }
7264
7265 auto HandleMemLoc = [&]() {
7266 const unsigned LocSize = LocVT.getStoreSize();
7267 const unsigned ValSize = ValVT.getStoreSize();
7268 assert((ValSize <= LocSize) &&
7269 "Object size is larger than size of MemLoc");
7270 int CurArgOffset = VA.getLocMemOffset();
7271 // Objects are right-justified because AIX is big-endian.
7272 if (LocSize > ValSize)
7273 CurArgOffset += LocSize - ValSize;
7274 // Potential tail calls could cause overwriting of argument stack slots.
7275 const bool IsImmutable =
7276 !(getTargetMachine().Options.GuaranteedTailCallOpt &&
7277 (CallConv == CallingConv::Fast));
7278 int FI = MFI.CreateFixedObject(Size: ValSize, SPOffset: CurArgOffset, IsImmutable);
7279 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7280 SDValue ArgValue =
7281 DAG.getLoad(VT: ValVT, dl, Chain, Ptr: FIN, PtrInfo: MachinePointerInfo());
7282
7283 // While the ABI specifies the argument type is (sign or zero) extended
7284 // out to register width, not all code is compliant. We truncate and
7285 // re-extend to be more forgiving of these callers when the argument type
7286 // is smaller than register width.
7287 if (!ArgVT.isVector() && !ValVT.isVector() && ArgVT.isInteger() &&
7288 ValVT.isInteger() &&
7289 ArgVT.getScalarSizeInBits() < ValVT.getScalarSizeInBits()) {
7290 // It is possible to have either real integer values
7291 // or integers that were not originally integers.
7292 // In the latter case, these could have came from structs,
7293 // and these integers would not have an extend on the parameter.
7294 // Since these types of integers do not have an extend specified
7295 // in the first place, the type of extend that we do should not matter.
7296 EVT TruncatedArgVT = ArgVT.isSimple() && ArgVT.getSimpleVT() == MVT::i1
7297 ? MVT::i8
7298 : ArgVT;
7299 SDValue ArgValueTrunc =
7300 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: TruncatedArgVT, Operand: ArgValue);
7301 SDValue ArgValueExt =
7302 ArgSignExt ? DAG.getSExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT)
7303 : DAG.getZExtOrTrunc(Op: ArgValueTrunc, DL: dl, VT: ValVT);
7304 InVals.push_back(Elt: ArgValueExt);
7305 } else {
7306 InVals.push_back(Elt: ArgValue);
7307 }
7308 };
7309
7310 // Vector arguments to VaArg functions are passed both on the stack, and
7311 // in any available GPRs. Load the value from the stack and add the GPRs
7312 // as live ins.
7313 if (VA.isMemLoc() && VA.needsCustom()) {
7314 assert(ValVT.isVector() && "Unexpected Custom MemLoc type.");
7315 assert(isVarArg && "Only use custom memloc for vararg.");
7316 // ValNo of the custom MemLoc, so we can compare it to the ValNo of the
7317 // matching custom RegLocs.
7318 const unsigned OriginalValNo = VA.getValNo();
7319 (void)OriginalValNo;
7320
7321 auto HandleCustomVecRegLoc = [&]() {
7322 assert(I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7323 "Missing custom RegLoc.");
7324 VA = ArgLocs[I++];
7325 assert(VA.getValVT().isVector() &&
7326 "Unexpected Val type for custom RegLoc.");
7327 assert(VA.getValNo() == OriginalValNo &&
7328 "ValNo mismatch between custom MemLoc and RegLoc.");
7329 MVT::SimpleValueType SVT = VA.getLocVT().SimpleTy;
7330 MF.addLiveIn(PReg: VA.getLocReg(),
7331 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7332 HasVSX: Subtarget.hasVSX()));
7333 };
7334
7335 HandleMemLoc();
7336 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7337 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7338 // R10.
7339 HandleCustomVecRegLoc();
7340 HandleCustomVecRegLoc();
7341
7342 // If we are targeting 32-bit, there might be 2 extra custom RegLocs if
7343 // we passed the vector in R5, R6, R7 and R8.
7344 if (I != End && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom()) {
7345 assert(!IsPPC64 &&
7346 "Only 2 custom RegLocs expected for 64-bit codegen.");
7347 HandleCustomVecRegLoc();
7348 HandleCustomVecRegLoc();
7349 }
7350
7351 continue;
7352 }
7353
7354 if (VA.isRegLoc()) {
7355 if (VA.getValVT().isScalarInteger())
7356 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7357 else if (VA.getValVT().isFloatingPoint() && !VA.getValVT().isVector()) {
7358 switch (VA.getValVT().SimpleTy) {
7359 default:
7360 report_fatal_error(reason: "Unhandled value type for argument.");
7361 case MVT::f32:
7362 FuncInfo->appendParameterType(Type: PPCFunctionInfo::ShortFloatingPoint);
7363 break;
7364 case MVT::f64:
7365 FuncInfo->appendParameterType(Type: PPCFunctionInfo::LongFloatingPoint);
7366 break;
7367 }
7368 } else if (VA.getValVT().isVector()) {
7369 switch (VA.getValVT().SimpleTy) {
7370 default:
7371 report_fatal_error(reason: "Unhandled value type for argument.");
7372 case MVT::v16i8:
7373 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorChar);
7374 break;
7375 case MVT::v8i16:
7376 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorShort);
7377 break;
7378 case MVT::v4i32:
7379 case MVT::v2i64:
7380 case MVT::v1i128:
7381 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorInt);
7382 break;
7383 case MVT::v4f32:
7384 case MVT::v2f64:
7385 FuncInfo->appendParameterType(Type: PPCFunctionInfo::VectorFloat);
7386 break;
7387 }
7388 }
7389 }
7390
7391 if (Flags.isByVal() && VA.isMemLoc()) {
7392 const unsigned Size =
7393 alignTo(Value: Flags.getByValSize() ? Flags.getByValSize() : PtrByteSize,
7394 Align: PtrByteSize);
7395 const int FI = MF.getFrameInfo().CreateFixedObject(
7396 Size, SPOffset: VA.getLocMemOffset(), /* IsImmutable */ false,
7397 /* IsAliased */ isAliased: true);
7398 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7399 InVals.push_back(Elt: FIN);
7400
7401 continue;
7402 }
7403
7404 if (Flags.isByVal()) {
7405 assert(VA.isRegLoc() && "MemLocs should already be handled.");
7406
7407 const MCPhysReg ArgReg = VA.getLocReg();
7408 const PPCFrameLowering *FL = Subtarget.getFrameLowering();
7409
7410 const unsigned StackSize = alignTo(Value: Flags.getByValSize(), Align: PtrByteSize);
7411 const int FI = MF.getFrameInfo().CreateFixedObject(
7412 Size: StackSize, SPOffset: mapArgRegToOffsetAIX(Reg: ArgReg, FL), /* IsImmutable */ false,
7413 /* IsAliased */ isAliased: true);
7414 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
7415 InVals.push_back(Elt: FIN);
7416
7417 // Add live ins for all the RegLocs for the same ByVal.
7418 const TargetRegisterClass *RegClass =
7419 IsPPC64 ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
7420
7421 auto HandleRegLoc = [&, RegClass, LocVT](const MCPhysReg PhysReg,
7422 unsigned Offset) {
7423 const Register VReg = MF.addLiveIn(PReg: PhysReg, RC: RegClass);
7424 // Since the callers side has left justified the aggregate in the
7425 // register, we can simply store the entire register into the stack
7426 // slot.
7427 SDValue CopyFrom = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7428 // The store to the fixedstack object is needed becuase accessing a
7429 // field of the ByVal will use a gep and load. Ideally we will optimize
7430 // to extracting the value from the register directly, and elide the
7431 // stores when the arguments address is not taken, but that will need to
7432 // be future work.
7433 SDValue Store = DAG.getStore(
7434 Chain: CopyFrom.getValue(R: 1), dl, Val: CopyFrom,
7435 Ptr: DAG.getObjectPtrOffset(SL: dl, Ptr: FIN, Offset: TypeSize::getFixed(ExactSize: Offset)),
7436 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset));
7437
7438 MemOps.push_back(Elt: Store);
7439 };
7440
7441 unsigned Offset = 0;
7442 HandleRegLoc(VA.getLocReg(), Offset);
7443 Offset += PtrByteSize;
7444 for (; Offset != StackSize && ArgLocs[I].isRegLoc();
7445 Offset += PtrByteSize) {
7446 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7447 "RegLocs should be for ByVal argument.");
7448
7449 const CCValAssign RL = ArgLocs[I++];
7450 HandleRegLoc(RL.getLocReg(), Offset);
7451 FuncInfo->appendParameterType(Type: PPCFunctionInfo::FixedType);
7452 }
7453
7454 if (Offset != StackSize) {
7455 assert(ArgLocs[I].getValNo() == VA.getValNo() &&
7456 "Expected MemLoc for remaining bytes.");
7457 assert(ArgLocs[I].isMemLoc() && "Expected MemLoc for remaining bytes.");
7458 // Consume the MemLoc.The InVal has already been emitted, so nothing
7459 // more needs to be done.
7460 ++I;
7461 }
7462
7463 continue;
7464 }
7465
7466 if (VA.isRegLoc() && !VA.needsCustom()) {
7467 MVT::SimpleValueType SVT = ValVT.SimpleTy;
7468 Register VReg =
7469 MF.addLiveIn(PReg: VA.getLocReg(),
7470 RC: getRegClassForSVT(SVT, IsPPC64, HasP8Vector: Subtarget.hasP8Vector(),
7471 HasVSX: Subtarget.hasVSX()));
7472 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: LocVT);
7473 if (ValVT.isScalarInteger() &&
7474 (ValVT.getFixedSizeInBits() < LocVT.getFixedSizeInBits())) {
7475 ArgValue =
7476 truncateScalarIntegerArg(Flags, ValVT, DAG, ArgValue, LocVT, dl);
7477 }
7478 InVals.push_back(Elt: ArgValue);
7479 continue;
7480 }
7481 if (VA.isMemLoc()) {
7482 HandleMemLoc();
7483 continue;
7484 }
7485 }
7486
7487 // On AIX a minimum of 8 words is saved to the parameter save area.
7488 const unsigned MinParameterSaveArea = 8 * PtrByteSize;
7489 // Area that is at least reserved in the caller of this function.
7490 unsigned CallerReservedArea = std::max<unsigned>(
7491 a: CCInfo.getStackSize(), b: LinkageSize + MinParameterSaveArea);
7492
7493 // Set the size that is at least reserved in caller of this function. Tail
7494 // call optimized function's reserved stack space needs to be aligned so
7495 // that taking the difference between two stack areas will result in an
7496 // aligned stack.
7497 CallerReservedArea =
7498 EnsureStackAlignment(Lowering: Subtarget.getFrameLowering(), NumBytes: CallerReservedArea);
7499 FuncInfo->setMinReservedArea(CallerReservedArea);
7500
7501 if (isVarArg) {
7502 int VAListIndex = 0;
7503 // If any of the optional arguments are passed in register then the fixed
7504 // stack object we spill into is not immutable. Create a fixed stack object
7505 // that overlaps the remainder of the parameter save area.
7506 if (CCInfo.getStackSize() < (LinkageSize + MinParameterSaveArea)) {
7507 unsigned FixedStackSize =
7508 LinkageSize + MinParameterSaveArea - CCInfo.getStackSize();
7509 VAListIndex =
7510 MFI.CreateFixedObject(Size: FixedStackSize, SPOffset: CCInfo.getStackSize(),
7511 /* IsImmutable */ false, /* IsAliased */ isAliased: true);
7512 } else {
7513 // All the arguments passed through ellipses are on the stack. Create a
7514 // dummy fixed stack object the same size as a pointer since we don't
7515 // know the actual size.
7516 VAListIndex =
7517 MFI.CreateFixedObject(Size: PtrByteSize, SPOffset: CCInfo.getStackSize(),
7518 /* IsImmutable */ true, /* IsAliased */ isAliased: true);
7519 }
7520
7521 FuncInfo->setVarArgsFrameIndex(VAListIndex);
7522 SDValue FIN = DAG.getFrameIndex(FI: VAListIndex, VT: PtrVT);
7523
7524 static const MCPhysReg GPR_32[] = {PPC::R3, PPC::R4, PPC::R5, PPC::R6,
7525 PPC::R7, PPC::R8, PPC::R9, PPC::R10};
7526
7527 static const MCPhysReg GPR_64[] = {PPC::X3, PPC::X4, PPC::X5, PPC::X6,
7528 PPC::X7, PPC::X8, PPC::X9, PPC::X10};
7529 const unsigned NumGPArgRegs = std::size(IsPPC64 ? GPR_64 : GPR_32);
7530
7531 // The fixed integer arguments of a variadic function are stored to the
7532 // VarArgsFrameIndex on the stack so that they may be loaded by
7533 // dereferencing the result of va_next.
7534 for (unsigned
7535 GPRIndex = (CCInfo.getStackSize() - LinkageSize) / PtrByteSize,
7536 Offset = 0;
7537 GPRIndex < NumGPArgRegs; ++GPRIndex, Offset += PtrByteSize) {
7538
7539 const Register VReg =
7540 IsPPC64 ? MF.addLiveIn(PReg: GPR_64[GPRIndex], RC: &PPC::G8RCRegClass)
7541 : MF.addLiveIn(PReg: GPR_32[GPRIndex], RC: &PPC::GPRCRegClass);
7542
7543 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: VReg, VT: PtrVT);
7544 MachinePointerInfo MPI =
7545 MachinePointerInfo::getFixedStack(MF, FI: VAListIndex, Offset);
7546 SDValue Store = DAG.getStore(Chain: Val.getValue(R: 1), dl, Val, Ptr: FIN, PtrInfo: MPI);
7547 MemOps.push_back(Elt: Store);
7548 // Increment the address for the next argument to store.
7549 SDValue PtrOff = DAG.getConstant(Val: PtrByteSize, DL: dl, VT: PtrVT);
7550 FIN = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrOff.getValueType(), N1: FIN, N2: PtrOff);
7551 }
7552 }
7553
7554 if (!MemOps.empty())
7555 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOps);
7556
7557 return Chain;
7558}
7559
7560SDValue PPCTargetLowering::LowerCall_AIX(
7561 SDValue Chain, SDValue Callee, CallFlags CFlags,
7562 const SmallVectorImpl<ISD::OutputArg> &Outs,
7563 const SmallVectorImpl<SDValue> &OutVals,
7564 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &dl,
7565 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals,
7566 const CallBase *CB) const {
7567 // See PPCTargetLowering::LowerFormalArguments_AIX() for a description of the
7568 // AIX ABI stack frame layout.
7569
7570 assert((CFlags.CallConv == CallingConv::C ||
7571 CFlags.CallConv == CallingConv::Cold ||
7572 CFlags.CallConv == CallingConv::Fast) &&
7573 "Unexpected calling convention!");
7574
7575 if (CFlags.IsPatchPoint)
7576 report_fatal_error(reason: "This call type is unimplemented on AIX.");
7577
7578 const PPCSubtarget &Subtarget = DAG.getSubtarget<PPCSubtarget>();
7579
7580 MachineFunction &MF = DAG.getMachineFunction();
7581 SmallVector<CCValAssign, 16> ArgLocs;
7582 CCState CCInfo(CFlags.CallConv, CFlags.IsVarArg, MF, ArgLocs,
7583 *DAG.getContext());
7584
7585 // Reserve space for the linkage save area (LSA) on the stack.
7586 // In both PPC32 and PPC64 there are 6 reserved slots in the LSA:
7587 // [SP][CR][LR][2 x reserved][TOC].
7588 // The LSA is 24 bytes (6x4) in PPC32 and 48 bytes (6x8) in PPC64.
7589 const unsigned LinkageSize = Subtarget.getFrameLowering()->getLinkageSize();
7590 const bool IsPPC64 = Subtarget.isPPC64();
7591 const EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7592 const unsigned PtrByteSize = IsPPC64 ? 8 : 4;
7593 CCInfo.AllocateStack(Size: LinkageSize, Alignment: Align(PtrByteSize));
7594 CCInfo.AnalyzeCallOperands(Outs, Fn: CC_AIX);
7595
7596 // The prolog code of the callee may store up to 8 GPR argument registers to
7597 // the stack, allowing va_start to index over them in memory if the callee
7598 // is variadic.
7599 // Because we cannot tell if this is needed on the caller side, we have to
7600 // conservatively assume that it is needed. As such, make sure we have at
7601 // least enough stack space for the caller to store the 8 GPRs.
7602 const unsigned MinParameterSaveAreaSize = 8 * PtrByteSize;
7603 const unsigned NumBytes = std::max<unsigned>(
7604 a: LinkageSize + MinParameterSaveAreaSize, b: CCInfo.getStackSize());
7605
7606 // Adjust the stack pointer for the new arguments...
7607 // These operations are automatically eliminated by the prolog/epilog pass.
7608 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: dl);
7609 SDValue CallSeqStart = Chain;
7610
7611 SmallVector<std::pair<unsigned, SDValue>, 8> RegsToPass;
7612 SmallVector<SDValue, 8> MemOpChains;
7613
7614 // Set up a copy of the stack pointer for loading and storing any
7615 // arguments that may not fit in the registers available for argument
7616 // passing.
7617 const SDValue StackPtr = IsPPC64 ? DAG.getRegister(Reg: PPC::X1, VT: MVT::i64)
7618 : DAG.getRegister(Reg: PPC::R1, VT: MVT::i32);
7619
7620 for (unsigned I = 0, E = ArgLocs.size(); I != E;) {
7621 const unsigned ValNo = ArgLocs[I].getValNo();
7622 SDValue Arg = OutVals[ValNo];
7623 ISD::ArgFlagsTy Flags = Outs[ValNo].Flags;
7624
7625 if (Flags.isByVal()) {
7626 const unsigned ByValSize = Flags.getByValSize();
7627
7628 // Nothing to do for zero-sized ByVals on the caller side.
7629 if (!ByValSize) {
7630 ++I;
7631 continue;
7632 }
7633
7634 auto GetLoad = [&](EVT VT, unsigned LoadOffset) {
7635 return DAG.getExtLoad(ExtType: ISD::ZEXTLOAD, dl, VT: PtrVT, Chain,
7636 Ptr: (LoadOffset != 0)
7637 ? DAG.getObjectPtrOffset(
7638 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7639 : Arg,
7640 PtrInfo: MachinePointerInfo(), MemVT: VT);
7641 };
7642
7643 unsigned LoadOffset = 0;
7644
7645 // Initialize registers, which are fully occupied by the by-val argument.
7646 while (LoadOffset + PtrByteSize <= ByValSize && ArgLocs[I].isRegLoc()) {
7647 SDValue Load = GetLoad(PtrVT, LoadOffset);
7648 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7649 LoadOffset += PtrByteSize;
7650 const CCValAssign &ByValVA = ArgLocs[I++];
7651 assert(ByValVA.getValNo() == ValNo &&
7652 "Unexpected location for pass-by-value argument.");
7653 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: Load));
7654 }
7655
7656 if (LoadOffset == ByValSize)
7657 continue;
7658
7659 // There must be one more loc to handle the remainder.
7660 assert(ArgLocs[I].getValNo() == ValNo &&
7661 "Expected additional location for by-value argument.");
7662
7663 if (ArgLocs[I].isMemLoc()) {
7664 assert(LoadOffset < ByValSize && "Unexpected memloc for by-val arg.");
7665 const CCValAssign &ByValVA = ArgLocs[I++];
7666 ISD::ArgFlagsTy MemcpyFlags = Flags;
7667 // Only memcpy the bytes that don't pass in register.
7668 MemcpyFlags.setByValSize(ByValSize - LoadOffset);
7669 Chain = CallSeqStart = createMemcpyOutsideCallSeq(
7670 Arg: (LoadOffset != 0) ? DAG.getObjectPtrOffset(
7671 SL: dl, Ptr: Arg, Offset: TypeSize::getFixed(ExactSize: LoadOffset))
7672 : Arg,
7673 PtrOff: DAG.getObjectPtrOffset(
7674 SL: dl, Ptr: StackPtr, Offset: TypeSize::getFixed(ExactSize: ByValVA.getLocMemOffset())),
7675 CallSeqStart, Flags: MemcpyFlags, DAG, dl);
7676 continue;
7677 }
7678
7679 // Initialize the final register residue.
7680 // Any residue that occupies the final by-val arg register must be
7681 // left-justified on AIX. Loads must be a power-of-2 size and cannot be
7682 // larger than the ByValSize. For example: a 7 byte by-val arg requires 4,
7683 // 2 and 1 byte loads.
7684 const unsigned ResidueBytes = ByValSize % PtrByteSize;
7685 assert(ResidueBytes != 0 && LoadOffset + PtrByteSize > ByValSize &&
7686 "Unexpected register residue for by-value argument.");
7687 SDValue ResidueVal;
7688 for (unsigned Bytes = 0; Bytes != ResidueBytes;) {
7689 const unsigned N = llvm::bit_floor(Value: ResidueBytes - Bytes);
7690 const MVT VT =
7691 N == 1 ? MVT::i8
7692 : ((N == 2) ? MVT::i16 : (N == 4 ? MVT::i32 : MVT::i64));
7693 SDValue Load = GetLoad(VT, LoadOffset);
7694 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7695 LoadOffset += N;
7696 Bytes += N;
7697
7698 // By-val arguments are passed left-justfied in register.
7699 // Every load here needs to be shifted, otherwise a full register load
7700 // should have been used.
7701 assert(PtrVT.getSimpleVT().getSizeInBits() > (Bytes * 8) &&
7702 "Unexpected load emitted during handling of pass-by-value "
7703 "argument.");
7704 unsigned NumSHLBits = PtrVT.getSimpleVT().getSizeInBits() - (Bytes * 8);
7705 EVT ShiftAmountTy =
7706 getShiftAmountTy(LHSTy: Load->getValueType(ResNo: 0), DL: DAG.getDataLayout());
7707 SDValue SHLAmt = DAG.getConstant(Val: NumSHLBits, DL: dl, VT: ShiftAmountTy);
7708 SDValue ShiftedLoad =
7709 DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: Load.getValueType(), N1: Load, N2: SHLAmt);
7710 ResidueVal = ResidueVal ? DAG.getNode(Opcode: ISD::OR, DL: dl, VT: PtrVT, N1: ResidueVal,
7711 N2: ShiftedLoad)
7712 : ShiftedLoad;
7713 }
7714
7715 const CCValAssign &ByValVA = ArgLocs[I++];
7716 RegsToPass.push_back(Elt: std::make_pair(x: ByValVA.getLocReg(), y&: ResidueVal));
7717 continue;
7718 }
7719
7720 CCValAssign &VA = ArgLocs[I++];
7721 const MVT LocVT = VA.getLocVT();
7722 const MVT ValVT = VA.getValVT();
7723
7724 switch (VA.getLocInfo()) {
7725 default:
7726 report_fatal_error(reason: "Unexpected argument extension type.");
7727 case CCValAssign::Full:
7728 break;
7729 case CCValAssign::ZExt:
7730 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7731 break;
7732 case CCValAssign::SExt:
7733 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7734 break;
7735 }
7736
7737 if (VA.isRegLoc() && !VA.needsCustom()) {
7738 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: Arg));
7739 continue;
7740 }
7741
7742 // Vector arguments passed to VarArg functions need custom handling when
7743 // they are passed (at least partially) in GPRs.
7744 if (VA.isMemLoc() && VA.needsCustom() && ValVT.isVector()) {
7745 assert(CFlags.IsVarArg && "Custom MemLocs only used for Vector args.");
7746 // Store value to its stack slot.
7747 SDValue PtrOff =
7748 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7749 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7750 SDValue Store =
7751 DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff, PtrInfo: MachinePointerInfo());
7752 MemOpChains.push_back(Elt: Store);
7753 const unsigned OriginalValNo = VA.getValNo();
7754 // Then load the GPRs from the stack
7755 unsigned LoadOffset = 0;
7756 auto HandleCustomVecRegLoc = [&]() {
7757 assert(I != E && "Unexpected end of CCvalAssigns.");
7758 assert(ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7759 "Expected custom RegLoc.");
7760 CCValAssign RegVA = ArgLocs[I++];
7761 assert(RegVA.getValNo() == OriginalValNo &&
7762 "Custom MemLoc ValNo and custom RegLoc ValNo must match.");
7763 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: PtrOff,
7764 N2: DAG.getConstant(Val: LoadOffset, DL: dl, VT: PtrVT));
7765 SDValue Load = DAG.getLoad(VT: PtrVT, dl, Chain: Store, Ptr: Add, PtrInfo: MachinePointerInfo());
7766 MemOpChains.push_back(Elt: Load.getValue(R: 1));
7767 RegsToPass.push_back(Elt: std::make_pair(x: RegVA.getLocReg(), y&: Load));
7768 LoadOffset += PtrByteSize;
7769 };
7770
7771 // In 64-bit there will be exactly 2 custom RegLocs that follow, and in
7772 // in 32-bit there will be 2 custom RegLocs if we are passing in R9 and
7773 // R10.
7774 HandleCustomVecRegLoc();
7775 HandleCustomVecRegLoc();
7776
7777 if (I != E && ArgLocs[I].isRegLoc() && ArgLocs[I].needsCustom() &&
7778 ArgLocs[I].getValNo() == OriginalValNo) {
7779 assert(!IsPPC64 &&
7780 "Only 2 custom RegLocs expected for 64-bit codegen.");
7781 HandleCustomVecRegLoc();
7782 HandleCustomVecRegLoc();
7783 }
7784
7785 continue;
7786 }
7787
7788 if (VA.isMemLoc()) {
7789 SDValue PtrOff =
7790 DAG.getConstant(Val: VA.getLocMemOffset(), DL: dl, VT: StackPtr.getValueType());
7791 PtrOff = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7792 MemOpChains.push_back(
7793 Elt: DAG.getStore(Chain, dl, Val: Arg, Ptr: PtrOff,
7794 PtrInfo: MachinePointerInfo::getStack(MF, Offset: VA.getLocMemOffset()),
7795 Alignment: Subtarget.getFrameLowering()->getStackAlign()));
7796
7797 continue;
7798 }
7799
7800 if (!ValVT.isFloatingPoint())
7801 report_fatal_error(
7802 reason: "Unexpected register handling for calling convention.");
7803
7804 // Custom handling is used for GPR initializations for vararg float
7805 // arguments.
7806 assert(VA.isRegLoc() && VA.needsCustom() && CFlags.IsVarArg &&
7807 LocVT.isInteger() &&
7808 "Custom register handling only expected for VarArg.");
7809
7810 SDValue ArgAsInt =
7811 DAG.getBitcast(VT: MVT::getIntegerVT(BitWidth: ValVT.getSizeInBits()), V: Arg);
7812
7813 if (Arg.getValueType().getStoreSize() == LocVT.getStoreSize())
7814 // f32 in 32-bit GPR
7815 // f64 in 64-bit GPR
7816 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgAsInt));
7817 else if (Arg.getValueType().getFixedSizeInBits() <
7818 LocVT.getFixedSizeInBits())
7819 // f32 in 64-bit GPR.
7820 RegsToPass.push_back(Elt: std::make_pair(
7821 x: VA.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: LocVT)));
7822 else {
7823 // f64 in two 32-bit GPRs
7824 // The 2 GPRs are marked custom and expected to be adjacent in ArgLocs.
7825 assert(Arg.getValueType() == MVT::f64 && CFlags.IsVarArg && !IsPPC64 &&
7826 "Unexpected custom register for argument!");
7827 CCValAssign &GPR1 = VA;
7828 SDValue MSWAsI64 = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: ArgAsInt,
7829 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i8));
7830 RegsToPass.push_back(Elt: std::make_pair(
7831 x: GPR1.getLocReg(), y: DAG.getZExtOrTrunc(Op: MSWAsI64, DL: dl, VT: MVT::i32)));
7832
7833 if (I != E) {
7834 // If only 1 GPR was available, there will only be one custom GPR and
7835 // the argument will also pass in memory.
7836 CCValAssign &PeekArg = ArgLocs[I];
7837 if (PeekArg.isRegLoc() && PeekArg.getValNo() == PeekArg.getValNo()) {
7838 assert(PeekArg.needsCustom() && "A second custom GPR is expected.");
7839 CCValAssign &GPR2 = ArgLocs[I++];
7840 RegsToPass.push_back(Elt: std::make_pair(
7841 x: GPR2.getLocReg(), y: DAG.getZExtOrTrunc(Op: ArgAsInt, DL: dl, VT: MVT::i32)));
7842 }
7843 }
7844 }
7845 }
7846
7847 if (!MemOpChains.empty())
7848 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: MemOpChains);
7849
7850 // For indirect calls, we need to save the TOC base to the stack for
7851 // restoration after the call.
7852 if (CFlags.IsIndirect && !Subtarget.usePointerGlueHelper()) {
7853 assert(!CFlags.IsTailCall && "Indirect tail-calls not supported.");
7854 const MCRegister TOCBaseReg = Subtarget.getTOCPointerRegister();
7855 const MCRegister StackPtrReg = Subtarget.getStackPointerRegister();
7856 const MVT PtrVT = Subtarget.getScalarIntVT();
7857 const unsigned TOCSaveOffset =
7858 Subtarget.getFrameLowering()->getTOCSaveOffset();
7859
7860 setUsesTOCBasePtr(DAG);
7861 SDValue Val = DAG.getCopyFromReg(Chain, dl, Reg: TOCBaseReg, VT: PtrVT);
7862 SDValue PtrOff = DAG.getIntPtrConstant(Val: TOCSaveOffset, DL: dl);
7863 SDValue StackPtr = DAG.getRegister(Reg: StackPtrReg, VT: PtrVT);
7864 SDValue AddPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackPtr, N2: PtrOff);
7865 Chain = DAG.getStore(
7866 Chain: Val.getValue(R: 1), dl, Val, Ptr: AddPtr,
7867 PtrInfo: MachinePointerInfo::getStack(MF&: DAG.getMachineFunction(), Offset: TOCSaveOffset));
7868 }
7869
7870 // Build a sequence of copy-to-reg nodes chained together with token chain
7871 // and flag operands which copy the outgoing args into the appropriate regs.
7872 SDValue InGlue;
7873 for (auto Reg : RegsToPass) {
7874 Chain = DAG.getCopyToReg(Chain, dl, Reg: Reg.first, N: Reg.second, Glue: InGlue);
7875 InGlue = Chain.getValue(R: 1);
7876 }
7877
7878 const int SPDiff = 0;
7879 return FinishCall(CFlags, dl, DAG, RegsToPass, Glue: InGlue, Chain, CallSeqStart,
7880 Callee, SPDiff, NumBytes, Ins, InVals, CB);
7881}
7882
7883bool
7884PPCTargetLowering::CanLowerReturn(CallingConv::ID CallConv,
7885 MachineFunction &MF, bool isVarArg,
7886 const SmallVectorImpl<ISD::OutputArg> &Outs,
7887 LLVMContext &Context,
7888 const Type *RetTy) const {
7889 SmallVector<CCValAssign, 16> RVLocs;
7890 CCState CCInfo(CallConv, isVarArg, MF, RVLocs, Context);
7891 return CCInfo.CheckReturn(
7892 Outs, Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7893 ? RetCC_PPC_Cold
7894 : RetCC_PPC);
7895}
7896
7897SDValue
7898PPCTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
7899 bool isVarArg,
7900 const SmallVectorImpl<ISD::OutputArg> &Outs,
7901 const SmallVectorImpl<SDValue> &OutVals,
7902 const SDLoc &dl, SelectionDAG &DAG) const {
7903 SmallVector<CCValAssign, 16> RVLocs;
7904 CCState CCInfo(CallConv, isVarArg, DAG.getMachineFunction(), RVLocs,
7905 *DAG.getContext());
7906 CCInfo.AnalyzeReturn(Outs,
7907 Fn: (Subtarget.isSVR4ABI() && CallConv == CallingConv::Cold)
7908 ? RetCC_PPC_Cold
7909 : RetCC_PPC);
7910
7911 SDValue Glue;
7912 SmallVector<SDValue, 4> RetOps(1, Chain);
7913
7914 // Copy the result values into the output registers.
7915 for (unsigned i = 0, RealResIdx = 0; i != RVLocs.size(); ++i, ++RealResIdx) {
7916 CCValAssign &VA = RVLocs[i];
7917 assert(VA.isRegLoc() && "Can only return in registers!");
7918
7919 SDValue Arg = OutVals[RealResIdx];
7920
7921 switch (VA.getLocInfo()) {
7922 default: llvm_unreachable("Unknown loc info!");
7923 case CCValAssign::Full: break;
7924 case CCValAssign::AExt:
7925 Arg = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7926 break;
7927 case CCValAssign::ZExt:
7928 Arg = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7929 break;
7930 case CCValAssign::SExt:
7931 Arg = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: VA.getLocVT(), Operand: Arg);
7932 break;
7933 }
7934 if (Subtarget.hasSPE() && VA.getLocVT() == MVT::f64) {
7935 bool isLittleEndian = Subtarget.isLittleEndian();
7936 // Legalize ret f64 -> ret 2 x i32.
7937 SDValue SVal =
7938 DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7939 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 0 : 1, DL: dl));
7940 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7941 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7942 SVal = DAG.getNode(Opcode: PPCISD::EXTRACT_SPE, DL: dl, VT: MVT::i32, N1: Arg,
7943 N2: DAG.getIntPtrConstant(Val: isLittleEndian ? 1 : 0, DL: dl));
7944 Glue = Chain.getValue(R: 1);
7945 VA = RVLocs[++i]; // skip ahead to next loc
7946 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: SVal, Glue);
7947 } else
7948 Chain = DAG.getCopyToReg(Chain, dl, Reg: VA.getLocReg(), N: Arg, Glue);
7949 Glue = Chain.getValue(R: 1);
7950 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
7951 }
7952
7953 RetOps[0] = Chain; // Update chain.
7954
7955 // Add the glue if we have it.
7956 if (Glue.getNode())
7957 RetOps.push_back(Elt: Glue);
7958
7959 return DAG.getNode(Opcode: PPCISD::RET_GLUE, DL: dl, VT: MVT::Other, Ops: RetOps);
7960}
7961
7962SDValue
7963PPCTargetLowering::LowerGET_DYNAMIC_AREA_OFFSET(SDValue Op,
7964 SelectionDAG &DAG) const {
7965 SDLoc dl(Op);
7966
7967 // Get the correct type for integers.
7968 EVT IntVT = Op.getValueType();
7969
7970 // Get the inputs.
7971 SDValue Chain = Op.getOperand(i: 0);
7972 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
7973 // Build a DYNAREAOFFSET node.
7974 SDValue Ops[2] = {Chain, FPSIdx};
7975 SDVTList VTs = DAG.getVTList(VT: IntVT);
7976 return DAG.getNode(Opcode: PPCISD::DYNAREAOFFSET, DL: dl, VTList: VTs, Ops);
7977}
7978
7979SDValue PPCTargetLowering::LowerSTACKRESTORE(SDValue Op,
7980 SelectionDAG &DAG) const {
7981 // When we pop the dynamic allocation we need to restore the SP link.
7982 SDLoc dl(Op);
7983
7984 // Get the correct type for pointers.
7985 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
7986
7987 // Construct the stack pointer operand.
7988 bool isPPC64 = Subtarget.isPPC64();
7989 unsigned SP = isPPC64 ? PPC::X1 : PPC::R1;
7990 SDValue StackPtr = DAG.getRegister(Reg: SP, VT: PtrVT);
7991
7992 // Get the operands for the STACKRESTORE.
7993 SDValue Chain = Op.getOperand(i: 0);
7994 SDValue SaveSP = Op.getOperand(i: 1);
7995
7996 // Load the old link SP.
7997 SDValue LoadLinkSP =
7998 DAG.getLoad(VT: PtrVT, dl, Chain, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
7999
8000 // Restore the stack pointer.
8001 Chain = DAG.getCopyToReg(Chain: LoadLinkSP.getValue(R: 1), dl, Reg: SP, N: SaveSP);
8002
8003 // Store the old link SP.
8004 return DAG.getStore(Chain, dl, Val: LoadLinkSP, Ptr: StackPtr, PtrInfo: MachinePointerInfo());
8005}
8006
8007SDValue PPCTargetLowering::getReturnAddrFrameIndex(SelectionDAG &DAG) const {
8008 MachineFunction &MF = DAG.getMachineFunction();
8009 bool isPPC64 = Subtarget.isPPC64();
8010 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8011
8012 // Get current frame pointer save index. The users of this index will be
8013 // primarily DYNALLOC instructions.
8014 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8015 int RASI = FI->getReturnAddrSaveIndex();
8016
8017 // If the frame pointer save index hasn't been defined yet.
8018 if (!RASI) {
8019 // Find out what the fix offset of the frame pointer save area.
8020 int LROffset = Subtarget.getFrameLowering()->getReturnSaveOffset();
8021 // Allocate the frame index for frame pointer save area.
8022 RASI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: LROffset, IsImmutable: false);
8023 // Save the result.
8024 FI->setReturnAddrSaveIndex(RASI);
8025 }
8026 return DAG.getFrameIndex(FI: RASI, VT: PtrVT);
8027}
8028
8029SDValue
8030PPCTargetLowering::getFramePointerFrameIndex(SelectionDAG & DAG) const {
8031 MachineFunction &MF = DAG.getMachineFunction();
8032 bool isPPC64 = Subtarget.isPPC64();
8033 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
8034
8035 // Get current frame pointer save index. The users of this index will be
8036 // primarily DYNALLOC instructions.
8037 PPCFunctionInfo *FI = MF.getInfo<PPCFunctionInfo>();
8038 int FPSI = FI->getFramePointerSaveIndex();
8039
8040 // If the frame pointer save index hasn't been defined yet.
8041 if (!FPSI) {
8042 // Find out what the fix offset of the frame pointer save area.
8043 int FPOffset = Subtarget.getFrameLowering()->getFramePointerSaveOffset();
8044 // Allocate the frame index for frame pointer save area.
8045 FPSI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64? 8 : 4, SPOffset: FPOffset, IsImmutable: true);
8046 // Save the result.
8047 FI->setFramePointerSaveIndex(FPSI);
8048 }
8049 return DAG.getFrameIndex(FI: FPSI, VT: PtrVT);
8050}
8051
8052SDValue PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDValue Op,
8053 SelectionDAG &DAG) const {
8054 MachineFunction &MF = DAG.getMachineFunction();
8055 // Get the inputs.
8056 SDValue Chain = Op.getOperand(i: 0);
8057 SDValue Size = Op.getOperand(i: 1);
8058 SDLoc dl(Op);
8059
8060 // Get the correct type for pointers.
8061 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8062 // Negate the size.
8063 SDValue NegSize = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: PtrVT,
8064 N1: DAG.getConstant(Val: 0, DL: dl, VT: PtrVT), N2: Size);
8065 // Construct a node for the frame pointer save index.
8066 SDValue FPSIdx = getFramePointerFrameIndex(DAG);
8067 SDValue Ops[3] = { Chain, NegSize, FPSIdx };
8068 SDVTList VTs = DAG.getVTList(VT1: PtrVT, VT2: MVT::Other);
8069 if (hasInlineStackProbe(MF))
8070 return DAG.getNode(Opcode: PPCISD::PROBED_ALLOCA, DL: dl, VTList: VTs, Ops);
8071 return DAG.getNode(Opcode: PPCISD::DYNALLOC, DL: dl, VTList: VTs, Ops);
8072}
8073
8074SDValue PPCTargetLowering::LowerEH_DWARF_CFA(SDValue Op,
8075 SelectionDAG &DAG) const {
8076 MachineFunction &MF = DAG.getMachineFunction();
8077
8078 bool isPPC64 = Subtarget.isPPC64();
8079 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8080
8081 int FI = MF.getFrameInfo().CreateFixedObject(Size: isPPC64 ? 8 : 4, SPOffset: 0, IsImmutable: false);
8082 return DAG.getFrameIndex(FI, VT: PtrVT);
8083}
8084
8085SDValue PPCTargetLowering::lowerEH_SJLJ_SETJMP(SDValue Op,
8086 SelectionDAG &DAG) const {
8087 SDLoc DL(Op);
8088 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_SETJMP, DL,
8089 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8090 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8091}
8092
8093SDValue PPCTargetLowering::lowerEH_SJLJ_LONGJMP(SDValue Op,
8094 SelectionDAG &DAG) const {
8095 SDLoc DL(Op);
8096 return DAG.getNode(Opcode: PPCISD::EH_SJLJ_LONGJMP, DL, VT: MVT::Other,
8097 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
8098}
8099
8100SDValue PPCTargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
8101 if (Op.getValueType().isVector())
8102 return LowerVectorLoad(Op, DAG);
8103
8104 assert(Op.getValueType() == MVT::i1 &&
8105 "Custom lowering only for i1 loads");
8106
8107 // First, load 8 bits into 32 bits, then truncate to 1 bit.
8108
8109 SDLoc dl(Op);
8110 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op);
8111
8112 SDValue Chain = LD->getChain();
8113 SDValue BasePtr = LD->getBasePtr();
8114 MachineMemOperand *MMO = LD->getMemOperand();
8115
8116 SDValue NewLD =
8117 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl, VT: getPointerTy(DL: DAG.getDataLayout()), Chain,
8118 Ptr: BasePtr, MemVT: MVT::i8, MMO);
8119 SDValue Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewLD);
8120
8121 SDValue Ops[] = { Result, SDValue(NewLD.getNode(), 1) };
8122 return DAG.getMergeValues(Ops, dl);
8123}
8124
8125SDValue PPCTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
8126 if (Op.getOperand(i: 1).getValueType().isVector())
8127 return LowerVectorStore(Op, DAG);
8128
8129 assert(Op.getOperand(1).getValueType() == MVT::i1 &&
8130 "Custom lowering only for i1 stores");
8131
8132 // First, zero extend to 32 bits, then use a truncating store to 8 bits.
8133
8134 SDLoc dl(Op);
8135 StoreSDNode *ST = cast<StoreSDNode>(Val&: Op);
8136
8137 SDValue Chain = ST->getChain();
8138 SDValue BasePtr = ST->getBasePtr();
8139 SDValue Value = ST->getValue();
8140 MachineMemOperand *MMO = ST->getMemOperand();
8141
8142 Value = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
8143 Operand: Value);
8144 return DAG.getTruncStore(Chain, dl, Val: Value, Ptr: BasePtr, SVT: MVT::i8, MMO);
8145}
8146
8147// FIXME: Remove this once the ANDI glue bug is fixed:
8148SDValue PPCTargetLowering::LowerTRUNCATE(SDValue Op, SelectionDAG &DAG) const {
8149 assert(Op.getValueType() == MVT::i1 &&
8150 "Custom lowering only for i1 results");
8151
8152 SDLoc DL(Op);
8153 return DAG.getNode(Opcode: PPCISD::ANDI_rec_1_GT_BIT, DL, VT: MVT::i1, Operand: Op.getOperand(i: 0));
8154}
8155
8156SDValue PPCTargetLowering::LowerTRUNCATEVector(SDValue Op,
8157 SelectionDAG &DAG) const {
8158
8159 // Implements a vector truncate that fits in a vector register as a shuffle.
8160 // We want to legalize vector truncates down to where the source fits in
8161 // a vector register (and target is therefore smaller than vector register
8162 // size). At that point legalization will try to custom lower the sub-legal
8163 // result and get here - where we can contain the truncate as a single target
8164 // operation.
8165
8166 // For example a trunc <2 x i16> to <2 x i8> could be visualized as follows:
8167 // <MSB1|LSB1, MSB2|LSB2> to <LSB1, LSB2>
8168 //
8169 // We will implement it for big-endian ordering as this (where x denotes
8170 // undefined):
8171 // < MSB1|LSB1, MSB2|LSB2, uu, uu, uu, uu, uu, uu> to
8172 // < LSB1, LSB2, u, u, u, u, u, u, u, u, u, u, u, u, u, u>
8173 //
8174 // The same operation in little-endian ordering will be:
8175 // <uu, uu, uu, uu, uu, uu, LSB2|MSB2, LSB1|MSB1> to
8176 // <u, u, u, u, u, u, u, u, u, u, u, u, u, u, LSB2, LSB1>
8177
8178 EVT TrgVT = Op.getValueType();
8179 assert(TrgVT.isVector() && "Vector type expected.");
8180 unsigned TrgNumElts = TrgVT.getVectorNumElements();
8181 EVT EltVT = TrgVT.getVectorElementType();
8182 if (!isOperationCustom(Op: Op.getOpcode(), VT: TrgVT) ||
8183 TrgVT.getSizeInBits() > 128 || !isPowerOf2_32(Value: TrgNumElts) ||
8184 !llvm::has_single_bit<uint32_t>(Value: EltVT.getSizeInBits()))
8185 return SDValue();
8186
8187 SDValue N1 = Op.getOperand(i: 0);
8188 EVT SrcVT = N1.getValueType();
8189 unsigned SrcSize = SrcVT.getSizeInBits();
8190 if (SrcSize > 256 || !isPowerOf2_32(Value: SrcVT.getVectorNumElements()) ||
8191 !llvm::has_single_bit<uint32_t>(
8192 Value: SrcVT.getVectorElementType().getSizeInBits()))
8193 return SDValue();
8194 if (SrcSize == 256 && SrcVT.getVectorNumElements() < 2)
8195 return SDValue();
8196
8197 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8198 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8199
8200 SDLoc DL(Op);
8201 SDValue Op1, Op2;
8202 if (SrcSize == 256) {
8203 EVT VecIdxTy = getVectorIdxTy(DL: DAG.getDataLayout());
8204 EVT SplitVT =
8205 N1.getValueType().getHalfNumVectorElementsVT(Context&: *DAG.getContext());
8206 unsigned SplitNumElts = SplitVT.getVectorNumElements();
8207 Op1 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8208 N2: DAG.getConstant(Val: 0, DL, VT: VecIdxTy));
8209 Op2 = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: SplitVT, N1,
8210 N2: DAG.getConstant(Val: SplitNumElts, DL, VT: VecIdxTy));
8211 }
8212 else {
8213 Op1 = SrcSize == 128 ? N1 : widenVec(DAG, Vec: N1, dl: DL);
8214 Op2 = DAG.getUNDEF(VT: WideVT);
8215 }
8216
8217 // First list the elements we want to keep.
8218 unsigned SizeMult = SrcSize / TrgVT.getSizeInBits();
8219 SmallVector<int, 16> ShuffV;
8220 if (Subtarget.isLittleEndian())
8221 for (unsigned i = 0; i < TrgNumElts; ++i)
8222 ShuffV.push_back(Elt: i * SizeMult);
8223 else
8224 for (unsigned i = 1; i <= TrgNumElts; ++i)
8225 ShuffV.push_back(Elt: i * SizeMult - 1);
8226
8227 // Populate the remaining elements with undefs.
8228 for (unsigned i = TrgNumElts; i < WideNumElts; ++i)
8229 // ShuffV.push_back(i + WideNumElts);
8230 ShuffV.push_back(Elt: WideNumElts + 1);
8231
8232 Op1 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op1);
8233 Op2 = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Op2);
8234 return DAG.getVectorShuffle(VT: WideVT, dl: DL, N1: Op1, N2: Op2, Mask: ShuffV);
8235}
8236
8237/// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
8238/// possible.
8239SDValue PPCTargetLowering::LowerSELECT_CC(SDValue Op, SelectionDAG &DAG) const {
8240 ISD::CondCode CC = Op.getOperandCondCode(i: 4);
8241 EVT ResVT = Op.getValueType();
8242 EVT CmpVT = Op.getOperand(i: 0).getValueType();
8243 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
8244 SDValue TV = Op.getOperand(i: 2), FV = Op.getOperand(i: 3);
8245 SDLoc dl(Op);
8246
8247 // Without power9-vector, we don't have native instruction for f128 comparison.
8248 // Following transformation to libcall is needed for setcc:
8249 // select_cc lhs, rhs, tv, fv, cc -> select_cc (setcc cc, x, y), 0, tv, fv, NE
8250 if (!Subtarget.hasP9Vector() && CmpVT == MVT::f128) {
8251 SDValue Z = DAG.getSetCC(
8252 DL: dl, VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: CmpVT),
8253 LHS, RHS, Cond: CC);
8254 SDValue Zero = DAG.getConstant(Val: 0, DL: dl, VT: Z.getValueType());
8255 return DAG.getSelectCC(DL: dl, LHS: Z, RHS: Zero, True: TV, False: FV, Cond: ISD::SETNE);
8256 }
8257
8258 // Not FP, or using SPE? Not a fsel.
8259 if (!CmpVT.isFloatingPoint() || !TV.getValueType().isFloatingPoint() ||
8260 Subtarget.hasSPE())
8261 return Op;
8262
8263 SDNodeFlags Flags = Op.getNode()->getFlags();
8264
8265 // We have xsmaxc[dq]p/xsminc[dq]p which are OK to emit even in the
8266 // presence of infinities.
8267 if (Subtarget.hasP9Vector() && LHS == TV && RHS == FV) {
8268 switch (CC) {
8269 default:
8270 break;
8271 case ISD::SETOGT:
8272 case ISD::SETGT:
8273 return DAG.getNode(Opcode: PPCISD::XSMAXC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8274 case ISD::SETOLT:
8275 case ISD::SETLT:
8276 return DAG.getNode(Opcode: PPCISD::XSMINC, DL: dl, VT: Op.getValueType(), N1: LHS, N2: RHS);
8277 }
8278 }
8279
8280 // We might be able to do better than this under some circumstances, but in
8281 // general, fsel-based lowering of select is a finite-math-only optimization.
8282 // For more information, see section F.3 of the 2.06 ISA specification.
8283 // With ISA 3.0
8284 if (!Flags.hasNoInfs() || !Flags.hasNoNaNs() || ResVT == MVT::f128)
8285 return Op;
8286
8287 // If the RHS of the comparison is a 0.0, we don't need to do the
8288 // subtraction at all.
8289 SDValue Sel1;
8290 if (isFloatingPointZero(Op: RHS))
8291 switch (CC) {
8292 default: break; // SETUO etc aren't handled by fsel.
8293 case ISD::SETNE:
8294 std::swap(a&: TV, b&: FV);
8295 [[fallthrough]];
8296 case ISD::SETEQ:
8297 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8298 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8299 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8300 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8301 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: Sel1, N3: FV);
8302 case ISD::SETULT:
8303 case ISD::SETLT:
8304 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8305 [[fallthrough]];
8306 case ISD::SETOGE:
8307 case ISD::SETGE:
8308 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8309 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8310 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: LHS, N2: TV, N3: FV);
8311 case ISD::SETUGT:
8312 case ISD::SETGT:
8313 std::swap(a&: TV, b&: FV); // fsel is natively setge, swap operands for setlt
8314 [[fallthrough]];
8315 case ISD::SETOLE:
8316 case ISD::SETLE:
8317 if (LHS.getValueType() == MVT::f32) // Comparison is always 64-bits
8318 LHS = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: LHS);
8319 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8320 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: LHS), N2: TV, N3: FV);
8321 }
8322
8323 SDValue Cmp;
8324 switch (CC) {
8325 default: break; // SETUO etc aren't handled by fsel.
8326 case ISD::SETNE:
8327 std::swap(a&: TV, b&: FV);
8328 [[fallthrough]];
8329 case ISD::SETEQ:
8330 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8331 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8332 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8333 Sel1 = DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8334 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT,
8335 N1: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT: MVT::f64, Operand: Cmp), N2: Sel1, N3: FV);
8336 case ISD::SETULT:
8337 case ISD::SETLT:
8338 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8339 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8340 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8341 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8342 case ISD::SETOGE:
8343 case ISD::SETGE:
8344 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: LHS, N2: RHS, Flags);
8345 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8346 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8347 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8348 case ISD::SETUGT:
8349 case ISD::SETGT:
8350 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8351 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8352 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8353 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: FV, N3: TV);
8354 case ISD::SETOLE:
8355 case ISD::SETLE:
8356 Cmp = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: CmpVT, N1: RHS, N2: LHS, Flags);
8357 if (Cmp.getValueType() == MVT::f32) // Comparison is always 64-bits
8358 Cmp = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Cmp);
8359 return DAG.getNode(Opcode: PPCISD::FSEL, DL: dl, VT: ResVT, N1: Cmp, N2: TV, N3: FV);
8360 }
8361 return Op;
8362}
8363
8364static unsigned getPPCStrictOpcode(unsigned Opc) {
8365 switch (Opc) {
8366 default:
8367 llvm_unreachable("No strict version of this opcode!");
8368 case PPCISD::FCTIDZ:
8369 return PPCISD::STRICT_FCTIDZ;
8370 case PPCISD::FCTIWZ:
8371 return PPCISD::STRICT_FCTIWZ;
8372 case PPCISD::FCTIDUZ:
8373 return PPCISD::STRICT_FCTIDUZ;
8374 case PPCISD::FCTIWUZ:
8375 return PPCISD::STRICT_FCTIWUZ;
8376 case PPCISD::FCFID:
8377 return PPCISD::STRICT_FCFID;
8378 case PPCISD::FCFIDU:
8379 return PPCISD::STRICT_FCFIDU;
8380 case PPCISD::FCFIDS:
8381 return PPCISD::STRICT_FCFIDS;
8382 case PPCISD::FCFIDUS:
8383 return PPCISD::STRICT_FCFIDUS;
8384 }
8385}
8386
8387static SDValue convertFPToInt(SDValue Op, SelectionDAG &DAG,
8388 const PPCSubtarget &Subtarget) {
8389 SDLoc dl(Op);
8390 bool IsStrict = Op->isStrictFPOpcode();
8391 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8392 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8393
8394 // TODO: Any other flags to propagate?
8395 SDNodeFlags Flags;
8396 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8397
8398 // For strict nodes, source is the second operand.
8399 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8400 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8401 MVT DestTy = Op.getSimpleValueType();
8402 assert(Src.getValueType().isFloatingPoint() &&
8403 (DestTy == MVT::i8 || DestTy == MVT::i16 || DestTy == MVT::i32 ||
8404 DestTy == MVT::i64) &&
8405 "Invalid FP_TO_INT types");
8406 if (Src.getValueType() == MVT::f32) {
8407 if (IsStrict) {
8408 Src =
8409 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL: dl,
8410 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8411 Chain = Src.getValue(R: 1);
8412 } else
8413 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
8414 }
8415 if ((DestTy == MVT::i8 || DestTy == MVT::i16) && Subtarget.hasP9Vector())
8416 DestTy = Subtarget.getScalarIntVT();
8417 unsigned Opc = ISD::DELETED_NODE;
8418 switch (DestTy.SimpleTy) {
8419 default: llvm_unreachable("Unhandled FP_TO_INT type in custom expander!");
8420 case MVT::i32:
8421 Opc = IsSigned ? PPCISD::FCTIWZ
8422 : (Subtarget.hasFPCVT() ? PPCISD::FCTIWUZ : PPCISD::FCTIDZ);
8423 break;
8424 case MVT::i64:
8425 assert((IsSigned || Subtarget.hasFPCVT()) &&
8426 "i64 FP_TO_UINT is supported only with FPCVT");
8427 Opc = IsSigned ? PPCISD::FCTIDZ : PPCISD::FCTIDUZ;
8428 }
8429 EVT ConvTy = Src.getValueType() == MVT::f128 ? MVT::f128 : MVT::f64;
8430 SDValue Conv;
8431 if (IsStrict) {
8432 Opc = getPPCStrictOpcode(Opc);
8433 Conv = DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src},
8434 Flags);
8435 } else {
8436 Conv = DAG.getNode(Opcode: Opc, DL: dl, VT: ConvTy, Operand: Src);
8437 }
8438 return Conv;
8439}
8440
8441void PPCTargetLowering::LowerFP_TO_INTForReuse(SDValue Op, ReuseLoadInfo &RLI,
8442 SelectionDAG &DAG,
8443 const SDLoc &dl) const {
8444 SDValue Tmp = convertFPToInt(Op, DAG, Subtarget);
8445 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8446 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8447 bool IsStrict = Op->isStrictFPOpcode();
8448
8449 // Convert the FP value to an int value through memory.
8450 bool i32Stack = Op.getValueType() == MVT::i32 && Subtarget.hasSTFIWX() &&
8451 (IsSigned || Subtarget.hasFPCVT());
8452 SDValue FIPtr = DAG.CreateStackTemporary(VT: i32Stack ? MVT::i32 : MVT::f64);
8453 int FI = cast<FrameIndexSDNode>(Val&: FIPtr)->getIndex();
8454 MachinePointerInfo MPI =
8455 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI);
8456
8457 // Emit a store to the stack slot.
8458 SDValue Chain = IsStrict ? Tmp.getValue(R: 1) : DAG.getEntryNode();
8459 Align Alignment(DAG.getEVTAlign(MemoryVT: Tmp.getValueType()));
8460 if (i32Stack) {
8461 MachineFunction &MF = DAG.getMachineFunction();
8462 Alignment = Align(4);
8463 MachineMemOperand *MMO =
8464 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Alignment);
8465 SDValue Ops[] = { Chain, Tmp, FIPtr };
8466 Chain = DAG.getMemIntrinsicNode(Opcode: PPCISD::STFIWX, dl,
8467 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT: MVT::i32, MMO);
8468 } else
8469 Chain = DAG.getStore(Chain, dl, Val: Tmp, Ptr: FIPtr, PtrInfo: MPI, Alignment);
8470
8471 // Result is a load from the stack slot. If loading 4 bytes, make sure to
8472 // add in a bias on big endian.
8473 if (Op.getValueType() == MVT::i32 && !i32Stack &&
8474 !Subtarget.isLittleEndian()) {
8475 FIPtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: FIPtr.getValueType(), N1: FIPtr,
8476 N2: DAG.getConstant(Val: 4, DL: dl, VT: FIPtr.getValueType()));
8477 MPI = MPI.getWithOffset(O: 4);
8478 }
8479
8480 RLI.Chain = Chain;
8481 RLI.Ptr = FIPtr;
8482 RLI.MPI = MPI;
8483 RLI.Alignment = Alignment;
8484}
8485
8486/// Custom lowers floating point to integer conversions to use
8487/// the direct move instructions available in ISA 2.07 to avoid the
8488/// need for load/store combinations.
8489SDValue PPCTargetLowering::LowerFP_TO_INTDirectMove(SDValue Op,
8490 SelectionDAG &DAG,
8491 const SDLoc &dl) const {
8492 SDValue Conv = convertFPToInt(Op, DAG, Subtarget);
8493 SDValue Mov = DAG.getNode(Opcode: PPCISD::MFVSR, DL: dl, VT: Op.getValueType(), Operand: Conv);
8494 if (Op->isStrictFPOpcode())
8495 return DAG.getMergeValues(Ops: {Mov, Conv.getValue(R: 1)}, dl);
8496 else
8497 return Mov;
8498}
8499
8500SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
8501 const SDLoc &dl) const {
8502 bool IsStrict = Op->isStrictFPOpcode();
8503 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT ||
8504 Op.getOpcode() == ISD::STRICT_FP_TO_SINT;
8505 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8506 EVT SrcVT = Src.getValueType();
8507 EVT DstVT = Op.getValueType();
8508
8509 // FP to INT conversions are legal for f128.
8510 if (SrcVT == MVT::f128)
8511 return Subtarget.hasP9Vector() ? Op : SDValue();
8512
8513 // Expand ppcf128 to i32 by hand for the benefit of llvm-gcc bootstrap on
8514 // PPC (the libcall is not available).
8515 if (SrcVT == MVT::ppcf128) {
8516 if (DstVT == MVT::i32) {
8517 // TODO: Conservatively pass only nofpexcept flag here. Need to check and
8518 // set other fast-math flags to FP operations in both strict and
8519 // non-strict cases. (FP_TO_SINT, FSUB)
8520 SDNodeFlags Flags;
8521 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8522
8523 if (IsSigned) {
8524 SDValue Lo, Hi;
8525 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Src, DL: dl, LoVT: MVT::f64, HiVT: MVT::f64);
8526
8527 // Add the two halves of the long double in round-to-zero mode, and use
8528 // a smaller FP_TO_SINT.
8529 if (IsStrict) {
8530 SDValue Res = DAG.getNode(Opcode: PPCISD::STRICT_FADDRTZ, DL: dl,
8531 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8532 Ops: {Op.getOperand(i: 0), Lo, Hi}, Flags);
8533 return DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8534 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other),
8535 Ops: {Res.getValue(R: 1), Res}, Flags);
8536 } else {
8537 SDValue Res = DAG.getNode(Opcode: PPCISD::FADDRTZ, DL: dl, VT: MVT::f64, N1: Lo, N2: Hi);
8538 return DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Res);
8539 }
8540 } else {
8541 const uint64_t TwoE31[] = {0x41e0000000000000LL, 0};
8542 APFloat APF = APFloat(APFloat::PPCDoubleDouble(), APInt(128, TwoE31));
8543 SDValue Cst = DAG.getConstantFP(Val: APF, DL: dl, VT: SrcVT);
8544 SDValue SignMask = DAG.getConstant(Val: 0x80000000, DL: dl, VT: DstVT);
8545 if (IsStrict) {
8546 // Sel = Src < 0x80000000
8547 // FltOfs = select Sel, 0.0, 0x80000000
8548 // IntOfs = select Sel, 0, 0x80000000
8549 // Result = fp_to_sint(Src - FltOfs) ^ IntOfs
8550 SDValue Chain = Op.getOperand(i: 0);
8551 EVT SetCCVT =
8552 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: SrcVT);
8553 EVT DstSetCCVT =
8554 getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: DstVT);
8555 SDValue Sel = DAG.getSetCC(DL: dl, VT: SetCCVT, LHS: Src, RHS: Cst, Cond: ISD::SETLT,
8556 Chain, IsSignaling: true);
8557 Chain = Sel.getValue(R: 1);
8558
8559 SDValue FltOfs = DAG.getSelect(
8560 DL: dl, VT: SrcVT, Cond: Sel, LHS: DAG.getConstantFP(Val: 0.0, DL: dl, VT: SrcVT), RHS: Cst);
8561 Sel = DAG.getBoolExtOrTrunc(Op: Sel, SL: dl, VT: DstSetCCVT, OpVT: DstVT);
8562
8563 SDValue Val = DAG.getNode(Opcode: ISD::STRICT_FSUB, DL: dl,
8564 VTList: DAG.getVTList(VT1: SrcVT, VT2: MVT::Other),
8565 Ops: {Chain, Src, FltOfs}, Flags);
8566 Chain = Val.getValue(R: 1);
8567 SDValue SInt = DAG.getNode(Opcode: ISD::STRICT_FP_TO_SINT, DL: dl,
8568 VTList: DAG.getVTList(VT1: DstVT, VT2: MVT::Other),
8569 Ops: {Chain, Val}, Flags);
8570 Chain = SInt.getValue(R: 1);
8571 SDValue IntOfs = DAG.getSelect(
8572 DL: dl, VT: DstVT, Cond: Sel, LHS: DAG.getConstant(Val: 0, DL: dl, VT: DstVT), RHS: SignMask);
8573 SDValue Result = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: DstVT, N1: SInt, N2: IntOfs);
8574 return DAG.getMergeValues(Ops: {Result, Chain}, dl);
8575 } else {
8576 // X>=2^31 ? (int)(X-2^31)+0x80000000 : (int)X
8577 // FIXME: generated code sucks.
8578 SDValue True = DAG.getNode(Opcode: ISD::FSUB, DL: dl, VT: MVT::ppcf128, N1: Src, N2: Cst);
8579 True = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: True);
8580 True = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i32, N1: True, N2: SignMask);
8581 SDValue False = DAG.getNode(Opcode: ISD::FP_TO_SINT, DL: dl, VT: MVT::i32, Operand: Src);
8582 return DAG.getSelectCC(DL: dl, LHS: Src, RHS: Cst, True, False, Cond: ISD::SETGE);
8583 }
8584 }
8585 }
8586
8587 return SDValue();
8588 }
8589
8590 if (Subtarget.hasDirectMove() && Subtarget.isPPC64())
8591 return LowerFP_TO_INTDirectMove(Op, DAG, dl);
8592
8593 ReuseLoadInfo RLI;
8594 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8595
8596 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI,
8597 Alignment: RLI.Alignment, MMOFlags: RLI.MMOFlags(),
8598 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8599}
8600
8601// We're trying to insert a regular store, S, and then a load, L. If the
8602// incoming value, O, is a load, we might just be able to have our load use the
8603// address used by O. However, we don't know if anything else will store to
8604// that address before we can load from it. To prevent this situation, we need
8605// to insert our load, L, into the chain as a peer of O. To do this, we give L
8606// the same chain operand as O, we create a token factor from the chain results
8607// of O and L, and we replace all uses of O's chain result with that token
8608// factor (this last part is handled by makeEquivalentMemoryOrdering).
8609bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT,
8610 ReuseLoadInfo &RLI,
8611 SelectionDAG &DAG,
8612 ISD::LoadExtType ET) const {
8613 // Conservatively skip reusing for constrained FP nodes.
8614 if (Op->isStrictFPOpcode())
8615 return false;
8616
8617 SDLoc dl(Op);
8618 bool ValidFPToUint = Op.getOpcode() == ISD::FP_TO_UINT &&
8619 (Subtarget.hasFPCVT() || Op.getValueType() == MVT::i32);
8620 if (ET == ISD::NON_EXTLOAD &&
8621 (ValidFPToUint || Op.getOpcode() == ISD::FP_TO_SINT) &&
8622 isOperationLegalOrCustom(Op: Op.getOpcode(),
8623 VT: Op.getOperand(i: 0).getValueType())) {
8624
8625 LowerFP_TO_INTForReuse(Op, RLI, DAG, dl);
8626 return true;
8627 }
8628
8629 LoadSDNode *LD = dyn_cast<LoadSDNode>(Val&: Op);
8630 if (!LD || LD->getExtensionType() != ET || LD->isVolatile() ||
8631 LD->isNonTemporal())
8632 return false;
8633 if (LD->getMemoryVT() != MemVT)
8634 return false;
8635
8636 // If the result of the load is an illegal type, then we can't build a
8637 // valid chain for reuse since the legalised loads and token factor node that
8638 // ties the legalised loads together uses a different output chain then the
8639 // illegal load.
8640 if (!isTypeLegal(VT: LD->getValueType(ResNo: 0)))
8641 return false;
8642
8643 RLI.Ptr = LD->getBasePtr();
8644 if (LD->isIndexed() && !LD->getOffset().isUndef()) {
8645 assert(LD->getAddressingMode() == ISD::PRE_INC &&
8646 "Non-pre-inc AM on PPC?");
8647 RLI.Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: RLI.Ptr.getValueType(), N1: RLI.Ptr,
8648 N2: LD->getOffset());
8649 }
8650
8651 RLI.Chain = LD->getChain();
8652 RLI.MPI = LD->getPointerInfo();
8653 RLI.IsDereferenceable = LD->isDereferenceable();
8654 RLI.IsInvariant = LD->isInvariant();
8655 RLI.Alignment = LD->getAlign();
8656 RLI.AAInfo = LD->getAAInfo();
8657 RLI.Ranges = LD->getRanges();
8658
8659 RLI.ResChain = SDValue(LD, LD->isIndexed() ? 2 : 1);
8660 return true;
8661}
8662
8663/// Analyze profitability of direct move
8664/// prefer float load to int load plus direct move
8665/// when there is no integer use of int load
8666bool PPCTargetLowering::directMoveIsProfitable(const SDValue &Op) const {
8667 SDNode *Origin = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0).getNode();
8668 if (Origin->getOpcode() != ISD::LOAD)
8669 return true;
8670
8671 // If there is no LXSIBZX/LXSIHZX, like Power8,
8672 // prefer direct move if the memory size is 1 or 2 bytes.
8673 MachineMemOperand *MMO = cast<LoadSDNode>(Val: Origin)->getMemOperand();
8674 if (!Subtarget.hasP9Vector() &&
8675 (!MMO->getSize().hasValue() || MMO->getSize().getValue() <= 2))
8676 return true;
8677
8678 for (SDUse &Use : Origin->uses()) {
8679
8680 // Only look at the users of the loaded value.
8681 if (Use.getResNo() != 0)
8682 continue;
8683
8684 SDNode *User = Use.getUser();
8685 if (User->getOpcode() != ISD::SINT_TO_FP &&
8686 User->getOpcode() != ISD::UINT_TO_FP &&
8687 User->getOpcode() != ISD::STRICT_SINT_TO_FP &&
8688 User->getOpcode() != ISD::STRICT_UINT_TO_FP)
8689 return true;
8690 }
8691
8692 return false;
8693}
8694
8695static SDValue convertIntToFP(SDValue Op, SDValue Src, SelectionDAG &DAG,
8696 const PPCSubtarget &Subtarget,
8697 SDValue Chain = SDValue()) {
8698 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8699 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8700 SDLoc dl(Op);
8701
8702 // TODO: Any other flags to propagate?
8703 SDNodeFlags Flags;
8704 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8705
8706 // If we have FCFIDS, then use it when converting to single-precision.
8707 // Otherwise, convert to double-precision and then round.
8708 bool IsSingle = Op.getValueType() == MVT::f32 && Subtarget.hasFPCVT();
8709 unsigned ConvOpc = IsSingle ? (IsSigned ? PPCISD::FCFIDS : PPCISD::FCFIDUS)
8710 : (IsSigned ? PPCISD::FCFID : PPCISD::FCFIDU);
8711 EVT ConvTy = IsSingle ? MVT::f32 : MVT::f64;
8712 if (Op->isStrictFPOpcode()) {
8713 if (!Chain)
8714 Chain = Op.getOperand(i: 0);
8715 return DAG.getNode(Opcode: getPPCStrictOpcode(Opc: ConvOpc), DL: dl,
8716 VTList: DAG.getVTList(VT1: ConvTy, VT2: MVT::Other), Ops: {Chain, Src}, Flags);
8717 } else
8718 return DAG.getNode(Opcode: ConvOpc, DL: dl, VT: ConvTy, Operand: Src);
8719}
8720
8721/// Custom lowers integer to floating point conversions to use
8722/// the direct move instructions available in ISA 2.07 to avoid the
8723/// need for load/store combinations.
8724SDValue PPCTargetLowering::LowerINT_TO_FPDirectMove(SDValue Op,
8725 SelectionDAG &DAG,
8726 const SDLoc &dl) const {
8727 assert((Op.getValueType() == MVT::f32 ||
8728 Op.getValueType() == MVT::f64) &&
8729 "Invalid floating point type as target of conversion");
8730 assert(Subtarget.hasFPCVT() &&
8731 "Int to FP conversions with direct moves require FPCVT");
8732 SDValue Src = Op.getOperand(i: Op->isStrictFPOpcode() ? 1 : 0);
8733 bool WordInt = Src.getSimpleValueType().SimpleTy == MVT::i32;
8734 bool Signed = Op.getOpcode() == ISD::SINT_TO_FP ||
8735 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8736 unsigned MovOpc = (WordInt && !Signed) ? PPCISD::MTVSRZ : PPCISD::MTVSRA;
8737 SDValue Mov = DAG.getNode(Opcode: MovOpc, DL: dl, VT: MVT::f64, Operand: Src);
8738 return convertIntToFP(Op, Src: Mov, DAG, Subtarget);
8739}
8740
8741static SDValue widenVec(SelectionDAG &DAG, SDValue Vec, const SDLoc &dl) {
8742
8743 EVT VecVT = Vec.getValueType();
8744 assert(VecVT.isVector() && "Expected a vector type.");
8745 assert(VecVT.getSizeInBits() < 128 && "Vector is already full width.");
8746
8747 EVT EltVT = VecVT.getVectorElementType();
8748 unsigned WideNumElts = 128 / EltVT.getSizeInBits();
8749 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: WideNumElts);
8750
8751 unsigned NumConcat = WideNumElts / VecVT.getVectorNumElements();
8752 SmallVector<SDValue, 16> Ops(NumConcat);
8753 Ops[0] = Vec;
8754 SDValue UndefVec = DAG.getUNDEF(VT: VecVT);
8755 for (unsigned i = 1; i < NumConcat; ++i)
8756 Ops[i] = UndefVec;
8757
8758 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL: dl, VT: WideVT, Ops);
8759}
8760
8761SDValue PPCTargetLowering::LowerINT_TO_FPVector(SDValue Op, SelectionDAG &DAG,
8762 const SDLoc &dl) const {
8763 bool IsStrict = Op->isStrictFPOpcode();
8764 unsigned Opc = Op.getOpcode();
8765 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8766 assert((Opc == ISD::UINT_TO_FP || Opc == ISD::SINT_TO_FP ||
8767 Opc == ISD::STRICT_UINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP) &&
8768 "Unexpected conversion type");
8769 assert((Op.getValueType() == MVT::v2f64 || Op.getValueType() == MVT::v4f32) &&
8770 "Supports conversions to v2f64/v4f32 only.");
8771
8772 // TODO: Any other flags to propagate?
8773 SDNodeFlags Flags;
8774 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8775
8776 bool SignedConv = Opc == ISD::SINT_TO_FP || Opc == ISD::STRICT_SINT_TO_FP;
8777 bool FourEltRes = Op.getValueType() == MVT::v4f32;
8778
8779 SDValue Wide = widenVec(DAG, Vec: Src, dl);
8780 EVT WideVT = Wide.getValueType();
8781 unsigned WideNumElts = WideVT.getVectorNumElements();
8782 MVT IntermediateVT = FourEltRes ? MVT::v4i32 : MVT::v2i64;
8783
8784 SmallVector<int, 16> ShuffV;
8785 for (unsigned i = 0; i < WideNumElts; ++i)
8786 ShuffV.push_back(Elt: i + WideNumElts);
8787
8788 int Stride = FourEltRes ? WideNumElts / 4 : WideNumElts / 2;
8789 int SaveElts = FourEltRes ? 4 : 2;
8790 if (Subtarget.isLittleEndian())
8791 for (int i = 0; i < SaveElts; i++)
8792 ShuffV[i * Stride] = i;
8793 else
8794 for (int i = 1; i <= SaveElts; i++)
8795 ShuffV[i * Stride - 1] = i - 1;
8796
8797 SDValue ShuffleSrc2 =
8798 SignedConv ? DAG.getUNDEF(VT: WideVT) : DAG.getConstant(Val: 0, DL: dl, VT: WideVT);
8799 SDValue Arrange = DAG.getVectorShuffle(VT: WideVT, dl, N1: Wide, N2: ShuffleSrc2, Mask: ShuffV);
8800
8801 SDValue Extend;
8802 if (SignedConv) {
8803 Arrange = DAG.getBitcast(VT: IntermediateVT, V: Arrange);
8804 EVT ExtVT = Src.getValueType();
8805 if (Subtarget.hasP9Altivec())
8806 ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideVT.getVectorElementType(),
8807 NumElements: IntermediateVT.getVectorNumElements());
8808
8809 Extend = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: IntermediateVT, N1: Arrange,
8810 N2: DAG.getValueType(ExtVT));
8811 } else
8812 Extend = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: IntermediateVT, Operand: Arrange);
8813
8814 if (IsStrict)
8815 return DAG.getNode(Opcode: Opc, DL: dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other),
8816 Ops: {Op.getOperand(i: 0), Extend}, Flags);
8817
8818 return DAG.getNode(Opcode: Opc, DL: dl, VT: Op.getValueType(), Operand: Extend);
8819}
8820
8821SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op,
8822 SelectionDAG &DAG) const {
8823 SDLoc dl(Op);
8824 bool IsSigned = Op.getOpcode() == ISD::SINT_TO_FP ||
8825 Op.getOpcode() == ISD::STRICT_SINT_TO_FP;
8826 bool IsStrict = Op->isStrictFPOpcode();
8827 SDValue Src = Op.getOperand(i: IsStrict ? 1 : 0);
8828 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : DAG.getEntryNode();
8829
8830 // TODO: Any other flags to propagate?
8831 SDNodeFlags Flags;
8832 Flags.setNoFPExcept(Op->getFlags().hasNoFPExcept());
8833
8834 EVT InVT = Src.getValueType();
8835 EVT OutVT = Op.getValueType();
8836 if (OutVT.isVector() && OutVT.isFloatingPoint() &&
8837 isOperationCustom(Op: Op.getOpcode(), VT: InVT))
8838 return LowerINT_TO_FPVector(Op, DAG, dl);
8839
8840 // Conversions to f128 are legal.
8841 if (Op.getValueType() == MVT::f128)
8842 return Subtarget.hasP9Vector() ? Op : SDValue();
8843
8844 // Don't handle ppc_fp128 here; let it be lowered to a libcall.
8845 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
8846 return SDValue();
8847
8848 if (Src.getValueType() == MVT::i1) {
8849 SDValue Sel = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: Op.getValueType(), N1: Src,
8850 N2: DAG.getConstantFP(Val: 1.0, DL: dl, VT: Op.getValueType()),
8851 N3: DAG.getConstantFP(Val: 0.0, DL: dl, VT: Op.getValueType()));
8852 if (IsStrict)
8853 return DAG.getMergeValues(Ops: {Sel, Chain}, dl);
8854 else
8855 return Sel;
8856 }
8857
8858 // If we have direct moves, we can do all the conversion, skip the store/load
8859 // however, without FPCVT we can't do most conversions.
8860 if (Subtarget.hasDirectMove() && directMoveIsProfitable(Op) &&
8861 Subtarget.isPPC64() && Subtarget.hasFPCVT())
8862 return LowerINT_TO_FPDirectMove(Op, DAG, dl);
8863
8864 assert((IsSigned || Subtarget.hasFPCVT()) &&
8865 "UINT_TO_FP is supported only with FPCVT");
8866
8867 if (Src.getValueType() == MVT::i64) {
8868 SDValue SINT = Src;
8869 // When converting to single-precision, we actually need to convert
8870 // to double-precision first and then round to single-precision.
8871 // To avoid double-rounding effects during that operation, we have
8872 // to prepare the input operand. Bits that might be truncated when
8873 // converting to double-precision are replaced by a bit that won't
8874 // be lost at this stage, but is below the single-precision rounding
8875 // position.
8876 //
8877 // However, if afn is in effect, accept double
8878 // rounding to avoid the extra overhead.
8879 // FIXME: Currently INT_TO_FP can't support fast math flags because
8880 // of nneg flag, thus Op->getFlags().hasApproximateFuncs() is always
8881 // false.
8882 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT() &&
8883 !Op->getFlags().hasApproximateFuncs()) {
8884
8885 // Twiddle input to make sure the low 11 bits are zero. (If this
8886 // is the case, we are guaranteed the value will fit into the 53 bit
8887 // mantissa of an IEEE double-precision value without rounding.)
8888 // If any of those low 11 bits were not zero originally, make sure
8889 // bit 12 (value 2048) is set instead, so that the final rounding
8890 // to single-precision gets the correct result.
8891 SDValue Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64,
8892 N1: SINT, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8893 Round = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8894 N1: Round, N2: DAG.getConstant(Val: 2047, DL: dl, VT: MVT::i64));
8895 Round = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i64, N1: Round, N2: SINT);
8896 Round = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i64, N1: Round,
8897 N2: DAG.getSignedConstant(Val: -2048, DL: dl, VT: MVT::i64));
8898
8899 // However, we cannot use that value unconditionally: if the magnitude
8900 // of the input value is small, the bit-twiddling we did above might
8901 // end up visibly changing the output. Fortunately, in that case, we
8902 // don't need to twiddle bits since the original input will convert
8903 // exactly to double-precision floating-point already. Therefore,
8904 // construct a conditional to use the original value if the top 11
8905 // bits are all sign-bit copies, and use the rounded value computed
8906 // above otherwise.
8907 SDValue Cond = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: MVT::i64,
8908 N1: SINT, N2: DAG.getConstant(Val: 53, DL: dl, VT: MVT::i32));
8909 Cond = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64,
8910 N1: Cond, N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64));
8911 Cond = DAG.getSetCC(
8912 DL: dl,
8913 VT: getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(), VT: MVT::i64),
8914 LHS: Cond, RHS: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i64), Cond: ISD::SETUGT);
8915
8916 SINT = DAG.getNode(Opcode: ISD::SELECT, DL: dl, VT: MVT::i64, N1: Cond, N2: Round, N3: SINT);
8917 }
8918
8919 ReuseLoadInfo RLI;
8920 SDValue Bits;
8921
8922 MachineFunction &MF = DAG.getMachineFunction();
8923 if (canReuseLoadAddress(Op: SINT, MemVT: MVT::i64, RLI, DAG)) {
8924 // Drop range metadata, as this metadata becomes invalid for f64 bit
8925 // reinterpretation of i64 values.
8926 Bits =
8927 DAG.getLoad(VT: MVT::f64, dl, Chain: RLI.Chain, Ptr: RLI.Ptr, PtrInfo: RLI.MPI, Alignment: RLI.Alignment,
8928 MMOFlags: RLI.MMOFlags(), Metadata: MMOMetadata(RLI.AAInfo));
8929 if (RLI.ResChain)
8930 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8931 } else if (Subtarget.hasLFIWAX() &&
8932 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::SEXTLOAD)) {
8933 MachineMemOperand *MMO = MF.getMachineMemOperand(
8934 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8935 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8936 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8937 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWAX, dl,
8938 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8939 Ops, MemVT: MVT::i32, MMO);
8940 if (RLI.ResChain)
8941 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8942 } else if (Subtarget.hasFPCVT() &&
8943 canReuseLoadAddress(Op: SINT, MemVT: MVT::i32, RLI, DAG, ET: ISD::ZEXTLOAD)) {
8944 MachineMemOperand *MMO = MF.getMachineMemOperand(
8945 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8946 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8947 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8948 Bits = DAG.getMemIntrinsicNode(Opcode: PPCISD::LFIWZX, dl,
8949 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8950 Ops, MemVT: MVT::i32, MMO);
8951 if (RLI.ResChain)
8952 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
8953 } else if (((Subtarget.hasLFIWAX() &&
8954 SINT.getOpcode() == ISD::SIGN_EXTEND) ||
8955 (Subtarget.hasFPCVT() &&
8956 SINT.getOpcode() == ISD::ZERO_EXTEND)) &&
8957 SINT.getOperand(i: 0).getValueType() == MVT::i32) {
8958 MachineFrameInfo &MFI = MF.getFrameInfo();
8959 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
8960
8961 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
8962 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
8963
8964 SDValue Store = DAG.getStore(Chain, dl, Val: SINT.getOperand(i: 0), Ptr: FIdx,
8965 PtrInfo: MachinePointerInfo::getFixedStack(
8966 MF&: DAG.getMachineFunction(), FI: FrameIdx));
8967 Chain = Store;
8968
8969 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
8970 "Expected an i32 store");
8971
8972 RLI.Ptr = FIdx;
8973 RLI.Chain = Chain;
8974 RLI.MPI =
8975 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
8976 RLI.Alignment = Align(4);
8977
8978 MachineMemOperand *MMO = MF.getMachineMemOperand(
8979 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
8980 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
8981 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
8982 Bits = DAG.getMemIntrinsicNode(Opcode: SINT.getOpcode() == ISD::ZERO_EXTEND ?
8983 PPCISD::LFIWZX : PPCISD::LFIWAX,
8984 dl, VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
8985 Ops, MemVT: MVT::i32, MMO);
8986 Chain = Bits.getValue(R: 1);
8987 } else
8988 Bits = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::f64, Operand: SINT);
8989
8990 SDValue FP = convertIntToFP(Op, Src: Bits, DAG, Subtarget, Chain);
8991 if (IsStrict)
8992 Chain = FP.getValue(R: 1);
8993
8994 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
8995 if (IsStrict)
8996 FP = DAG.getNode(
8997 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
8998 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)},
8999 Flags);
9000 else
9001 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9002 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9003 }
9004 return FP;
9005 }
9006
9007 assert(Src.getValueType() == MVT::i32 &&
9008 "Unhandled INT_TO_FP type in custom expander!");
9009 // Since we only generate this in 64-bit mode, we can take advantage of
9010 // 64-bit registers. In particular, sign extend the input value into the
9011 // 64-bit register with extsw, store the WHOLE 64-bit value into the stack
9012 // then lfd it and fcfid it.
9013 MachineFunction &MF = DAG.getMachineFunction();
9014 MachineFrameInfo &MFI = MF.getFrameInfo();
9015 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9016
9017 SDValue Ld;
9018 if (Subtarget.hasLFIWAX() || Subtarget.hasFPCVT()) {
9019 ReuseLoadInfo RLI;
9020 bool ReusingLoad;
9021 if (!(ReusingLoad = canReuseLoadAddress(Op: Src, MemVT: MVT::i32, RLI, DAG))) {
9022 int FrameIdx = MFI.CreateStackObject(Size: 4, Alignment: Align(4), isSpillSlot: false);
9023 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9024
9025 SDValue Store = DAG.getStore(Chain, dl, Val: Src, Ptr: FIdx,
9026 PtrInfo: MachinePointerInfo::getFixedStack(
9027 MF&: DAG.getMachineFunction(), FI: FrameIdx));
9028 Chain = Store;
9029
9030 assert(cast<StoreSDNode>(Store)->getMemoryVT() == MVT::i32 &&
9031 "Expected an i32 store");
9032
9033 RLI.Ptr = FIdx;
9034 RLI.Chain = Chain;
9035 RLI.MPI =
9036 MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx);
9037 RLI.Alignment = Align(4);
9038 }
9039
9040 MachineMemOperand *MMO = MF.getMachineMemOperand(
9041 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
9042 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
9043 SDValue Ops[] = { RLI.Chain, RLI.Ptr };
9044 Ld = DAG.getMemIntrinsicNode(Opcode: IsSigned ? PPCISD::LFIWAX : PPCISD::LFIWZX, dl,
9045 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other), Ops,
9046 MemVT: MVT::i32, MMO);
9047 Chain = Ld.getValue(R: 1);
9048 if (ReusingLoad && RLI.ResChain) {
9049 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Ld.getValue(R: 1));
9050 }
9051 } else {
9052 assert(Subtarget.isPPC64() &&
9053 "i32->FP without LFIWAX supported only on PPC64");
9054
9055 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9056 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
9057
9058 SDValue Ext64 = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL: dl, VT: MVT::i64, Operand: Src);
9059
9060 // STD the extended value into the stack slot.
9061 SDValue Store = DAG.getStore(
9062 Chain, dl, Val: Ext64, Ptr: FIdx,
9063 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9064 Chain = Store;
9065
9066 // Load the value as a double.
9067 Ld = DAG.getLoad(
9068 VT: MVT::f64, dl, Chain, Ptr: FIdx,
9069 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI: FrameIdx));
9070 Chain = Ld.getValue(R: 1);
9071 }
9072
9073 // FCFID it and return it.
9074 SDValue FP = convertIntToFP(Op, Src: Ld, DAG, Subtarget, Chain);
9075 if (IsStrict)
9076 Chain = FP.getValue(R: 1);
9077 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
9078 if (IsStrict)
9079 FP = DAG.getNode(
9080 Opcode: ISD::STRICT_FP_ROUND, DL: dl, VTList: DAG.getVTList(VT1: MVT::f32, VT2: MVT::Other),
9081 Ops: {Chain, FP, DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true)}, Flags);
9082 else
9083 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
9084 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
9085 }
9086 return FP;
9087}
9088
9089SDValue PPCTargetLowering::LowerSET_ROUNDING(SDValue Op,
9090 SelectionDAG &DAG) const {
9091 SDLoc Dl(Op);
9092 MachineFunction &MF = DAG.getMachineFunction();
9093 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9094 SDValue Chain = Op.getOperand(i: 0);
9095
9096 // If requested mode is constant, just use simpler mtfsb/mffscrni
9097 if (auto *CVal = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
9098 uint64_t Mode = CVal->getZExtValue();
9099 assert(Mode < 4 && "Unsupported rounding mode!");
9100 unsigned InternalRnd = Mode ^ (~(Mode >> 1) & 1);
9101 if (Subtarget.isISA3_0())
9102 return SDValue(
9103 DAG.getMachineNode(
9104 Opcode: PPC::MFFSCRNI, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9105 Ops: {DAG.getConstant(Val: InternalRnd, DL: Dl, VT: MVT::i32, isTarget: true), Chain}),
9106 1);
9107 SDNode *SetHi = DAG.getMachineNode(
9108 Opcode: (InternalRnd & 2) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9109 Ops: {DAG.getConstant(Val: 30, DL: Dl, VT: MVT::i32, isTarget: true), Chain});
9110 SDNode *SetLo = DAG.getMachineNode(
9111 Opcode: (InternalRnd & 1) ? PPC::MTFSB1 : PPC::MTFSB0, dl: Dl, VT: MVT::Other,
9112 Ops: {DAG.getConstant(Val: 31, DL: Dl, VT: MVT::i32, isTarget: true), SDValue(SetHi, 0)});
9113 return SDValue(SetLo, 0);
9114 }
9115
9116 // Use x ^ (~(x >> 1) & 1) to transform LLVM rounding mode to Power format.
9117 SDValue One = DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32);
9118 SDValue SrcFlag = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
9119 N2: DAG.getConstant(Val: 3, DL: Dl, VT: MVT::i32));
9120 SDValue DstFlag = DAG.getNode(
9121 Opcode: ISD::XOR, DL: Dl, VT: MVT::i32, N1: SrcFlag,
9122 N2: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32,
9123 N1: DAG.getNOT(DL: Dl,
9124 Val: DAG.getNode(Opcode: ISD::SRL, DL: Dl, VT: MVT::i32, N1: SrcFlag, N2: One),
9125 VT: MVT::i32),
9126 N2: One));
9127 // For Power9, there's faster mffscrn, and we don't need to read FPSCR
9128 SDValue MFFS;
9129 if (!Subtarget.isISA3_0()) {
9130 MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: Dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9131 Chain = MFFS.getValue(R: 1);
9132 }
9133 SDValue NewFPSCR;
9134 if (Subtarget.isPPC64()) {
9135 if (Subtarget.isISA3_0()) {
9136 NewFPSCR = DAG.getAnyExtOrTrunc(Op: DstFlag, DL: Dl, VT: MVT::i64);
9137 } else {
9138 // Set the last two bits (rounding mode) of bitcasted FPSCR.
9139 SDNode *InsertRN = DAG.getMachineNode(
9140 Opcode: PPC::RLDIMI, dl: Dl, VT: MVT::i64,
9141 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::i64, Operand: MFFS),
9142 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: Dl, VT: MVT::i64, Operand: DstFlag),
9143 DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9144 DAG.getTargetConstant(Val: 62, DL: Dl, VT: MVT::i32)});
9145 NewFPSCR = SDValue(InsertRN, 0);
9146 }
9147 NewFPSCR = DAG.getNode(Opcode: ISD::BITCAST, DL: Dl, VT: MVT::f64, Operand: NewFPSCR);
9148 } else {
9149 // In 32-bit mode, store f64, load and update the lower half.
9150 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9151 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9152 SDValue Addr = Subtarget.isLittleEndian()
9153 ? StackSlot
9154 : DAG.getNode(Opcode: ISD::ADD, DL: Dl, VT: PtrVT, N1: StackSlot,
9155 N2: DAG.getConstant(Val: 4, DL: Dl, VT: PtrVT));
9156 if (Subtarget.isISA3_0()) {
9157 Chain = DAG.getStore(Chain, dl: Dl, Val: DstFlag, Ptr: Addr, PtrInfo: MachinePointerInfo());
9158 } else {
9159 Chain = DAG.getStore(Chain, dl: Dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9160 SDValue Tmp =
9161 DAG.getLoad(VT: MVT::i32, dl: Dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9162 Chain = Tmp.getValue(R: 1);
9163 Tmp = SDValue(DAG.getMachineNode(
9164 Opcode: PPC::RLWIMI, dl: Dl, VT: MVT::i32,
9165 Ops: {Tmp, DstFlag, DAG.getTargetConstant(Val: 0, DL: Dl, VT: MVT::i32),
9166 DAG.getTargetConstant(Val: 30, DL: Dl, VT: MVT::i32),
9167 DAG.getTargetConstant(Val: 31, DL: Dl, VT: MVT::i32)}),
9168 0);
9169 Chain = DAG.getStore(Chain, dl: Dl, Val: Tmp, Ptr: Addr, PtrInfo: MachinePointerInfo());
9170 }
9171 NewFPSCR =
9172 DAG.getLoad(VT: MVT::f64, dl: Dl, Chain, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9173 Chain = NewFPSCR.getValue(R: 1);
9174 }
9175 if (Subtarget.isISA3_0())
9176 return SDValue(DAG.getMachineNode(Opcode: PPC::MFFSCRN, dl: Dl, ResultTys: {MVT::f64, MVT::Other},
9177 Ops: {NewFPSCR, Chain}),
9178 1);
9179 SDValue Zero = DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32, isTarget: true);
9180 SDNode *MTFSF = DAG.getMachineNode(
9181 Opcode: PPC::MTFSF, dl: Dl, VT: MVT::Other,
9182 Ops: {DAG.getConstant(Val: 255, DL: Dl, VT: MVT::i32, isTarget: true), NewFPSCR, Zero, Zero, Chain});
9183 return SDValue(MTFSF, 0);
9184}
9185
9186SDValue PPCTargetLowering::LowerGET_ROUNDING(SDValue Op,
9187 SelectionDAG &DAG) const {
9188 SDLoc dl(Op);
9189 /*
9190 The rounding mode is in bits 30:31 of FPSR, and has the following
9191 settings:
9192 00 Round to nearest
9193 01 Round to 0
9194 10 Round to +inf
9195 11 Round to -inf
9196
9197 GET_ROUNDING, on the other hand, expects the following:
9198 -1 Undefined
9199 0 Round to 0
9200 1 Round to nearest
9201 2 Round to +inf
9202 3 Round to -inf
9203
9204 To perform the conversion, we do:
9205 ((FPSCR & 0x3) ^ ((~FPSCR & 0x3) >> 1))
9206 */
9207
9208 MachineFunction &MF = DAG.getMachineFunction();
9209 EVT VT = Op.getValueType();
9210 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
9211
9212 // Save FP Control Word to register
9213 SDValue Chain = Op.getOperand(i: 0);
9214 SDValue MFFS = DAG.getNode(Opcode: PPCISD::MFFS, DL: dl, ResultTys: {MVT::f64, MVT::Other}, Ops: Chain);
9215 Chain = MFFS.getValue(R: 1);
9216
9217 SDValue CWD;
9218 if (isTypeLegal(VT: MVT::i64)) {
9219 CWD = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
9220 Operand: DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::i64, Operand: MFFS));
9221 } else {
9222 // Save FP register to stack slot
9223 int SSFI = MF.getFrameInfo().CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
9224 SDValue StackSlot = DAG.getFrameIndex(FI: SSFI, VT: PtrVT);
9225 Chain = DAG.getStore(Chain, dl, Val: MFFS, Ptr: StackSlot, PtrInfo: MachinePointerInfo());
9226
9227 // Load FP Control Word from low 32 bits of stack slot.
9228 assert(hasBigEndianPartOrdering(MVT::i64, MF.getDataLayout()) &&
9229 "Stack slot adjustment is valid only on big endian subtargets!");
9230 SDValue Four = DAG.getConstant(Val: 4, DL: dl, VT: PtrVT);
9231 SDValue Addr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: StackSlot, N2: Four);
9232 CWD = DAG.getLoad(VT: MVT::i32, dl, Chain, Ptr: Addr, PtrInfo: MachinePointerInfo());
9233 Chain = CWD.getValue(R: 1);
9234 }
9235
9236 // Transform as necessary
9237 SDValue CWD1 =
9238 DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9239 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32));
9240 SDValue CWD2 =
9241 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32,
9242 N1: DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32,
9243 N1: DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32,
9244 N1: CWD, N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9245 N2: DAG.getConstant(Val: 3, DL: dl, VT: MVT::i32)),
9246 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
9247
9248 SDValue RetVal =
9249 DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: CWD1, N2: CWD2);
9250
9251 RetVal =
9252 DAG.getNode(Opcode: (VT.getSizeInBits() < 16 ? ISD::TRUNCATE : ISD::ZERO_EXTEND),
9253 DL: dl, VT, Operand: RetVal);
9254
9255 return DAG.getMergeValues(Ops: {RetVal, Chain}, dl);
9256}
9257
9258SDValue PPCTargetLowering::LowerSHL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9259 EVT VT = Op.getValueType();
9260 uint64_t BitWidth = VT.getSizeInBits();
9261 SDLoc dl(Op);
9262 assert(Op.getNumOperands() == 3 &&
9263 VT == Op.getOperand(1).getValueType() &&
9264 "Unexpected SHL!");
9265
9266 // Expand into a bunch of logical ops. Note that these ops
9267 // depend on the PPC behavior for oversized shift amounts.
9268 SDValue Lo = Op.getOperand(i: 0);
9269 SDValue Hi = Op.getOperand(i: 1);
9270 SDValue Amt = Op.getOperand(i: 2);
9271 EVT AmtVT = Amt.getValueType();
9272
9273 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9274 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9275 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Amt);
9276 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Tmp1);
9277 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR , DL: dl, VT, N1: Tmp2, N2: Tmp3);
9278 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9279 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9280 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Tmp5);
9281 SDValue OutHi = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9282 SDValue OutLo = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Lo, N2: Amt);
9283 SDValue OutOps[] = { OutLo, OutHi };
9284 return DAG.getMergeValues(Ops: OutOps, dl);
9285}
9286
9287SDValue PPCTargetLowering::LowerSRL_PARTS(SDValue Op, SelectionDAG &DAG) const {
9288 EVT VT = Op.getValueType();
9289 SDLoc dl(Op);
9290 uint64_t BitWidth = VT.getSizeInBits();
9291 assert(Op.getNumOperands() == 3 &&
9292 VT == Op.getOperand(1).getValueType() &&
9293 "Unexpected SRL!");
9294
9295 // Expand into a bunch of logical ops. Note that these ops
9296 // depend on the PPC behavior for oversized shift amounts.
9297 SDValue Lo = Op.getOperand(i: 0);
9298 SDValue Hi = Op.getOperand(i: 1);
9299 SDValue Amt = Op.getOperand(i: 2);
9300 EVT AmtVT = Amt.getValueType();
9301
9302 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9303 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9304 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9305 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9306 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9307 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9308 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9309 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Tmp5);
9310 SDValue OutLo = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp4, N2: Tmp6);
9311 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Hi, N2: Amt);
9312 SDValue OutOps[] = { OutLo, OutHi };
9313 return DAG.getMergeValues(Ops: OutOps, dl);
9314}
9315
9316SDValue PPCTargetLowering::LowerSRA_PARTS(SDValue Op, SelectionDAG &DAG) const {
9317 SDLoc dl(Op);
9318 EVT VT = Op.getValueType();
9319 uint64_t BitWidth = VT.getSizeInBits();
9320 assert(Op.getNumOperands() == 3 &&
9321 VT == Op.getOperand(1).getValueType() &&
9322 "Unexpected SRA!");
9323
9324 // Expand into a bunch of logical ops, followed by a select_cc.
9325 SDValue Lo = Op.getOperand(i: 0);
9326 SDValue Hi = Op.getOperand(i: 1);
9327 SDValue Amt = Op.getOperand(i: 2);
9328 EVT AmtVT = Amt.getValueType();
9329
9330 SDValue Tmp1 = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT,
9331 N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Amt);
9332 SDValue Tmp2 = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Lo, N2: Amt);
9333 SDValue Tmp3 = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: Hi, N2: Tmp1);
9334 SDValue Tmp4 = DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: Tmp2, N2: Tmp3);
9335 SDValue Tmp5 = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: AmtVT, N1: Amt,
9336 N2: DAG.getSignedConstant(Val: -BitWidth, DL: dl, VT: AmtVT));
9337 SDValue Tmp6 = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Tmp5);
9338 SDValue OutHi = DAG.getNode(Opcode: PPCISD::SRA, DL: dl, VT, N1: Hi, N2: Amt);
9339 SDValue OutLo = DAG.getSelectCC(DL: dl, LHS: Tmp5, RHS: DAG.getConstant(Val: 0, DL: dl, VT: AmtVT),
9340 True: Tmp4, False: Tmp6, Cond: ISD::SETLE);
9341 SDValue OutOps[] = { OutLo, OutHi };
9342 return DAG.getMergeValues(Ops: OutOps, dl);
9343}
9344
9345SDValue PPCTargetLowering::LowerFunnelShift(SDValue Op,
9346 SelectionDAG &DAG) const {
9347 SDLoc dl(Op);
9348 EVT VT = Op.getValueType();
9349 unsigned BitWidth = VT.getSizeInBits();
9350
9351 bool IsFSHL = Op.getOpcode() == ISD::FSHL;
9352 SDValue X = Op.getOperand(i: 0);
9353 SDValue Y = Op.getOperand(i: 1);
9354 SDValue Z = Op.getOperand(i: 2);
9355 EVT AmtVT = Z.getValueType();
9356
9357 // fshl: (X << (Z % BW)) | (Y >> (BW - (Z % BW)))
9358 // fshr: (X << (BW - (Z % BW))) | (Y >> (Z % BW))
9359 // This is simpler than TargetLowering::expandFunnelShift because we can rely
9360 // on PowerPC shift by BW being well defined.
9361 Z = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: AmtVT, N1: Z,
9362 N2: DAG.getConstant(Val: BitWidth - 1, DL: dl, VT: AmtVT));
9363 SDValue SubZ =
9364 DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: AmtVT, N1: DAG.getConstant(Val: BitWidth, DL: dl, VT: AmtVT), N2: Z);
9365 X = DAG.getNode(Opcode: PPCISD::SHL, DL: dl, VT, N1: X, N2: IsFSHL ? Z : SubZ);
9366 Y = DAG.getNode(Opcode: PPCISD::SRL, DL: dl, VT, N1: Y, N2: IsFSHL ? SubZ : Z);
9367 return DAG.getNode(Opcode: ISD::OR, DL: dl, VT, N1: X, N2: Y);
9368}
9369
9370//===----------------------------------------------------------------------===//
9371// Vector related lowering.
9372//
9373
9374/// getCanonicalConstSplat - Build a canonical splat immediate of Val with an
9375/// element size of SplatSize. Cast the result to VT.
9376static SDValue getCanonicalConstSplat(uint64_t Val, unsigned SplatSize, EVT VT,
9377 SelectionDAG &DAG, const SDLoc &dl) {
9378 static const MVT VTys[] = { // canonical VT to use for each size.
9379 MVT::v16i8, MVT::v8i16, MVT::Other, MVT::v4i32
9380 };
9381
9382 EVT ReqVT = VT != MVT::Other ? VT : VTys[SplatSize-1];
9383
9384 // For a splat with all ones, turn it to vspltisb 0xFF to canonicalize.
9385 if (Val == ((1LLU << (SplatSize * 8)) - 1)) {
9386 SplatSize = 1;
9387 Val = 0xFF;
9388 }
9389
9390 EVT CanonicalVT = VTys[SplatSize-1];
9391
9392 // Build a canonical splat for this value.
9393 // Explicitly truncate APInt here, as this API is used with a mix of
9394 // signed and unsigned values.
9395 return DAG.getBitcast(
9396 VT: ReqVT,
9397 V: DAG.getConstant(Val: APInt(64, Val).trunc(width: SplatSize * 8), DL: dl, VT: CanonicalVT));
9398}
9399
9400/// BuildIntrinsicOp - Return a unary operator intrinsic node with the
9401/// specified intrinsic ID.
9402static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op, SelectionDAG &DAG,
9403 const SDLoc &dl, EVT DestVT = MVT::Other) {
9404 if (DestVT == MVT::Other) DestVT = Op.getValueType();
9405 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9406 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op);
9407}
9408
9409/// BuildIntrinsicOp - Return a binary operator intrinsic node with the
9410/// specified intrinsic ID.
9411static SDValue BuildIntrinsicOp(unsigned IID, SDValue LHS, SDValue RHS,
9412 SelectionDAG &DAG, const SDLoc &dl,
9413 EVT DestVT = MVT::Other) {
9414 if (DestVT == MVT::Other) DestVT = LHS.getValueType();
9415 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9416 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: LHS, N3: RHS);
9417}
9418
9419/// BuildIntrinsicOp - Return a ternary operator intrinsic node with the
9420/// specified intrinsic ID.
9421static SDValue BuildIntrinsicOp(unsigned IID, SDValue Op0, SDValue Op1,
9422 SDValue Op2, SelectionDAG &DAG, const SDLoc &dl,
9423 EVT DestVT = MVT::Other) {
9424 if (DestVT == MVT::Other) DestVT = Op0.getValueType();
9425 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: DestVT,
9426 N1: DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32), N2: Op0, N3: Op1, N4: Op2);
9427}
9428
9429/// BuildVSLDOI - Return a VECTOR_SHUFFLE that is a vsldoi of the specified
9430/// amount. The result has the specified value type.
9431static SDValue BuildVSLDOI(SDValue LHS, SDValue RHS, unsigned Amt, EVT VT,
9432 SelectionDAG &DAG, const SDLoc &dl) {
9433 // Force LHS/RHS to be the right type.
9434 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: LHS);
9435 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: RHS);
9436
9437 int Ops[16];
9438 for (unsigned i = 0; i != 16; ++i)
9439 Ops[i] = i + Amt;
9440 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: LHS, N2: RHS, Mask: Ops);
9441 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
9442}
9443
9444/// Do we have an efficient pattern in a .td file for this node?
9445///
9446/// \param V - pointer to the BuildVectorSDNode being matched
9447/// \param HasDirectMove - does this subtarget have VSR <-> GPR direct moves?
9448///
9449/// There are some patterns where it is beneficial to keep a BUILD_VECTOR
9450/// node as a BUILD_VECTOR node rather than expanding it. The patterns where
9451/// the opposite is true (expansion is beneficial) are:
9452/// - The node builds a vector out of integers that are not 32 or 64-bits
9453/// - The node builds a vector out of constants
9454/// - The node is a "load-and-splat"
9455/// In all other cases, we will choose to keep the BUILD_VECTOR.
9456static bool haveEfficientBuildVectorPattern(BuildVectorSDNode *V,
9457 bool HasDirectMove,
9458 bool HasP8Vector) {
9459 EVT VecVT = V->getValueType(ResNo: 0);
9460 bool RightType = VecVT == MVT::v2f64 ||
9461 (HasP8Vector && VecVT == MVT::v4f32) ||
9462 (HasDirectMove && (VecVT == MVT::v2i64 || VecVT == MVT::v4i32));
9463 if (!RightType)
9464 return false;
9465
9466 bool IsSplat = true;
9467 bool IsLoad = false;
9468 SDValue Op0 = V->getOperand(Num: 0);
9469
9470 // This function is called in a block that confirms the node is not a constant
9471 // splat. So a constant BUILD_VECTOR here means the vector is built out of
9472 // different constants.
9473 if (V->isConstant())
9474 return false;
9475 for (int i = 0, e = V->getNumOperands(); i < e; ++i) {
9476 if (V->getOperand(Num: i).isUndef())
9477 return false;
9478 // We want to expand nodes that represent load-and-splat even if the
9479 // loaded value is a floating point truncation or conversion to int.
9480 if (V->getOperand(Num: i).getOpcode() == ISD::LOAD ||
9481 (V->getOperand(Num: i).getOpcode() == ISD::FP_ROUND &&
9482 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9483 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_SINT &&
9484 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD) ||
9485 (V->getOperand(Num: i).getOpcode() == ISD::FP_TO_UINT &&
9486 V->getOperand(Num: i).getOperand(i: 0).getOpcode() == ISD::LOAD))
9487 IsLoad = true;
9488 // If the operands are different or the input is not a load and has more
9489 // uses than just this BV node, then it isn't a splat.
9490 if (V->getOperand(Num: i) != Op0 ||
9491 (!IsLoad && !V->isOnlyUserOf(N: V->getOperand(Num: i).getNode())))
9492 IsSplat = false;
9493 }
9494 return !(IsSplat && IsLoad);
9495}
9496
9497// Lower BITCAST(f128, (build_pair i64, i64)) to BUILD_FP128.
9498SDValue PPCTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
9499
9500 SDLoc dl(Op);
9501 SDValue Op0 = Op->getOperand(Num: 0);
9502
9503 if (!Subtarget.isPPC64() || (Op0.getOpcode() != ISD::BUILD_PAIR) ||
9504 (Op.getValueType() != MVT::f128))
9505 return SDValue();
9506
9507 SDValue Lo = Op0.getOperand(i: 0);
9508 SDValue Hi = Op0.getOperand(i: 1);
9509 if ((Lo.getValueType() != MVT::i64) || (Hi.getValueType() != MVT::i64))
9510 return SDValue();
9511
9512 if (!Subtarget.isLittleEndian())
9513 std::swap(a&: Lo, b&: Hi);
9514
9515 return DAG.getNode(Opcode: PPCISD::BUILD_FP128, DL: dl, VT: MVT::f128, N1: Lo, N2: Hi);
9516}
9517
9518// Look through bitcasts and scalar_to_vector ops to find the input load.
9519// If any of the nodes in the chain have multiple uses, stop searching as
9520// the input load won't be removable (due to the other uses).
9521static const SDValue *getNormalLoadInput(const SDValue &Op, bool &IsPermuted) {
9522 const SDValue *InputLoad = &Op;
9523 while (InputLoad->getOpcode() == ISD::BITCAST) {
9524 if (!InputLoad->hasOneUse())
9525 return nullptr;
9526 InputLoad = &InputLoad->getOperand(i: 0);
9527 }
9528 if (InputLoad->getOpcode() == ISD::SCALAR_TO_VECTOR ||
9529 InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED) {
9530 if (!InputLoad->hasOneUse())
9531 return nullptr;
9532 IsPermuted = InputLoad->getOpcode() == PPCISD::SCALAR_TO_VECTOR_PERMUTED;
9533 InputLoad = &InputLoad->getOperand(i: 0);
9534 }
9535 if (InputLoad->getOpcode() != ISD::LOAD)
9536 return nullptr;
9537 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9538 return (ISD::isNormalLoad(N: LD) && LD->hasOneUse()) ? InputLoad : nullptr;
9539}
9540
9541// Convert the argument APFloat to a single precision APFloat if there is no
9542// loss in information during the conversion to single precision APFloat and the
9543// resulting number is not a denormal number. Return true if successful.
9544bool llvm::convertToNonDenormSingle(APFloat &ArgAPFloat) {
9545 APFloat APFloatToConvert = ArgAPFloat;
9546 bool LosesInfo = true;
9547 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9548 losesInfo: &LosesInfo);
9549 bool Success = (!LosesInfo && !APFloatToConvert.isDenormal());
9550 if (Success)
9551 ArgAPFloat = APFloatToConvert;
9552 return Success;
9553}
9554
9555// Bitcast the argument APInt to a double and convert it to a single precision
9556// APFloat, bitcast the APFloat to an APInt and assign it to the original
9557// argument if there is no loss in information during the conversion from
9558// double to single precision APFloat and the resulting number is not a denormal
9559// number. Return true if successful.
9560bool llvm::convertToNonDenormSingle(APInt &ArgAPInt) {
9561 double DpValue = ArgAPInt.bitsToDouble();
9562 APFloat APFloatDp(DpValue);
9563 bool Success = convertToNonDenormSingle(ArgAPFloat&: APFloatDp);
9564 if (Success)
9565 ArgAPInt = APFloatDp.bitcastToAPInt();
9566 return Success;
9567}
9568
9569// Nondestructive check for convertTonNonDenormSingle.
9570bool llvm::checkConvertToNonDenormSingle(APFloat &ArgAPFloat) {
9571 // Only convert if it loses info, since XXSPLTIDP should
9572 // handle the other case.
9573 APFloat APFloatToConvert = ArgAPFloat;
9574 bool LosesInfo = true;
9575 APFloatToConvert.convert(ToSemantics: APFloat::IEEEsingle(), RM: APFloat::rmNearestTiesToEven,
9576 losesInfo: &LosesInfo);
9577
9578 return (!LosesInfo && !APFloatToConvert.isDenormal());
9579}
9580
9581static bool isValidSplatLoad(const PPCSubtarget &Subtarget, const SDValue &Op,
9582 unsigned &Opcode) {
9583 LoadSDNode *InputNode = dyn_cast<LoadSDNode>(Val: Op.getOperand(i: 0));
9584 if (!InputNode || !Subtarget.hasVSX() || !ISD::isUNINDEXEDLoad(N: InputNode))
9585 return false;
9586
9587 EVT Ty = Op->getValueType(ResNo: 0);
9588 // For v2f64, v4f32 and v4i32 types, we require the load to be non-extending
9589 // as we cannot handle extending loads for these types.
9590 if ((Ty == MVT::v2f64 || Ty == MVT::v4f32 || Ty == MVT::v4i32) &&
9591 ISD::isNON_EXTLoad(N: InputNode))
9592 return true;
9593
9594 EVT MemVT = InputNode->getMemoryVT();
9595 // For v8i16 and v16i8 types, extending loads can be handled as long as the
9596 // memory VT is the same vector element VT type.
9597 // The loads feeding into the v8i16 and v16i8 types will be extending because
9598 // scalar i8/i16 are not legal types.
9599 if ((Ty == MVT::v8i16 || Ty == MVT::v16i8) && ISD::isEXTLoad(N: InputNode) &&
9600 (MemVT == Ty.getVectorElementType()))
9601 return true;
9602
9603 if (Ty == MVT::v2i64) {
9604 // Check the extend type, when the input type is i32, and the output vector
9605 // type is v2i64.
9606 if (MemVT == MVT::i32) {
9607 if (ISD::isZEXTLoad(N: InputNode))
9608 Opcode = PPCISD::ZEXT_LD_SPLAT;
9609 if (ISD::isSEXTLoad(N: InputNode))
9610 Opcode = PPCISD::SEXT_LD_SPLAT;
9611 }
9612 return true;
9613 }
9614 return false;
9615}
9616
9617bool isValidMtVsrBmi(APInt &BitMask, BuildVectorSDNode &BVN,
9618 bool IsLittleEndian) {
9619 assert(BVN.getNumOperands() > 0 && "Unexpected 0-size build vector");
9620
9621 BitMask.clearAllBits();
9622 EVT VT = BVN.getValueType(ResNo: 0);
9623 unsigned VTSize = VT.getSizeInBits();
9624 APInt ConstValue(VTSize, 0);
9625
9626 unsigned EltWidth = VT.getScalarSizeInBits();
9627
9628 unsigned BitPos = 0;
9629 for (auto OpVal : BVN.op_values()) {
9630 auto *CN = dyn_cast<ConstantSDNode>(Val&: OpVal);
9631
9632 if (!CN)
9633 return false;
9634 // The elements in a vector register are ordered in reverse byte order
9635 // between little-endian and big-endian modes.
9636 ConstValue.insertBits(SubBits: CN->getAPIntValue().zextOrTrunc(width: EltWidth),
9637 bitPosition: IsLittleEndian ? BitPos : VTSize - EltWidth - BitPos);
9638 BitPos += EltWidth;
9639 }
9640
9641 for (unsigned J = 0; J < 16; ++J) {
9642 APInt ExtractValue = ConstValue.extractBits(numBits: 8, bitPosition: J * 8);
9643 if (ExtractValue != 0x00 && ExtractValue != 0xFF)
9644 return false;
9645 if (ExtractValue == 0xFF)
9646 BitMask.setBit(J);
9647 }
9648 return true;
9649}
9650
9651// If this is a case we can't handle, return null and let the default
9652// expansion code take care of it. If we CAN select this case, and if it
9653// selects to a single instruction, return Op. Otherwise, if we can codegen
9654// this case more efficiently than a constant pool load, lower it to the
9655// sequence of ops that should be used.
9656SDValue PPCTargetLowering::LowerBUILD_VECTOR(SDValue Op,
9657 SelectionDAG &DAG) const {
9658 SDLoc dl(Op);
9659 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: Op.getNode());
9660 assert(BVN && "Expected a BuildVectorSDNode in LowerBUILD_VECTOR");
9661
9662 if (Subtarget.hasP10Vector()) {
9663 APInt BitMask(32, 0);
9664 // If the value of the vector is all zeros or all ones,
9665 // we do not convert it to MTVSRBMI.
9666 // The xxleqv instruction sets a vector with all ones.
9667 // The xxlxor instruction sets a vector with all zeros.
9668 if (isValidMtVsrBmi(BitMask, BVN&: *BVN, IsLittleEndian: Subtarget.isLittleEndian()) &&
9669 BitMask != 0 && BitMask != 0xffff) {
9670 SDValue SDConstant = DAG.getTargetConstant(Val: BitMask, DL: dl, VT: MVT::i32);
9671 MachineSDNode *MSDNode =
9672 DAG.getMachineNode(Opcode: PPC::MTVSRBMI, dl, VT: MVT::v16i8, Op1: SDConstant);
9673 SDValue SDV = SDValue(MSDNode, 0);
9674 EVT DVT = BVN->getValueType(ResNo: 0);
9675 EVT SVT = SDV.getValueType();
9676 if (SVT != DVT) {
9677 SDV = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: DVT, Operand: SDV);
9678 }
9679 return SDV;
9680 }
9681 // Recognize build vector patterns to emit VSX vector instructions
9682 // instead of loading value from memory.
9683 if (SDValue VecPat = combineBVLoadsSpecialValue(Operand: Op, DAG))
9684 return VecPat;
9685 }
9686 // Check if this is a splat of a constant value.
9687 APInt APSplatBits, APSplatUndef;
9688 unsigned SplatBitSize = 0;
9689 bool HasAnyUndefs;
9690 bool BVNIsConstantSplat =
9691 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
9692 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
9693
9694 // If it is a splat of a double, check if we can shrink it to a 32 bit
9695 // non-denormal float which when converted back to double gives us the same
9696 // double. This is to exploit the XXSPLTIDP instruction.
9697 // If we lose precision, we use XXSPLTI32DX.
9698 if (BVNIsConstantSplat && (SplatBitSize == 64) &&
9699 Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
9700 // Check the type first to short-circuit so we don't modify APSplatBits if
9701 // this block isn't executed.
9702 if ((Op->getValueType(ResNo: 0) == MVT::v2f64) &&
9703 convertToNonDenormSingle(ArgAPInt&: APSplatBits)) {
9704 SDValue SplatNode = DAG.getNode(
9705 Opcode: PPCISD::XXSPLTI_SP_TO_DP, DL: dl, VT: MVT::v2f64,
9706 Operand: DAG.getTargetConstant(Val: APSplatBits.getZExtValue(), DL: dl, VT: MVT::i32));
9707 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9708 } else {
9709 // We may lose precision, so we have to use XXSPLTI32DX.
9710
9711 uint32_t Hi = Hi_32(Value: APSplatBits.getZExtValue());
9712 uint32_t Lo = Lo_32(Value: APSplatBits.getZExtValue());
9713 SDValue SplatNode = DAG.getUNDEF(VT: MVT::v2i64);
9714
9715 if (!Hi || !Lo)
9716 // If either load is 0, then we should generate XXLXOR to set to 0.
9717 SplatNode = DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::v2i64);
9718
9719 if (Hi)
9720 SplatNode = DAG.getNode(
9721 Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9722 N2: DAG.getTargetConstant(Val: 0, DL: dl, VT: MVT::i32),
9723 N3: DAG.getTargetConstant(Val: Hi, DL: dl, VT: MVT::i32));
9724
9725 if (Lo)
9726 SplatNode =
9727 DAG.getNode(Opcode: PPCISD::XXSPLTI32DX, DL: dl, VT: MVT::v2i64, N1: SplatNode,
9728 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i32),
9729 N3: DAG.getTargetConstant(Val: Lo, DL: dl, VT: MVT::i32));
9730
9731 return DAG.getBitcast(VT: Op.getValueType(), V: SplatNode);
9732 }
9733 }
9734
9735 if (SDValue V =
9736 LowerVecSplatSmallFP(Op, DAG, BVNIsConstantSplat, SplatBitSize))
9737 return V;
9738
9739 bool IsSplat64 = false;
9740 uint64_t SplatBits = 0;
9741 int32_t SextVal = 0;
9742 if (BVNIsConstantSplat && SplatBitSize <= 64) {
9743 SplatBits = APSplatBits.getZExtValue();
9744 if (SplatBitSize <= 32) {
9745 SextVal = SignExtend32(X: SplatBits, B: SplatBitSize);
9746 } else if (SplatBitSize == 64 && Subtarget.hasP8Altivec()) {
9747 int64_t Splat64Val = static_cast<int64_t>(SplatBits);
9748 bool P9Vector = Subtarget.hasP9Vector();
9749 int32_t Hi = P9Vector ? 127 : 15;
9750 int32_t Lo = P9Vector ? -128 : -16;
9751 IsSplat64 = Splat64Val >= Lo && Splat64Val <= Hi;
9752 SextVal = static_cast<int32_t>(SplatBits);
9753 }
9754 }
9755
9756 if (!BVNIsConstantSplat || (SplatBitSize > 32 && !IsSplat64)) {
9757 unsigned NewOpcode = PPCISD::LD_SPLAT;
9758
9759 // Handle load-and-splat patterns as we have instructions that will do this
9760 // in one go.
9761 if (DAG.isSplatValue(V: Op, AllowUndefs: true) &&
9762 isValidSplatLoad(Subtarget, Op, Opcode&: NewOpcode)) {
9763 const SDValue *InputLoad = &Op.getOperand(i: 0);
9764 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
9765
9766 // If the input load is an extending load, it will be an i32 -> i64
9767 // extending load and isValidSplatLoad() will update NewOpcode.
9768 unsigned MemorySize = LD->getMemoryVT().getScalarSizeInBits();
9769 unsigned ElementSize =
9770 MemorySize * ((NewOpcode == PPCISD::LD_SPLAT) ? 1 : 2);
9771
9772 assert(((ElementSize == 2 * MemorySize)
9773 ? (NewOpcode == PPCISD::ZEXT_LD_SPLAT ||
9774 NewOpcode == PPCISD::SEXT_LD_SPLAT)
9775 : (NewOpcode == PPCISD::LD_SPLAT)) &&
9776 "Unmatched element size and opcode!\n");
9777
9778 // Checking for a single use of this load, we have to check for vector
9779 // width (128 bits) / ElementSize uses (since each operand of the
9780 // BUILD_VECTOR is a separate use of the value.
9781 unsigned NumUsesOfInputLD = 128 / ElementSize;
9782 for (SDValue BVInOp : Op->ops())
9783 if (BVInOp.isUndef())
9784 NumUsesOfInputLD--;
9785
9786 // Exclude somes case where LD_SPLAT is worse than scalar_to_vector:
9787 // Below cases should also happen for "lfiwzx/lfiwax + LE target + index
9788 // 1" and "lxvrhx + BE target + index 7" and "lxvrbx + BE target + index
9789 // 15", but function IsValidSplatLoad() now will only return true when
9790 // the data at index 0 is not nullptr. So we will not get into trouble for
9791 // these cases.
9792 //
9793 // case 1 - lfiwzx/lfiwax
9794 // 1.1: load result is i32 and is sign/zero extend to i64;
9795 // 1.2: build a v2i64 vector type with above loaded value;
9796 // 1.3: the vector has only one value at index 0, others are all undef;
9797 // 1.4: on BE target, so that lfiwzx/lfiwax does not need any permute.
9798 if (NumUsesOfInputLD == 1 &&
9799 (Op->getValueType(ResNo: 0) == MVT::v2i64 && NewOpcode != PPCISD::LD_SPLAT &&
9800 !Subtarget.isLittleEndian() && Subtarget.hasVSX() &&
9801 Subtarget.hasLFIWAX()))
9802 return SDValue();
9803
9804 // case 2 - lxvr[hb]x
9805 // 2.1: load result is at most i16;
9806 // 2.2: build a vector with above loaded value;
9807 // 2.3: the vector has only one value at index 0, others are all undef;
9808 // 2.4: on LE target, so that lxvr[hb]x does not need any permute.
9809 if (NumUsesOfInputLD == 1 && Subtarget.isLittleEndian() &&
9810 Subtarget.isISA3_1() && ElementSize <= 16)
9811 return SDValue();
9812
9813 assert(NumUsesOfInputLD > 0 && "No uses of input LD of a build_vector?");
9814 if (InputLoad->getNode()->hasNUsesOfValue(NUses: NumUsesOfInputLD, Value: 0) &&
9815 Subtarget.hasVSX()) {
9816 SDValue Ops[] = {
9817 LD->getChain(), // Chain
9818 LD->getBasePtr(), // Ptr
9819 DAG.getValueType(Op.getValueType()) // VT
9820 };
9821 SDValue LdSplt = DAG.getMemIntrinsicNode(
9822 Opcode: NewOpcode, dl, VTList: DAG.getVTList(VT1: Op.getValueType(), VT2: MVT::Other), Ops,
9823 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
9824 // Replace all uses of the output chain of the original load with the
9825 // output chain of the new load.
9826 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1),
9827 To: LdSplt.getValue(R: 1));
9828 return LdSplt;
9829 }
9830 }
9831
9832 // In 64BIT mode BUILD_VECTOR nodes that are not constant splats of up to
9833 // 32-bits can be lowered to VSX instructions under certain conditions.
9834 // Without VSX, there is no pattern more efficient than expanding the node.
9835 if (Subtarget.hasVSX() && Subtarget.isPPC64() &&
9836 haveEfficientBuildVectorPattern(V: BVN, HasDirectMove: Subtarget.hasDirectMove(),
9837 HasP8Vector: Subtarget.hasP8Vector()))
9838 return Op;
9839 return SDValue();
9840 }
9841
9842 uint64_t SplatUndef = APSplatUndef.getZExtValue();
9843 unsigned SplatSize = SplatBitSize / 8;
9844
9845 // First, handle single instruction cases.
9846
9847 // All zeros?
9848 if (SplatBits == 0) {
9849 // Canonicalize all zero vectors to be v4i32.
9850 if (Op.getValueType() != MVT::v4i32 || HasAnyUndefs) {
9851 SDValue Z = DAG.getConstant(Val: 0, DL: dl, VT: MVT::v4i32);
9852 Op = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Z);
9853 }
9854 return Op;
9855 }
9856
9857 // We have XXSPLTIW for constant splats four bytes wide.
9858 // Given vector length is a multiple of 4, 2-byte splats can be replaced
9859 // with 4-byte splats. We replicate the SplatBits in case of 2-byte splat to
9860 // make a 4-byte splat element. For example: 2-byte splat of 0xABAB can be
9861 // turned into a 4-byte splat of 0xABABABAB.
9862 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 2)
9863 return getCanonicalConstSplat(Val: SplatBits | (SplatBits << 16), SplatSize: SplatSize * 2,
9864 VT: Op.getValueType(), DAG, dl);
9865
9866 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector() && SplatSize == 4)
9867 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9868 dl);
9869
9870 // We have XXSPLTIB for constant splats one byte wide.
9871 if (Subtarget.hasP9Vector() && SplatSize == 1)
9872 return getCanonicalConstSplat(Val: SplatBits, SplatSize, VT: Op.getValueType(), DAG,
9873 dl);
9874
9875 // If the sign extended value is in the range [-16,15], use VSPLTI[bhw].
9876 // Use VSPLTIW/VUPKLSW for v2i64 in range [-16,15].
9877 if (SextVal >= -16 && SextVal <= 15) {
9878 // SplatSize may be 1, 2, 4, or 8. Use size 4 instead of 8 for the splat to
9879 // generate a splat word with extend for size 8.
9880 unsigned UseSize = SplatSize == 8 ? 4 : SplatSize;
9881 SDValue Res =
9882 getCanonicalConstSplat(Val: SextVal, SplatSize: UseSize, VT: Op.getValueType(), DAG, dl);
9883 if (SplatSize != 8)
9884 return Res;
9885 SDValue IntrinsicOp =
9886 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vupklsw,
9887 Op: DAG.getBitcast(VT: MVT::v4i32, V: Res), DAG, dl, DestVT: MVT::v2i64);
9888 return DAG.getBitcast(VT: Op.getValueType(), V: IntrinsicOp);
9889 }
9890
9891 // Two instruction sequences.
9892
9893 if (Subtarget.hasP9Vector() && SextVal >= -128 && SextVal <= 127) {
9894 SDValue C = DAG.getConstant(Val: (unsigned char)SextVal, DL: dl, VT: MVT::i32);
9895 SmallVector<SDValue, 16> Ops(16, C);
9896 SDValue BV = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops);
9897 unsigned IID;
9898 EVT VT;
9899 switch (SplatSize) {
9900 default:
9901 llvm_unreachable("Unexpected type for vector constant.");
9902 case 2:
9903 IID = Intrinsic::ppc_altivec_vupklsb;
9904 VT = MVT::v8i16;
9905 break;
9906 case 4:
9907 IID = Intrinsic::ppc_altivec_vextsb2w;
9908 VT = MVT::v4i32;
9909 break;
9910 case 8:
9911 IID = Intrinsic::ppc_altivec_vextsb2d;
9912 VT = MVT::v2i64;
9913 break;
9914 }
9915 SDValue Extend = BuildIntrinsicOp(IID, Op: BV, DAG, dl, DestVT: VT);
9916 return DAG.getBitcast(VT: Op->getValueType(ResNo: 0), V: Extend);
9917 }
9918 assert(!IsSplat64 && "Unhandled 64-bit splat pattern");
9919
9920 // If this value is in the range [-32,30] and is even, use:
9921 // VSPLTI[bhw](val/2) + VSPLTI[bhw](val/2)
9922 // If this value is in the range [17,31] and is odd, use:
9923 // VSPLTI[bhw](val-16) - VSPLTI[bhw](-16)
9924 // If this value is in the range [-31,-17] and is odd, use:
9925 // VSPLTI[bhw](val+16) + VSPLTI[bhw](-16)
9926 // Note the last two are three-instruction sequences.
9927 if (SextVal >= -32 && SextVal <= 31) {
9928 // To avoid having these optimizations undone by constant folding,
9929 // we convert to a pseudo that will be expanded later into one of
9930 // the above forms.
9931 SDValue Elt = DAG.getSignedConstant(Val: SextVal, DL: dl, VT: MVT::i32);
9932 EVT VT = (SplatSize == 1 ? MVT::v16i8 :
9933 (SplatSize == 2 ? MVT::v8i16 : MVT::v4i32));
9934 SDValue EltSize = DAG.getConstant(Val: SplatSize, DL: dl, VT: MVT::i32);
9935 SDValue RetVal = DAG.getNode(Opcode: PPCISD::VADD_SPLAT, DL: dl, VT, N1: Elt, N2: EltSize);
9936 if (VT == Op.getValueType())
9937 return RetVal;
9938 else
9939 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: RetVal);
9940 }
9941
9942 // If this is 0x8000_0000 x 4, turn into vspltisw + vslw. If it is
9943 // 0x7FFF_FFFF x 4, turn it into not(0x8000_0000). This is important
9944 // for fneg/fabs.
9945 if (SplatSize == 4 && SplatBits == (0x7FFFFFFF&~SplatUndef)) {
9946 // Make -1 and vspltisw -1:
9947 SDValue OnesV = getCanonicalConstSplat(Val: -1, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
9948
9949 // Make the VSLW intrinsic, computing 0x8000_0000.
9950 SDValue Res = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: OnesV,
9951 RHS: OnesV, DAG, dl);
9952
9953 // xor by OnesV to invert it.
9954 Res = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::v4i32, N1: Res, N2: OnesV);
9955 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9956 }
9957
9958 // Check to see if this is a wide variety of vsplti*, binop self cases.
9959 static const signed char SplatCsts[] = {
9960 -1, 1, -2, 2, -3, 3, -4, 4, -5, 5, -6, 6, -7, 7,
9961 -8, 8, -9, 9, -10, 10, -11, 11, -12, 12, -13, 13, 14, -14, 15, -15, -16
9962 };
9963
9964 for (unsigned idx = 0; idx < std::size(SplatCsts); ++idx) {
9965 // Indirect through the SplatCsts array so that we favor 'vsplti -1' for
9966 // cases which are ambiguous (e.g. formation of 0x8000_0000). 'vsplti -1'
9967 int i = SplatCsts[idx];
9968
9969 // Figure out what shift amount will be used by altivec if shifted by i in
9970 // this splat size.
9971 unsigned TypeShiftAmt = i & (SplatBitSize-1);
9972
9973 // vsplti + shl self.
9974 if (SextVal == (int)((unsigned)i << TypeShiftAmt)) {
9975 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9976 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9977 Intrinsic::ppc_altivec_vslb, Intrinsic::ppc_altivec_vslh, 0,
9978 Intrinsic::ppc_altivec_vslw
9979 };
9980 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9981 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9982 }
9983
9984 // vsplti + srl self.
9985 if (SextVal == (int)((unsigned)i >> TypeShiftAmt)) {
9986 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9987 static const unsigned IIDs[] = { // Intrinsic to use for each size.
9988 Intrinsic::ppc_altivec_vsrb, Intrinsic::ppc_altivec_vsrh, 0,
9989 Intrinsic::ppc_altivec_vsrw
9990 };
9991 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
9992 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
9993 }
9994
9995 // vsplti + rol self.
9996 if (SextVal == (int)(((unsigned)i << TypeShiftAmt) |
9997 ((unsigned)i >> (SplatBitSize-TypeShiftAmt)))) {
9998 SDValue Res = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::Other, DAG, dl);
9999 static const unsigned IIDs[] = { // Intrinsic to use for each size.
10000 Intrinsic::ppc_altivec_vrlb, Intrinsic::ppc_altivec_vrlh, 0,
10001 Intrinsic::ppc_altivec_vrlw
10002 };
10003 Res = BuildIntrinsicOp(IID: IIDs[SplatSize-1], LHS: Res, RHS: Res, DAG, dl);
10004 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Res);
10005 }
10006
10007 // t = vsplti c, result = vsldoi t, t, 1
10008 if (SextVal == (int)(((unsigned)i << 8) | (i < 0 ? 0xFF : 0))) {
10009 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10010 unsigned Amt = Subtarget.isLittleEndian() ? 15 : 1;
10011 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10012 }
10013 // t = vsplti c, result = vsldoi t, t, 2
10014 if (SextVal == (int)(((unsigned)i << 16) | (i < 0 ? 0xFFFF : 0))) {
10015 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10016 unsigned Amt = Subtarget.isLittleEndian() ? 14 : 2;
10017 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10018 }
10019 // t = vsplti c, result = vsldoi t, t, 3
10020 if (SextVal == (int)(((unsigned)i << 24) | (i < 0 ? 0xFFFFFF : 0))) {
10021 SDValue T = getCanonicalConstSplat(Val: i, SplatSize, VT: MVT::v16i8, DAG, dl);
10022 unsigned Amt = Subtarget.isLittleEndian() ? 13 : 3;
10023 return BuildVSLDOI(LHS: T, RHS: T, Amt, VT: Op.getValueType(), DAG, dl);
10024 }
10025 }
10026
10027 return SDValue();
10028}
10029
10030/// GeneratePerfectShuffle - Given an entry in the perfect-shuffle table, emit
10031/// the specified operations to build the shuffle.
10032static SDValue GeneratePerfectShuffle(unsigned PFEntry, SDValue LHS,
10033 SDValue RHS, SelectionDAG &DAG,
10034 const SDLoc &dl) {
10035 unsigned OpNum = (PFEntry >> 26) & 0x0F;
10036 unsigned LHSID = (PFEntry >> 13) & ((1 << 13)-1);
10037 unsigned RHSID = (PFEntry >> 0) & ((1 << 13)-1);
10038
10039 enum {
10040 OP_COPY = 0, // Copy, used for things like <u,u,u,3> to say it is <0,1,2,3>
10041 OP_VMRGHW,
10042 OP_VMRGLW,
10043 OP_VSPLTISW0,
10044 OP_VSPLTISW1,
10045 OP_VSPLTISW2,
10046 OP_VSPLTISW3,
10047 OP_VSLDOI4,
10048 OP_VSLDOI8,
10049 OP_VSLDOI12
10050 };
10051
10052 if (OpNum == OP_COPY) {
10053 if (LHSID == (1*9+2)*9+3) return LHS;
10054 assert(LHSID == ((4*9+5)*9+6)*9+7 && "Illegal OP_COPY!");
10055 return RHS;
10056 }
10057
10058 SDValue OpLHS, OpRHS;
10059 OpLHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[LHSID], LHS, RHS, DAG, dl);
10060 OpRHS = GeneratePerfectShuffle(PFEntry: PerfectShuffleTable[RHSID], LHS, RHS, DAG, dl);
10061
10062 int ShufIdxs[16];
10063 switch (OpNum) {
10064 default: llvm_unreachable("Unknown i32 permute!");
10065 case OP_VMRGHW:
10066 ShufIdxs[ 0] = 0; ShufIdxs[ 1] = 1; ShufIdxs[ 2] = 2; ShufIdxs[ 3] = 3;
10067 ShufIdxs[ 4] = 16; ShufIdxs[ 5] = 17; ShufIdxs[ 6] = 18; ShufIdxs[ 7] = 19;
10068 ShufIdxs[ 8] = 4; ShufIdxs[ 9] = 5; ShufIdxs[10] = 6; ShufIdxs[11] = 7;
10069 ShufIdxs[12] = 20; ShufIdxs[13] = 21; ShufIdxs[14] = 22; ShufIdxs[15] = 23;
10070 break;
10071 case OP_VMRGLW:
10072 ShufIdxs[ 0] = 8; ShufIdxs[ 1] = 9; ShufIdxs[ 2] = 10; ShufIdxs[ 3] = 11;
10073 ShufIdxs[ 4] = 24; ShufIdxs[ 5] = 25; ShufIdxs[ 6] = 26; ShufIdxs[ 7] = 27;
10074 ShufIdxs[ 8] = 12; ShufIdxs[ 9] = 13; ShufIdxs[10] = 14; ShufIdxs[11] = 15;
10075 ShufIdxs[12] = 28; ShufIdxs[13] = 29; ShufIdxs[14] = 30; ShufIdxs[15] = 31;
10076 break;
10077 case OP_VSPLTISW0:
10078 for (unsigned i = 0; i != 16; ++i)
10079 ShufIdxs[i] = (i&3)+0;
10080 break;
10081 case OP_VSPLTISW1:
10082 for (unsigned i = 0; i != 16; ++i)
10083 ShufIdxs[i] = (i&3)+4;
10084 break;
10085 case OP_VSPLTISW2:
10086 for (unsigned i = 0; i != 16; ++i)
10087 ShufIdxs[i] = (i&3)+8;
10088 break;
10089 case OP_VSPLTISW3:
10090 for (unsigned i = 0; i != 16; ++i)
10091 ShufIdxs[i] = (i&3)+12;
10092 break;
10093 case OP_VSLDOI4:
10094 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 4, VT: OpLHS.getValueType(), DAG, dl);
10095 case OP_VSLDOI8:
10096 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 8, VT: OpLHS.getValueType(), DAG, dl);
10097 case OP_VSLDOI12:
10098 return BuildVSLDOI(LHS: OpLHS, RHS: OpRHS, Amt: 12, VT: OpLHS.getValueType(), DAG, dl);
10099 }
10100 EVT VT = OpLHS.getValueType();
10101 OpLHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpLHS);
10102 OpRHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OpRHS);
10103 SDValue T = DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OpLHS, N2: OpRHS, Mask: ShufIdxs);
10104 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: T);
10105}
10106
10107/// lowerToVINSERTB - Return the SDValue if this VECTOR_SHUFFLE can be handled
10108/// by the VINSERTB instruction introduced in ISA 3.0, else just return default
10109/// SDValue.
10110SDValue PPCTargetLowering::lowerToVINSERTB(ShuffleVectorSDNode *N,
10111 SelectionDAG &DAG) const {
10112 const unsigned BytesInVector = 16;
10113 bool IsLE = Subtarget.isLittleEndian();
10114 SDLoc dl(N);
10115 SDValue V1 = N->getOperand(Num: 0);
10116 SDValue V2 = N->getOperand(Num: 1);
10117 unsigned ShiftElts = 0, InsertAtByte = 0;
10118 bool Swap = false;
10119
10120 // Shifts required to get the byte we want at element 7.
10121 unsigned LittleEndianShifts[] = {8, 7, 6, 5, 4, 3, 2, 1,
10122 0, 15, 14, 13, 12, 11, 10, 9};
10123 unsigned BigEndianShifts[] = {9, 10, 11, 12, 13, 14, 15, 0,
10124 1, 2, 3, 4, 5, 6, 7, 8};
10125
10126 ArrayRef<int> Mask = N->getMask();
10127 int OriginalOrder[] = {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15};
10128
10129 // For each mask element, find out if we're just inserting something
10130 // from V2 into V1 or vice versa.
10131 // Possible permutations inserting an element from V2 into V1:
10132 // X, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10133 // 0, X, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15
10134 // ...
10135 // 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, X
10136 // Inserting from V1 into V2 will be similar, except mask range will be
10137 // [16,31].
10138
10139 bool FoundCandidate = false;
10140 // If both vector operands for the shuffle are the same vector, the mask
10141 // will contain only elements from the first one and the second one will be
10142 // undef.
10143 unsigned VINSERTBSrcElem = IsLE ? 8 : 7;
10144 // Go through the mask of half-words to find an element that's being moved
10145 // from one vector to the other.
10146 for (unsigned i = 0; i < BytesInVector; ++i) {
10147 unsigned CurrentElement = Mask[i];
10148 // If 2nd operand is undefined, we should only look for element 7 in the
10149 // Mask.
10150 if (V2.isUndef() && CurrentElement != VINSERTBSrcElem)
10151 continue;
10152
10153 bool OtherElementsInOrder = true;
10154 // Examine the other elements in the Mask to see if they're in original
10155 // order.
10156 for (unsigned j = 0; j < BytesInVector; ++j) {
10157 if (j == i)
10158 continue;
10159 // If CurrentElement is from V1 [0,15], then we the rest of the Mask to be
10160 // from V2 [16,31] and vice versa. Unless the 2nd operand is undefined,
10161 // in which we always assume we're always picking from the 1st operand.
10162 int MaskOffset =
10163 (!V2.isUndef() && CurrentElement < BytesInVector) ? BytesInVector : 0;
10164 if (Mask[j] != OriginalOrder[j] + MaskOffset) {
10165 OtherElementsInOrder = false;
10166 break;
10167 }
10168 }
10169 // If other elements are in original order, we record the number of shifts
10170 // we need to get the element we want into element 7. Also record which byte
10171 // in the vector we should insert into.
10172 if (OtherElementsInOrder) {
10173 // If 2nd operand is undefined, we assume no shifts and no swapping.
10174 if (V2.isUndef()) {
10175 ShiftElts = 0;
10176 Swap = false;
10177 } else {
10178 // Only need the last 4-bits for shifts because operands will be swapped if CurrentElement is >= 2^4.
10179 ShiftElts = IsLE ? LittleEndianShifts[CurrentElement & 0xF]
10180 : BigEndianShifts[CurrentElement & 0xF];
10181 Swap = CurrentElement < BytesInVector;
10182 }
10183 InsertAtByte = IsLE ? BytesInVector - (i + 1) : i;
10184 FoundCandidate = true;
10185 break;
10186 }
10187 }
10188
10189 if (!FoundCandidate)
10190 return SDValue();
10191
10192 // Candidate found, construct the proper SDAG sequence with VINSERTB,
10193 // optionally with VECSHL if shift is required.
10194 if (Swap)
10195 std::swap(a&: V1, b&: V2);
10196 if (V2.isUndef())
10197 V2 = V1;
10198 if (ShiftElts) {
10199 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10200 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10201 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: Shl,
10202 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10203 }
10204 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v16i8, N1: V1, N2: V2,
10205 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10206}
10207
10208/// lowerToVINSERTH - Return the SDValue if this VECTOR_SHUFFLE can be handled
10209/// by the VINSERTH instruction introduced in ISA 3.0, else just return default
10210/// SDValue.
10211SDValue PPCTargetLowering::lowerToVINSERTH(ShuffleVectorSDNode *N,
10212 SelectionDAG &DAG) const {
10213 const unsigned NumHalfWords = 8;
10214 const unsigned BytesInVector = NumHalfWords * 2;
10215 // Check that the shuffle is on half-words.
10216 if (!isNByteElemShuffleMask(N, Width: 2, StepLen: 1))
10217 return SDValue();
10218
10219 bool IsLE = Subtarget.isLittleEndian();
10220 SDLoc dl(N);
10221 SDValue V1 = N->getOperand(Num: 0);
10222 SDValue V2 = N->getOperand(Num: 1);
10223 unsigned ShiftElts = 0, InsertAtByte = 0;
10224 bool Swap = false;
10225
10226 // Shifts required to get the half-word we want at element 3.
10227 unsigned LittleEndianShifts[] = {4, 3, 2, 1, 0, 7, 6, 5};
10228 unsigned BigEndianShifts[] = {5, 6, 7, 0, 1, 2, 3, 4};
10229
10230 uint32_t Mask = 0;
10231 uint32_t OriginalOrderLow = 0x1234567;
10232 uint32_t OriginalOrderHigh = 0x89ABCDEF;
10233 // Now we look at mask elements 0,2,4,6,8,10,12,14. Pack the mask into a
10234 // 32-bit space, only need 4-bit nibbles per element.
10235 for (unsigned i = 0; i < NumHalfWords; ++i) {
10236 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10237 Mask |= ((uint32_t)(N->getMaskElt(Idx: i * 2) / 2) << MaskShift);
10238 }
10239
10240 // For each mask element, find out if we're just inserting something
10241 // from V2 into V1 or vice versa. Possible permutations inserting an element
10242 // from V2 into V1:
10243 // X, 1, 2, 3, 4, 5, 6, 7
10244 // 0, X, 2, 3, 4, 5, 6, 7
10245 // 0, 1, X, 3, 4, 5, 6, 7
10246 // 0, 1, 2, X, 4, 5, 6, 7
10247 // 0, 1, 2, 3, X, 5, 6, 7
10248 // 0, 1, 2, 3, 4, X, 6, 7
10249 // 0, 1, 2, 3, 4, 5, X, 7
10250 // 0, 1, 2, 3, 4, 5, 6, X
10251 // Inserting from V1 into V2 will be similar, except mask range will be [8,15].
10252
10253 bool FoundCandidate = false;
10254 // Go through the mask of half-words to find an element that's being moved
10255 // from one vector to the other.
10256 for (unsigned i = 0; i < NumHalfWords; ++i) {
10257 unsigned MaskShift = (NumHalfWords - 1 - i) * 4;
10258 uint32_t MaskOneElt = (Mask >> MaskShift) & 0xF;
10259 uint32_t MaskOtherElts = ~(0xF << MaskShift);
10260 uint32_t TargetOrder = 0x0;
10261
10262 // If both vector operands for the shuffle are the same vector, the mask
10263 // will contain only elements from the first one and the second one will be
10264 // undef.
10265 if (V2.isUndef()) {
10266 ShiftElts = 0;
10267 unsigned VINSERTHSrcElem = IsLE ? 4 : 3;
10268 TargetOrder = OriginalOrderLow;
10269 Swap = false;
10270 // Skip if not the correct element or mask of other elements don't equal
10271 // to our expected order.
10272 if (MaskOneElt == VINSERTHSrcElem &&
10273 (Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10274 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10275 FoundCandidate = true;
10276 break;
10277 }
10278 } else { // If both operands are defined.
10279 // Target order is [8,15] if the current mask is between [0,7].
10280 TargetOrder =
10281 (MaskOneElt < NumHalfWords) ? OriginalOrderHigh : OriginalOrderLow;
10282 // Skip if mask of other elements don't equal our expected order.
10283 if ((Mask & MaskOtherElts) == (TargetOrder & MaskOtherElts)) {
10284 // We only need the last 3 bits for the number of shifts.
10285 ShiftElts = IsLE ? LittleEndianShifts[MaskOneElt & 0x7]
10286 : BigEndianShifts[MaskOneElt & 0x7];
10287 InsertAtByte = IsLE ? BytesInVector - (i + 1) * 2 : i * 2;
10288 Swap = MaskOneElt < NumHalfWords;
10289 FoundCandidate = true;
10290 break;
10291 }
10292 }
10293 }
10294
10295 if (!FoundCandidate)
10296 return SDValue();
10297
10298 // Candidate found, construct the proper SDAG sequence with VINSERTH,
10299 // optionally with VECSHL if shift is required.
10300 if (Swap)
10301 std::swap(a&: V1, b&: V2);
10302 if (V2.isUndef())
10303 V2 = V1;
10304 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10305 if (ShiftElts) {
10306 // Double ShiftElts because we're left shifting on v16i8 type.
10307 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v16i8, N1: V2, N2: V2,
10308 N3: DAG.getConstant(Val: 2 * ShiftElts, DL: dl, VT: MVT::i32));
10309 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: Shl);
10310 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10311 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10312 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10313 }
10314 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V2);
10315 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v8i16, N1: Conv1, N2: Conv2,
10316 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10317 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10318}
10319
10320/// lowerToXXSPLTI32DX - Return the SDValue if this VECTOR_SHUFFLE can be
10321/// handled by the XXSPLTI32DX instruction introduced in ISA 3.1, otherwise
10322/// return the default SDValue.
10323SDValue PPCTargetLowering::lowerToXXSPLTI32DX(ShuffleVectorSDNode *SVN,
10324 SelectionDAG &DAG) const {
10325 // The LHS and RHS may be bitcasts to v16i8 as we canonicalize shuffles
10326 // to v16i8. Peek through the bitcasts to get the actual operands.
10327 SDValue LHS = peekThroughBitcasts(V: SVN->getOperand(Num: 0));
10328 SDValue RHS = peekThroughBitcasts(V: SVN->getOperand(Num: 1));
10329
10330 auto ShuffleMask = SVN->getMask();
10331 SDValue VecShuffle(SVN, 0);
10332 SDLoc DL(SVN);
10333
10334 // Check that we have a four byte shuffle.
10335 if (!isNByteElemShuffleMask(N: SVN, Width: 4, StepLen: 1))
10336 return SDValue();
10337
10338 // Canonicalize the RHS being a BUILD_VECTOR when lowering to xxsplti32dx.
10339 if (RHS->getOpcode() != ISD::BUILD_VECTOR) {
10340 std::swap(a&: LHS, b&: RHS);
10341 VecShuffle = peekThroughBitcasts(V: DAG.getCommutedVectorShuffle(SV: *SVN));
10342 ShuffleVectorSDNode *CommutedSV = dyn_cast<ShuffleVectorSDNode>(Val&: VecShuffle);
10343 if (!CommutedSV)
10344 return SDValue();
10345 ShuffleMask = CommutedSV->getMask();
10346 }
10347
10348 // Ensure that the RHS is a vector of constants.
10349 BuildVectorSDNode *BVN = dyn_cast<BuildVectorSDNode>(Val: RHS.getNode());
10350 if (!BVN)
10351 return SDValue();
10352
10353 // Check if RHS is a splat of 4-bytes (or smaller).
10354 APInt APSplatValue, APSplatUndef;
10355 unsigned SplatBitSize;
10356 bool HasAnyUndefs;
10357 if (!BVN->isConstantSplat(SplatValue&: APSplatValue, SplatUndef&: APSplatUndef, SplatBitSize,
10358 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian()) ||
10359 SplatBitSize > 32)
10360 return SDValue();
10361
10362 // Check that the shuffle mask matches the semantics of XXSPLTI32DX.
10363 // The instruction splats a constant C into two words of the source vector
10364 // producing { C, Unchanged, C, Unchanged } or { Unchanged, C, Unchanged, C }.
10365 // Thus we check that the shuffle mask is the equivalent of
10366 // <0, [4-7], 2, [4-7]> or <[4-7], 1, [4-7], 3> respectively.
10367 // Note: the check above of isNByteElemShuffleMask() ensures that the bytes
10368 // within each word are consecutive, so we only need to check the first byte.
10369 SDValue Index;
10370 bool IsLE = Subtarget.isLittleEndian();
10371 if ((ShuffleMask[0] == 0 && ShuffleMask[8] == 8) &&
10372 (ShuffleMask[4] % 4 == 0 && ShuffleMask[12] % 4 == 0 &&
10373 ShuffleMask[4] > 15 && ShuffleMask[12] > 15))
10374 Index = DAG.getTargetConstant(Val: IsLE ? 0 : 1, DL, VT: MVT::i32);
10375 else if ((ShuffleMask[4] == 4 && ShuffleMask[12] == 12) &&
10376 (ShuffleMask[0] % 4 == 0 && ShuffleMask[8] % 4 == 0 &&
10377 ShuffleMask[0] > 15 && ShuffleMask[8] > 15))
10378 Index = DAG.getTargetConstant(Val: IsLE ? 1 : 0, DL, VT: MVT::i32);
10379 else
10380 return SDValue();
10381
10382 // If the splat is narrower than 32-bits, we need to get the 32-bit value
10383 // for XXSPLTI32DX.
10384 unsigned SplatVal = APSplatValue.getZExtValue();
10385 for (; SplatBitSize < 32; SplatBitSize <<= 1)
10386 SplatVal |= (SplatVal << SplatBitSize);
10387
10388 SDValue SplatNode = DAG.getNode(
10389 Opcode: PPCISD::XXSPLTI32DX, DL, VT: MVT::v2i64, N1: DAG.getBitcast(VT: MVT::v2i64, V: LHS),
10390 N2: Index, N3: DAG.getTargetConstant(Val: SplatVal, DL, VT: MVT::i32));
10391 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::v16i8, Operand: SplatNode);
10392}
10393
10394/// LowerROTL - Custom lowering for ROTL(v1i128) to vector_shuffle(v16i8).
10395/// We lower ROTL(v1i128) to vector_shuffle(v16i8) only if shift amount is
10396/// a multiple of 8. Otherwise convert it to a scalar rotation(i128)
10397/// i.e (or (shl x, C1), (srl x, 128-C1)).
10398SDValue PPCTargetLowering::LowerROTL(SDValue Op, SelectionDAG &DAG) const {
10399 assert(Op.getOpcode() == ISD::ROTL && "Should only be called for ISD::ROTL");
10400 assert(Op.getValueType() == MVT::v1i128 &&
10401 "Only set v1i128 as custom, other type shouldn't reach here!");
10402 SDLoc dl(Op);
10403 SDValue N0 = peekThroughBitcasts(V: Op.getOperand(i: 0));
10404 SDValue N1 = peekThroughBitcasts(V: Op.getOperand(i: 1));
10405 unsigned SHLAmt = N1.getConstantOperandVal(i: 0);
10406 if (SHLAmt % 8 == 0) {
10407 std::array<int, 16> Mask;
10408 std::iota(first: Mask.begin(), last: Mask.end(), value: 0);
10409 std::rotate(first: Mask.begin(), middle: Mask.begin() + SHLAmt / 8, last: Mask.end());
10410 if (SDValue Shuffle =
10411 DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: DAG.getBitcast(VT: MVT::v16i8, V: N0),
10412 N2: DAG.getUNDEF(VT: MVT::v16i8), Mask))
10413 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: Shuffle);
10414 }
10415 SDValue ArgVal = DAG.getBitcast(VT: MVT::i128, V: N0);
10416 SDValue SHLOp = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ArgVal,
10417 N2: DAG.getConstant(Val: SHLAmt, DL: dl, VT: MVT::i32));
10418 SDValue SRLOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: ArgVal,
10419 N2: DAG.getConstant(Val: 128 - SHLAmt, DL: dl, VT: MVT::i32));
10420 SDValue OROp = DAG.getNode(Opcode: ISD::OR, DL: dl, VT: MVT::i128, N1: SHLOp, N2: SRLOp);
10421 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: OROp);
10422}
10423
10424/// LowerVECTOR_SHUFFLE - Return the code we lower for VECTOR_SHUFFLE. If this
10425/// is a shuffle we can handle in a single instruction, return it. Otherwise,
10426/// return the code it can be lowered into. Worst case, it can always be
10427/// lowered into a vperm.
10428SDValue PPCTargetLowering::LowerVECTOR_SHUFFLE(SDValue Op,
10429 SelectionDAG &DAG) const {
10430 SDLoc dl(Op);
10431 SDValue V1 = Op.getOperand(i: 0);
10432 SDValue V2 = Op.getOperand(i: 1);
10433 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10434
10435 // Any nodes that were combined in the target-independent combiner prior
10436 // to vector legalization will not be sent to the target combine. Try to
10437 // combine it here.
10438 if (SDValue NewShuffle = combineVectorShuffle(SVN: SVOp, DAG)) {
10439 if (!isa<ShuffleVectorSDNode>(Val: NewShuffle))
10440 return NewShuffle;
10441 Op = NewShuffle;
10442 SVOp = cast<ShuffleVectorSDNode>(Val&: Op);
10443 V1 = Op.getOperand(i: 0);
10444 V2 = Op.getOperand(i: 1);
10445 }
10446 EVT VT = Op.getValueType();
10447 bool isLittleEndian = Subtarget.isLittleEndian();
10448
10449 unsigned ShiftElts, InsertAtByte;
10450 bool Swap = false;
10451
10452 // If this is a load-and-splat, we can do that with a single instruction
10453 // in some cases. However if the load has multiple uses, we don't want to
10454 // combine it because that will just produce multiple loads. Since we are
10455 // looking through bitcasts and scalar_to_vector, we must also ensure that
10456 // the intermediate nodes have a single use.
10457 bool IsPermutedLoad = false;
10458 const SDValue *InputLoad = getNormalLoadInput(Op: V1, IsPermuted&: IsPermutedLoad);
10459 if (InputLoad && Subtarget.hasVSX() && V2.isUndef() &&
10460 (PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) || PPC::isSplatShuffleMask(N: SVOp, EltSize: 8))) {
10461 bool IsFourByte = PPC::isSplatShuffleMask(N: SVOp, EltSize: 4);
10462 int SplatIdx =
10463 PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: IsFourByte ? 4 : 8, DAG);
10464
10465 // The splat index for permuted loads will be in the left half of the vector
10466 // which is strictly wider than the loaded value by 8 bytes. So we need to
10467 // adjust the splat index to point to the correct address in memory.
10468 if (IsPermutedLoad) {
10469 assert((isLittleEndian || IsFourByte) &&
10470 "Unexpected size for permuted load on big endian target");
10471 SplatIdx += IsFourByte ? 2 : 1;
10472 assert((SplatIdx < (IsFourByte ? 4 : 2)) &&
10473 "Splat of a value outside of the loaded memory");
10474 }
10475
10476 LoadSDNode *LD = cast<LoadSDNode>(Val: *InputLoad);
10477 // For 4-byte load-and-splat, we need Power9.
10478 if ((IsFourByte && Subtarget.hasP9Vector()) || !IsFourByte) {
10479 uint64_t Offset = 0;
10480 if (IsFourByte)
10481 Offset = isLittleEndian ? (3 - SplatIdx) * 4 : SplatIdx * 4;
10482 else
10483 Offset = isLittleEndian ? (1 - SplatIdx) * 8 : SplatIdx * 8;
10484
10485 // If the width of the load is the same as the width of the splat,
10486 // loading with an offset would load the wrong memory.
10487 if (LD->getValueType(ResNo: 0).getSizeInBits() == (IsFourByte ? 32 : 64))
10488 Offset = 0;
10489
10490 SDValue BasePtr = LD->getBasePtr();
10491 if (Offset != 0)
10492 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout()),
10493 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: Offset, DL: dl));
10494 SDValue Ops[] = {
10495 LD->getChain(), // Chain
10496 BasePtr, // BasePtr
10497 DAG.getValueType(Op.getValueType()) // VT
10498 };
10499 SDVTList VTL =
10500 DAG.getVTList(VT1: IsFourByte ? MVT::v4i32 : MVT::v2i64, VT2: MVT::Other);
10501 SDValue LdSplt =
10502 DAG.getMemIntrinsicNode(Opcode: PPCISD::LD_SPLAT, dl, VTList: VTL,
10503 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
10504 DAG.ReplaceAllUsesOfValueWith(From: InputLoad->getValue(R: 1), To: LdSplt.getValue(R: 1));
10505 if (LdSplt.getValueType() != SVOp->getValueType(ResNo: 0))
10506 LdSplt = DAG.getBitcast(VT: SVOp->getValueType(ResNo: 0), V: LdSplt);
10507 return LdSplt;
10508 }
10509 }
10510
10511 // All v2i64 and v2f64 shuffles are legal
10512 if (VT == MVT::v2i64 || VT == MVT::v2f64)
10513 return Op;
10514
10515 if (Subtarget.hasP9Vector() &&
10516 PPC::isXXINSERTWMask(N: SVOp, ShiftElts, InsertAtByte, Swap,
10517 IsLE: isLittleEndian)) {
10518 if (V2.isUndef())
10519 V2 = V1;
10520 else if (Swap)
10521 std::swap(a&: V1, b&: V2);
10522 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10523 SDValue Conv2 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2);
10524 if (ShiftElts) {
10525 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv2, N2: Conv2,
10526 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10527 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Shl,
10528 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10529 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10530 }
10531 SDValue Ins = DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10532 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
10533 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Ins);
10534 }
10535
10536 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
10537 SDValue SplatInsertNode;
10538 if ((SplatInsertNode = lowerToXXSPLTI32DX(SVN: SVOp, DAG)))
10539 return SplatInsertNode;
10540 }
10541
10542 if (Subtarget.hasP9Altivec()) {
10543 SDValue NewISDNode;
10544 if ((NewISDNode = lowerToVINSERTH(N: SVOp, DAG)))
10545 return NewISDNode;
10546
10547 if ((NewISDNode = lowerToVINSERTB(N: SVOp, DAG)))
10548 return NewISDNode;
10549 }
10550
10551 if (Subtarget.hasVSX() &&
10552 PPC::isXXSLDWIShuffleMask(N: SVOp, ShiftElts, Swap, IsLE: isLittleEndian)) {
10553 if (Swap)
10554 std::swap(a&: V1, b&: V2);
10555 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10556 SDValue Conv2 =
10557 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V2.isUndef() ? V1 : V2);
10558
10559 SDValue Shl = DAG.getNode(Opcode: PPCISD::VECSHL, DL: dl, VT: MVT::v4i32, N1: Conv1, N2: Conv2,
10560 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10561 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Shl);
10562 }
10563
10564 if (Subtarget.hasVSX() &&
10565 PPC::isXXPERMDIShuffleMask(N: SVOp, DM&: ShiftElts, Swap, IsLE: isLittleEndian)) {
10566 if (Swap)
10567 std::swap(a&: V1, b&: V2);
10568 SDValue Conv1 = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10569 SDValue Conv2 =
10570 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V2.isUndef() ? V1 : V2);
10571
10572 SDValue PermDI = DAG.getNode(Opcode: PPCISD::XXPERMDI, DL: dl, VT: MVT::v2i64, N1: Conv1, N2: Conv2,
10573 N3: DAG.getConstant(Val: ShiftElts, DL: dl, VT: MVT::i32));
10574 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: PermDI);
10575 }
10576
10577 if (Subtarget.hasP9Vector()) {
10578 if (PPC::isXXBRHShuffleMask(N: SVOp)) {
10579 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: V1);
10580 SDValue ReveHWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v8i16, Operand: Conv);
10581 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveHWord);
10582 } else if (PPC::isXXBRWShuffleMask(N: SVOp)) {
10583 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10584 SDValue ReveWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v4i32, Operand: Conv);
10585 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveWord);
10586 } else if (PPC::isXXBRDShuffleMask(N: SVOp)) {
10587 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2i64, Operand: V1);
10588 SDValue ReveDWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Conv);
10589 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveDWord);
10590 } else if (PPC::isXXBRQShuffleMask(N: SVOp)) {
10591 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v1i128, Operand: V1);
10592 SDValue ReveQWord = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v1i128, Operand: Conv);
10593 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: ReveQWord);
10594 }
10595 }
10596
10597 if (Subtarget.hasVSX()) {
10598 if (V2.isUndef() && PPC::isSplatShuffleMask(N: SVOp, EltSize: 4)) {
10599 int SplatIdx = PPC::getSplatIdxForPPCMnemonics(N: SVOp, EltSize: 4, DAG);
10600
10601 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: V1);
10602 SDValue Splat = DAG.getNode(Opcode: PPCISD::XXSPLT, DL: dl, VT: MVT::v4i32, N1: Conv,
10603 N2: DAG.getConstant(Val: SplatIdx, DL: dl, VT: MVT::i32));
10604 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Splat);
10605 }
10606
10607 // Left shifts of 8 bytes are actually swaps. Convert accordingly.
10608 if (V2.isUndef() && PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) == 8) {
10609 SDValue Conv = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: V1);
10610 SDValue Swap = DAG.getNode(Opcode: PPCISD::SWAP_NO_CHAIN, DL: dl, VT: MVT::v2f64, Operand: Conv);
10611 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: Swap);
10612 }
10613 }
10614
10615 // Cases that are handled by instructions that take permute immediates
10616 // (such as vsplt*) should be left as VECTOR_SHUFFLE nodes so they can be
10617 // selected by the instruction selector.
10618 if (V2.isUndef()) {
10619 if (PPC::isSplatShuffleMask(N: SVOp, EltSize: 1) ||
10620 PPC::isSplatShuffleMask(N: SVOp, EltSize: 2) ||
10621 PPC::isSplatShuffleMask(N: SVOp, EltSize: 4) ||
10622 PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10623 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10624 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind: 1, DAG) != -1 ||
10625 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10626 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10627 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10628 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind: 1, DAG) ||
10629 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind: 1, DAG) ||
10630 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind: 1, DAG) ||
10631 (Subtarget.hasP8Altivec() && (
10632 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind: 1, DAG) ||
10633 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind: 1, DAG) ||
10634 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind: 1, DAG)))) {
10635 return Op;
10636 }
10637 }
10638
10639 // Altivec has a variety of "shuffle immediates" that take two vector inputs
10640 // and produce a fixed permutation. If any of these match, do not lower to
10641 // VPERM.
10642 unsigned int ShuffleKind = isLittleEndian ? 2 : 0;
10643 if (PPC::isVPKUWUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10644 PPC::isVPKUHUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10645 PPC::isVSLDOIShuffleMask(N: SVOp, ShuffleKind, DAG) != -1 ||
10646 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10647 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10648 PPC::isVMRGLShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10649 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 1, ShuffleKind, DAG) ||
10650 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 2, ShuffleKind, DAG) ||
10651 PPC::isVMRGHShuffleMask(N: SVOp, UnitSize: 4, ShuffleKind, DAG) ||
10652 (Subtarget.hasP8Altivec() && (
10653 PPC::isVPKUDUMShuffleMask(N: SVOp, ShuffleKind, DAG) ||
10654 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: true, ShuffleKind, DAG) ||
10655 PPC::isVMRGEOShuffleMask(N: SVOp, CheckEven: false, ShuffleKind, DAG))))
10656 return Op;
10657
10658 // Check to see if this is a shuffle of 4-byte values. If so, we can use our
10659 // perfect shuffle table to emit an optimal matching sequence.
10660 ArrayRef<int> PermMask = SVOp->getMask();
10661
10662 if (EnablePerfectShuffle && !isLittleEndian) {
10663 unsigned PFIndexes[4];
10664 bool isFourElementShuffle = true;
10665 for (unsigned i = 0; i != 4 && isFourElementShuffle;
10666 ++i) { // Element number
10667 unsigned EltNo = 8; // Start out undef.
10668 for (unsigned j = 0; j != 4; ++j) { // Intra-element byte.
10669 if (PermMask[i * 4 + j] < 0)
10670 continue; // Undef, ignore it.
10671
10672 unsigned ByteSource = PermMask[i * 4 + j];
10673 if ((ByteSource & 3) != j) {
10674 isFourElementShuffle = false;
10675 break;
10676 }
10677
10678 if (EltNo == 8) {
10679 EltNo = ByteSource / 4;
10680 } else if (EltNo != ByteSource / 4) {
10681 isFourElementShuffle = false;
10682 break;
10683 }
10684 }
10685 PFIndexes[i] = EltNo;
10686 }
10687
10688 // If this shuffle can be expressed as a shuffle of 4-byte elements, use the
10689 // perfect shuffle vector to determine if it is cost effective to do this as
10690 // discrete instructions, or whether we should use a vperm.
10691 // For now, we skip this for little endian until such time as we have a
10692 // little-endian perfect shuffle table.
10693 if (isFourElementShuffle) {
10694 // Compute the index in the perfect shuffle table.
10695 unsigned PFTableIndex = PFIndexes[0] * 9 * 9 * 9 + PFIndexes[1] * 9 * 9 +
10696 PFIndexes[2] * 9 + PFIndexes[3];
10697
10698 unsigned PFEntry = PerfectShuffleTable[PFTableIndex];
10699 unsigned Cost = (PFEntry >> 30);
10700
10701 // Determining when to avoid vperm is tricky. Many things affect the cost
10702 // of vperm, particularly how many times the perm mask needs to be
10703 // computed. For example, if the perm mask can be hoisted out of a loop or
10704 // is already used (perhaps because there are multiple permutes with the
10705 // same shuffle mask?) the vperm has a cost of 1. OTOH, hoisting the
10706 // permute mask out of the loop requires an extra register.
10707 //
10708 // As a compromise, we only emit discrete instructions if the shuffle can
10709 // be generated in 3 or fewer operations. When we have loop information
10710 // available, if this block is within a loop, we should avoid using vperm
10711 // for 3-operation perms and use a constant pool load instead.
10712 if (Cost < 3)
10713 return GeneratePerfectShuffle(PFEntry, LHS: V1, RHS: V2, DAG, dl);
10714 }
10715 }
10716
10717 // Lower this to a VPERM(V1, V2, V3) expression, where V3 is a constant
10718 // vector that will get spilled to the constant pool.
10719 if (V2.isUndef()) V2 = V1;
10720
10721 return LowerVPERM(Op, DAG, PermMask, VT, V1, V2);
10722}
10723
10724SDValue PPCTargetLowering::LowerVPERM(SDValue Op, SelectionDAG &DAG,
10725 ArrayRef<int> PermMask, EVT VT,
10726 SDValue V1, SDValue V2) const {
10727 unsigned Opcode = PPCISD::VPERM;
10728 EVT ValType = V1.getValueType();
10729 SDLoc dl(Op);
10730 bool NeedSwap = false;
10731 bool isLittleEndian = Subtarget.isLittleEndian();
10732 bool isPPC64 = Subtarget.isPPC64();
10733
10734 if (Subtarget.hasVSX() && Subtarget.hasP9Vector() &&
10735 (V1->hasOneUse() || V2->hasOneUse())) {
10736 LLVM_DEBUG(dbgs() << "At least one of two input vectors are dead - using "
10737 "XXPERM instead\n");
10738 Opcode = PPCISD::XXPERM;
10739
10740 // The second input to XXPERM is also an output so if the second input has
10741 // multiple uses then copying is necessary, as a result we want the
10742 // single-use operand to be used as the second input to prevent copying.
10743 if ((!isLittleEndian && !V2->hasOneUse() && V1->hasOneUse()) ||
10744 (isLittleEndian && !V1->hasOneUse() && V2->hasOneUse())) {
10745 std::swap(a&: V1, b&: V2);
10746 NeedSwap = !NeedSwap;
10747 }
10748 }
10749
10750 // The SHUFFLE_VECTOR mask is almost exactly what we want for vperm, except
10751 // that it is in input element units, not in bytes. Convert now.
10752
10753 // For little endian, the order of the input vectors is reversed, and
10754 // the permutation mask is complemented with respect to 31. This is
10755 // necessary to produce proper semantics with the big-endian-based vperm
10756 // instruction.
10757 EVT EltVT = V1.getValueType().getVectorElementType();
10758 unsigned BytesPerElement = EltVT.getSizeInBits() / 8;
10759
10760 bool V1HasXXSWAPD = V1->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10761 bool V2HasXXSWAPD = V2->getOperand(Num: 0)->getOpcode() == PPCISD::XXSWAPD;
10762
10763 /*
10764 Vectors will be appended like so: [ V1 | v2 ]
10765 XXSWAPD on V1:
10766 [ A | B | C | D ] -> [ C | D | A | B ]
10767 0-3 4-7 8-11 12-15 0-3 4-7 8-11 12-15
10768 i.e. index of A, B += 8, and index of C, D -= 8.
10769 XXSWAPD on V2:
10770 [ E | F | G | H ] -> [ G | H | E | F ]
10771 16-19 20-23 24-27 28-31 16-19 20-23 24-27 28-31
10772 i.e. index of E, F += 8, index of G, H -= 8
10773 Swap V1 and V2:
10774 [ V1 | V2 ] -> [ V2 | V1 ]
10775 0-15 16-31 0-15 16-31
10776 i.e. index of V1 += 16, index of V2 -= 16
10777 */
10778
10779 SmallVector<SDValue, 16> ResultMask;
10780 for (unsigned i = 0, e = VT.getVectorNumElements(); i != e; ++i) {
10781 unsigned SrcElt = PermMask[i] < 0 ? 0 : PermMask[i];
10782
10783 if (V1HasXXSWAPD) {
10784 if (SrcElt < 8)
10785 SrcElt += 8;
10786 else if (SrcElt < 16)
10787 SrcElt -= 8;
10788 }
10789 if (V2HasXXSWAPD) {
10790 if (SrcElt > 23)
10791 SrcElt -= 8;
10792 else if (SrcElt > 15)
10793 SrcElt += 8;
10794 }
10795 if (NeedSwap) {
10796 if (SrcElt < 16)
10797 SrcElt += 16;
10798 else
10799 SrcElt -= 16;
10800 }
10801 for (unsigned j = 0; j != BytesPerElement; ++j)
10802 if (isLittleEndian)
10803 ResultMask.push_back(
10804 Elt: DAG.getConstant(Val: 31 - (SrcElt * BytesPerElement + j), DL: dl, VT: MVT::i32));
10805 else
10806 ResultMask.push_back(
10807 Elt: DAG.getConstant(Val: SrcElt * BytesPerElement + j, DL: dl, VT: MVT::i32));
10808 }
10809
10810 if (V1HasXXSWAPD) {
10811 dl = SDLoc(V1->getOperand(Num: 0));
10812 V1 = V1->getOperand(Num: 0)->getOperand(Num: 1);
10813 }
10814 if (V2HasXXSWAPD) {
10815 dl = SDLoc(V2->getOperand(Num: 0));
10816 V2 = V2->getOperand(Num: 0)->getOperand(Num: 1);
10817 }
10818
10819 if (isPPC64 && (V1HasXXSWAPD || V2HasXXSWAPD)) {
10820 if (ValType != MVT::v2f64)
10821 V1 = DAG.getBitcast(VT: MVT::v2f64, V: V1);
10822 if (V2.getValueType() != MVT::v2f64)
10823 V2 = DAG.getBitcast(VT: MVT::v2f64, V: V2);
10824 }
10825
10826 ShufflesHandledWithVPERM++;
10827 SDValue VPermMask = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: ResultMask);
10828 LLVM_DEBUG({
10829 ShuffleVectorSDNode *SVOp = cast<ShuffleVectorSDNode>(Op);
10830 if (Opcode == PPCISD::XXPERM) {
10831 dbgs() << "Emitting a XXPERM for the following shuffle:\n";
10832 } else {
10833 dbgs() << "Emitting a VPERM for the following shuffle:\n";
10834 }
10835 SVOp->dump();
10836 dbgs() << "With the following permute control vector:\n";
10837 VPermMask.dump();
10838 });
10839
10840 if (Opcode == PPCISD::XXPERM)
10841 VPermMask = DAG.getBitcast(VT: MVT::v4i32, V: VPermMask);
10842
10843 // Only need to place items backwards in LE,
10844 // the mask was properly calculated.
10845 if (isLittleEndian)
10846 std::swap(a&: V1, b&: V2);
10847
10848 SDValue VPERMNode =
10849 DAG.getNode(Opcode, DL: dl, VT: V1.getValueType(), N1: V1, N2: V2, N3: VPermMask);
10850
10851 VPERMNode = DAG.getBitcast(VT: ValType, V: VPERMNode);
10852 return VPERMNode;
10853}
10854
10855/// getVectorCompareInfo - Given an intrinsic, return false if it is not a
10856/// vector comparison. If it is, return true and fill in Opc/isDot with
10857/// information about the intrinsic.
10858static bool getVectorCompareInfo(SDValue Intrin, int &CompareOpc,
10859 bool &isDot, const PPCSubtarget &Subtarget) {
10860 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 0);
10861 CompareOpc = -1;
10862 isDot = false;
10863 switch (IntrinsicID) {
10864 default:
10865 return false;
10866 // Comparison predicates.
10867 case Intrinsic::ppc_altivec_vcmpbfp_p:
10868 CompareOpc = 966;
10869 isDot = true;
10870 break;
10871 case Intrinsic::ppc_altivec_vcmpeqfp_p:
10872 CompareOpc = 198;
10873 isDot = true;
10874 break;
10875 case Intrinsic::ppc_altivec_vcmpequb_p:
10876 CompareOpc = 6;
10877 isDot = true;
10878 break;
10879 case Intrinsic::ppc_altivec_vcmpequh_p:
10880 CompareOpc = 70;
10881 isDot = true;
10882 break;
10883 case Intrinsic::ppc_altivec_vcmpequw_p:
10884 CompareOpc = 134;
10885 isDot = true;
10886 break;
10887 case Intrinsic::ppc_altivec_vcmpequd_p:
10888 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10889 CompareOpc = 199;
10890 isDot = true;
10891 } else
10892 return false;
10893 break;
10894 case Intrinsic::ppc_altivec_vcmpneb_p:
10895 case Intrinsic::ppc_altivec_vcmpneh_p:
10896 case Intrinsic::ppc_altivec_vcmpnew_p:
10897 case Intrinsic::ppc_altivec_vcmpnezb_p:
10898 case Intrinsic::ppc_altivec_vcmpnezh_p:
10899 case Intrinsic::ppc_altivec_vcmpnezw_p:
10900 if (Subtarget.hasP9Altivec()) {
10901 switch (IntrinsicID) {
10902 default:
10903 llvm_unreachable("Unknown comparison intrinsic.");
10904 case Intrinsic::ppc_altivec_vcmpneb_p:
10905 CompareOpc = 7;
10906 break;
10907 case Intrinsic::ppc_altivec_vcmpneh_p:
10908 CompareOpc = 71;
10909 break;
10910 case Intrinsic::ppc_altivec_vcmpnew_p:
10911 CompareOpc = 135;
10912 break;
10913 case Intrinsic::ppc_altivec_vcmpnezb_p:
10914 CompareOpc = 263;
10915 break;
10916 case Intrinsic::ppc_altivec_vcmpnezh_p:
10917 CompareOpc = 327;
10918 break;
10919 case Intrinsic::ppc_altivec_vcmpnezw_p:
10920 CompareOpc = 391;
10921 break;
10922 }
10923 isDot = true;
10924 } else
10925 return false;
10926 break;
10927 case Intrinsic::ppc_altivec_vcmpgefp_p:
10928 CompareOpc = 454;
10929 isDot = true;
10930 break;
10931 case Intrinsic::ppc_altivec_vcmpgtfp_p:
10932 CompareOpc = 710;
10933 isDot = true;
10934 break;
10935 case Intrinsic::ppc_altivec_vcmpgtsb_p:
10936 CompareOpc = 774;
10937 isDot = true;
10938 break;
10939 case Intrinsic::ppc_altivec_vcmpgtsh_p:
10940 CompareOpc = 838;
10941 isDot = true;
10942 break;
10943 case Intrinsic::ppc_altivec_vcmpgtsw_p:
10944 CompareOpc = 902;
10945 isDot = true;
10946 break;
10947 case Intrinsic::ppc_altivec_vcmpgtsd_p:
10948 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10949 CompareOpc = 967;
10950 isDot = true;
10951 } else
10952 return false;
10953 break;
10954 case Intrinsic::ppc_altivec_vcmpgtub_p:
10955 CompareOpc = 518;
10956 isDot = true;
10957 break;
10958 case Intrinsic::ppc_altivec_vcmpgtuh_p:
10959 CompareOpc = 582;
10960 isDot = true;
10961 break;
10962 case Intrinsic::ppc_altivec_vcmpgtuw_p:
10963 CompareOpc = 646;
10964 isDot = true;
10965 break;
10966 case Intrinsic::ppc_altivec_vcmpgtud_p:
10967 if (Subtarget.hasVSX() || Subtarget.hasP8Altivec()) {
10968 CompareOpc = 711;
10969 isDot = true;
10970 } else
10971 return false;
10972 break;
10973
10974 case Intrinsic::ppc_altivec_vcmpequq:
10975 case Intrinsic::ppc_altivec_vcmpgtsq:
10976 case Intrinsic::ppc_altivec_vcmpgtuq:
10977 if (!Subtarget.isISA3_1())
10978 return false;
10979 switch (IntrinsicID) {
10980 default:
10981 llvm_unreachable("Unknown comparison intrinsic.");
10982 case Intrinsic::ppc_altivec_vcmpequq:
10983 CompareOpc = 455;
10984 break;
10985 case Intrinsic::ppc_altivec_vcmpgtsq:
10986 CompareOpc = 903;
10987 break;
10988 case Intrinsic::ppc_altivec_vcmpgtuq:
10989 CompareOpc = 647;
10990 break;
10991 }
10992 break;
10993
10994 // VSX predicate comparisons use the same infrastructure
10995 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
10996 case Intrinsic::ppc_vsx_xvcmpgedp_p:
10997 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
10998 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
10999 case Intrinsic::ppc_vsx_xvcmpgesp_p:
11000 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
11001 if (Subtarget.hasVSX()) {
11002 switch (IntrinsicID) {
11003 case Intrinsic::ppc_vsx_xvcmpeqdp_p:
11004 CompareOpc = 99;
11005 break;
11006 case Intrinsic::ppc_vsx_xvcmpgedp_p:
11007 CompareOpc = 115;
11008 break;
11009 case Intrinsic::ppc_vsx_xvcmpgtdp_p:
11010 CompareOpc = 107;
11011 break;
11012 case Intrinsic::ppc_vsx_xvcmpeqsp_p:
11013 CompareOpc = 67;
11014 break;
11015 case Intrinsic::ppc_vsx_xvcmpgesp_p:
11016 CompareOpc = 83;
11017 break;
11018 case Intrinsic::ppc_vsx_xvcmpgtsp_p:
11019 CompareOpc = 75;
11020 break;
11021 }
11022 isDot = true;
11023 } else
11024 return false;
11025 break;
11026
11027 // Normal Comparisons.
11028 case Intrinsic::ppc_altivec_vcmpbfp:
11029 CompareOpc = 966;
11030 break;
11031 case Intrinsic::ppc_altivec_vcmpeqfp:
11032 CompareOpc = 198;
11033 break;
11034 case Intrinsic::ppc_altivec_vcmpequb:
11035 CompareOpc = 6;
11036 break;
11037 case Intrinsic::ppc_altivec_vcmpequh:
11038 CompareOpc = 70;
11039 break;
11040 case Intrinsic::ppc_altivec_vcmpequw:
11041 CompareOpc = 134;
11042 break;
11043 case Intrinsic::ppc_altivec_vcmpequd:
11044 if (Subtarget.hasP8Altivec())
11045 CompareOpc = 199;
11046 else
11047 return false;
11048 break;
11049 case Intrinsic::ppc_altivec_vcmpneb:
11050 case Intrinsic::ppc_altivec_vcmpneh:
11051 case Intrinsic::ppc_altivec_vcmpnew:
11052 case Intrinsic::ppc_altivec_vcmpnezb:
11053 case Intrinsic::ppc_altivec_vcmpnezh:
11054 case Intrinsic::ppc_altivec_vcmpnezw:
11055 if (Subtarget.hasP9Altivec())
11056 switch (IntrinsicID) {
11057 default:
11058 llvm_unreachable("Unknown comparison intrinsic.");
11059 case Intrinsic::ppc_altivec_vcmpneb:
11060 CompareOpc = 7;
11061 break;
11062 case Intrinsic::ppc_altivec_vcmpneh:
11063 CompareOpc = 71;
11064 break;
11065 case Intrinsic::ppc_altivec_vcmpnew:
11066 CompareOpc = 135;
11067 break;
11068 case Intrinsic::ppc_altivec_vcmpnezb:
11069 CompareOpc = 263;
11070 break;
11071 case Intrinsic::ppc_altivec_vcmpnezh:
11072 CompareOpc = 327;
11073 break;
11074 case Intrinsic::ppc_altivec_vcmpnezw:
11075 CompareOpc = 391;
11076 break;
11077 }
11078 else
11079 return false;
11080 break;
11081 case Intrinsic::ppc_altivec_vcmpgefp:
11082 CompareOpc = 454;
11083 break;
11084 case Intrinsic::ppc_altivec_vcmpgtfp:
11085 CompareOpc = 710;
11086 break;
11087 case Intrinsic::ppc_altivec_vcmpgtsb:
11088 CompareOpc = 774;
11089 break;
11090 case Intrinsic::ppc_altivec_vcmpgtsh:
11091 CompareOpc = 838;
11092 break;
11093 case Intrinsic::ppc_altivec_vcmpgtsw:
11094 CompareOpc = 902;
11095 break;
11096 case Intrinsic::ppc_altivec_vcmpgtsd:
11097 if (Subtarget.hasP8Altivec())
11098 CompareOpc = 967;
11099 else
11100 return false;
11101 break;
11102 case Intrinsic::ppc_altivec_vcmpgtub:
11103 CompareOpc = 518;
11104 break;
11105 case Intrinsic::ppc_altivec_vcmpgtuh:
11106 CompareOpc = 582;
11107 break;
11108 case Intrinsic::ppc_altivec_vcmpgtuw:
11109 CompareOpc = 646;
11110 break;
11111 case Intrinsic::ppc_altivec_vcmpgtud:
11112 if (Subtarget.hasP8Altivec())
11113 CompareOpc = 711;
11114 else
11115 return false;
11116 break;
11117 case Intrinsic::ppc_altivec_vcmpequq_p:
11118 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11119 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11120 if (!Subtarget.isISA3_1())
11121 return false;
11122 switch (IntrinsicID) {
11123 default:
11124 llvm_unreachable("Unknown comparison intrinsic.");
11125 case Intrinsic::ppc_altivec_vcmpequq_p:
11126 CompareOpc = 455;
11127 break;
11128 case Intrinsic::ppc_altivec_vcmpgtsq_p:
11129 CompareOpc = 903;
11130 break;
11131 case Intrinsic::ppc_altivec_vcmpgtuq_p:
11132 CompareOpc = 647;
11133 break;
11134 }
11135 isDot = true;
11136 break;
11137 }
11138 return true;
11139}
11140
11141/// LowerINTRINSIC_WO_CHAIN - If this is an intrinsic that we want to custom
11142/// lower, do it, otherwise return null.
11143SDValue PPCTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11144 SelectionDAG &DAG) const {
11145 unsigned IntrinsicID = Op.getConstantOperandVal(i: 0);
11146
11147 SDLoc dl(Op);
11148 // Note: BCD instructions expect the immediate operand in vector form (v4i32),
11149 // but the builtin provides it as a scalar. To satisfy the instruction
11150 // encoding, we splat the scalar across all lanes using SPLAT_VECTOR.
11151 auto MapNodeWithSplatVector =
11152 [&](unsigned Opcode,
11153 std::initializer_list<SDValue> ExtraOps = {}) -> SDValue {
11154 SDValue SplatVal =
11155 DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL: dl, VT: MVT::v4i32, Operand: Op.getOperand(i: 2));
11156
11157 SmallVector<SDValue, 4> Ops{SplatVal, Op.getOperand(i: 1)};
11158 Ops.append(in_start: ExtraOps.begin(), in_end: ExtraOps.end());
11159 return DAG.getNode(Opcode, DL: dl, VT: MVT::v16i8, Ops);
11160 };
11161
11162 switch (IntrinsicID) {
11163 case Intrinsic::thread_pointer:
11164 // Reads the thread pointer register, used for __builtin_thread_pointer.
11165 if (Subtarget.isPPC64())
11166 return DAG.getRegister(Reg: PPC::X13, VT: MVT::i64);
11167 return DAG.getRegister(Reg: PPC::R2, VT: MVT::i32);
11168
11169 case Intrinsic::ppc_rldimi: {
11170 assert(Subtarget.isPPC64() && "rldimi is only available in 64-bit!");
11171 SDValue Src = Op.getOperand(i: 1);
11172 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11173 if (Mask.isZero())
11174 return Op.getOperand(i: 2);
11175 if (Mask.isAllOnes())
11176 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src, N2: Op.getOperand(i: 3));
11177 uint64_t SH = Op.getConstantOperandVal(i: 3);
11178 unsigned MB = 0, ME = 0;
11179 if (!isRunOfOnes64(Val: Mask.getZExtValue(), MB, ME))
11180 report_fatal_error(reason: "invalid rldimi mask!");
11181 // rldimi requires ME=63-SH, otherwise rotation is needed before rldimi.
11182 if (ME < 63 - SH) {
11183 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11184 N2: DAG.getConstant(Val: ME + SH + 1, DL: dl, VT: MVT::i32));
11185 } else if (ME > 63 - SH) {
11186 Src = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i64, N1: Src,
11187 N2: DAG.getConstant(Val: ME + SH - 63, DL: dl, VT: MVT::i32));
11188 }
11189 return SDValue(
11190 DAG.getMachineNode(Opcode: PPC::RLDIMI, dl, VT: MVT::i64,
11191 Ops: {Op.getOperand(i: 2), Src,
11192 DAG.getTargetConstant(Val: 63 - ME, DL: dl, VT: MVT::i32),
11193 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32)}),
11194 0);
11195 }
11196
11197 case Intrinsic::ppc_rlwimi: {
11198 APInt Mask = Op.getConstantOperandAPInt(i: 4);
11199 if (Mask.isZero())
11200 return Op.getOperand(i: 2);
11201 if (Mask.isAllOnes())
11202 return DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Op.getOperand(i: 1),
11203 N2: Op.getOperand(i: 3));
11204 unsigned MB = 0, ME = 0;
11205 if (!isRunOfOnes(Val: Mask.getZExtValue(), MB, ME))
11206 report_fatal_error(reason: "invalid rlwimi mask!");
11207 return SDValue(DAG.getMachineNode(
11208 Opcode: PPC::RLWIMI, dl, VT: MVT::i32,
11209 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1), Op.getOperand(i: 3),
11210 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11211 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11212 0);
11213 }
11214
11215 case Intrinsic::ppc_bcdshift:
11216 return MapNodeWithSplatVector(PPCISD::BCDSHIFT, {Op.getOperand(i: 3)});
11217 case Intrinsic::ppc_bcdshiftround:
11218 return MapNodeWithSplatVector(PPCISD::BCDSHIFTROUND, {Op.getOperand(i: 3)});
11219 case Intrinsic::ppc_bcdtruncate:
11220 return MapNodeWithSplatVector(PPCISD::BCDTRUNC, {Op.getOperand(i: 3)});
11221 case Intrinsic::ppc_bcdunsignedtruncate:
11222 return MapNodeWithSplatVector(PPCISD::BCDUTRUNC);
11223 case Intrinsic::ppc_bcdunsignedshift:
11224 return MapNodeWithSplatVector(PPCISD::BCDUSHIFT);
11225
11226 case Intrinsic::ppc_rlwnm: {
11227 if (Op.getConstantOperandVal(i: 3) == 0)
11228 return DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32);
11229 unsigned MB = 0, ME = 0;
11230 if (!isRunOfOnes(Val: Op.getConstantOperandVal(i: 3), MB, ME))
11231 report_fatal_error(reason: "invalid rlwnm mask!");
11232 return SDValue(
11233 DAG.getMachineNode(Opcode: PPC::RLWNM, dl, VT: MVT::i32,
11234 Ops: {Op.getOperand(i: 1), Op.getOperand(i: 2),
11235 DAG.getTargetConstant(Val: MB, DL: dl, VT: MVT::i32),
11236 DAG.getTargetConstant(Val: ME, DL: dl, VT: MVT::i32)}),
11237 0);
11238 }
11239
11240 case Intrinsic::ppc_mma_disassemble_acc: {
11241 if (Subtarget.isISAFuture()) {
11242 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11243 SDValue WideVec =
11244 SDValue(DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes,
11245 Ops: Op.getOperand(i: 1)),
11246 0);
11247 SmallVector<SDValue, 4> RetOps;
11248 SDValue Value = SDValue(WideVec.getNode(), 0);
11249 SDValue Value2 = SDValue(WideVec.getNode(), 1);
11250
11251 SDValue Extract;
11252 Extract = DAG.getNode(
11253 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11254 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11255 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11256 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11257 RetOps.push_back(Elt: Extract);
11258 Extract = DAG.getNode(
11259 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11260 N1: Subtarget.isLittleEndian() ? Value2 : Value,
11261 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11262 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11263 RetOps.push_back(Elt: Extract);
11264 Extract = DAG.getNode(
11265 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11266 N1: Subtarget.isLittleEndian() ? Value : Value2,
11267 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 1 : 0,
11268 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11269 RetOps.push_back(Elt: Extract);
11270 Extract = DAG.getNode(
11271 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
11272 N1: Subtarget.isLittleEndian() ? Value : Value2,
11273 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? 0 : 1,
11274 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11275 RetOps.push_back(Elt: Extract);
11276 return DAG.getMergeValues(Ops: RetOps, dl);
11277 }
11278 [[fallthrough]];
11279 }
11280 case Intrinsic::ppc_vsx_disassemble_pair: {
11281 int NumVecs = 2;
11282 SDValue WideVec = Op.getOperand(i: 1);
11283 if (IntrinsicID == Intrinsic::ppc_mma_disassemble_acc) {
11284 NumVecs = 4;
11285 WideVec = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: WideVec);
11286 }
11287 SmallVector<SDValue, 4> RetOps;
11288 for (int VecNo = 0; VecNo < NumVecs; VecNo++) {
11289 SDValue Extract = DAG.getNode(
11290 Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: WideVec,
11291 N2: DAG.getConstant(Val: Subtarget.isLittleEndian() ? NumVecs - 1 - VecNo
11292 : VecNo,
11293 DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
11294 RetOps.push_back(Elt: Extract);
11295 }
11296 return DAG.getMergeValues(Ops: RetOps, dl);
11297 }
11298
11299 case Intrinsic::ppc_build_dmr: {
11300 SmallVector<SDValue, 8> Pairs;
11301 SmallVector<SDValue, 8> Chains;
11302 for (int i = 1; i < 9; i += 2) {
11303 SDValue Hi = Op.getOperand(i);
11304 SDValue Lo = Op.getOperand(i: i + 1);
11305 if (Hi->getOpcode() == ISD::LOAD)
11306 Chains.push_back(Elt: Hi.getValue(R: 1));
11307 if (Lo->getOpcode() == ISD::LOAD)
11308 Chains.push_back(Elt: Lo.getValue(R: 1));
11309 Pairs.push_back(
11310 Elt: DAG.getNode(Opcode: PPCISD::PAIR_BUILD, DL: dl, VT: MVT::v256i1, Ops: {Hi, Lo}));
11311 }
11312 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: Chains);
11313 SDValue Value = DMFInsert1024(Pairs, dl: SDLoc(Op), DAG);
11314 return DAG.getMergeValues(Ops: {Value, TF}, dl);
11315 }
11316
11317 case Intrinsic::ppc_mma_dmxxextfdmr512: {
11318 assert(Subtarget.isISAFuture() && "dmxxextfdmr512 requires ISA Future");
11319 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11320 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11321 "Specify P of 0 or 1 for lower or upper 512 bytes");
11322 unsigned HiLo = Idx->getSExtValue();
11323 unsigned Opcode;
11324 unsigned Subx;
11325 if (HiLo == 0) {
11326 Opcode = PPC::DMXXEXTFDMR512;
11327 Subx = PPC::sub_wacc_lo;
11328 } else {
11329 Opcode = PPC::DMXXEXTFDMR512_HI;
11330 Subx = PPC::sub_wacc_hi;
11331 }
11332 SDValue Subreg(
11333 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
11334 Op1: Op.getOperand(i: 1),
11335 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11336 0);
11337 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
11338 return SDValue(DAG.getMachineNode(Opcode, dl, ResultTys: ReturnTypes, Ops: Subreg), 0);
11339 }
11340
11341 case Intrinsic::ppc_mma_dmxxextfdmr256: {
11342 assert(Subtarget.isISAFuture() && "dmxxextfdmr256 requires ISA Future");
11343 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11344 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11345 "Specify a dmr row pair 0-3");
11346 unsigned IdxVal = Idx->getSExtValue();
11347 unsigned Subx;
11348 switch (IdxVal) {
11349 case 0:
11350 Subx = PPC::sub_dmrrowp0;
11351 break;
11352 case 1:
11353 Subx = PPC::sub_dmrrowp1;
11354 break;
11355 case 2:
11356 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11357 break;
11358 case 3:
11359 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11360 break;
11361 }
11362 SDValue Subreg(
11363 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v256i1,
11364 Op1: Op.getOperand(i: 1),
11365 Op2: DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32)),
11366 0);
11367 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11368 return SDValue(
11369 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR256, dl, VT: MVT::v256i1, Ops: {Subreg, P}),
11370 0);
11371 }
11372
11373 case Intrinsic::ppc_mma_dmxxinstdmr512: {
11374 assert(Subtarget.isISAFuture() && "dmxxinstdmr512 requires ISA Future");
11375 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 4));
11376 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11377 "Specify P of 0 or 1 for lower or upper 512 bytes");
11378 unsigned HiLo = Idx->getSExtValue();
11379 unsigned Opcode;
11380 unsigned Subx;
11381 if (HiLo == 0) {
11382 Opcode = PPCISD::INST512;
11383 Subx = PPC::sub_wacc_lo;
11384 } else {
11385 Opcode = PPCISD::INST512HI;
11386 Subx = PPC::sub_wacc_hi;
11387 }
11388 SDValue Wacc = DAG.getNode(Opcode, DL: dl, VT: MVT::v512i1, N1: Op.getOperand(i: 2),
11389 N2: Op.getOperand(i: 3));
11390 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11391 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11392 Op1: Op.getOperand(i: 1), Op2: Wacc, Op3: SubReg),
11393 0);
11394 }
11395
11396 case Intrinsic::ppc_mma_dmxxinstdmr256: {
11397 assert(Subtarget.isISAFuture() && "dmxxinstdmr256 requires ISA Future");
11398 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 3));
11399 assert(Idx && (Idx->getSExtValue() >= 0 || Idx->getSExtValue() <= 3) &&
11400 "Specify a dmr row pair 0-3");
11401 unsigned IdxVal = Idx->getSExtValue();
11402 unsigned Subx;
11403 switch (IdxVal) {
11404 case 0:
11405 Subx = PPC::sub_dmrrowp0;
11406 break;
11407 case 1:
11408 Subx = PPC::sub_dmrrowp1;
11409 break;
11410 case 2:
11411 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp0;
11412 break;
11413 case 3:
11414 Subx = PPC::sub_wacc_hi_then_sub_dmrrowp1;
11415 break;
11416 }
11417 SDValue SubReg = DAG.getTargetConstant(Val: Subx, DL: dl, VT: MVT::i32);
11418 SDValue P = DAG.getTargetConstant(Val: IdxVal, DL: dl, VT: MVT::i32);
11419 SDValue DMRRowp =
11420 DAG.getNode(Opcode: PPCISD::INST256, DL: dl, VT: MVT::v256i1, N1: Op.getOperand(i: 2), N2: P);
11421 return SDValue(DAG.getMachineNode(Opcode: PPC::INSERT_SUBREG, dl, VT: MVT::v1024i1,
11422 Op1: Op.getOperand(i: 1), Op2: DMRRowp, Op3: SubReg),
11423 0);
11424 }
11425
11426 case Intrinsic::ppc_mma_xxmfacc:
11427 case Intrinsic::ppc_mma_xxmtacc: {
11428 // Allow pre-isa-future subtargets to lower as normal.
11429 if (!Subtarget.isISAFuture())
11430 return SDValue();
11431 // The intrinsics for xxmtacc and xxmfacc take one argument of
11432 // type v512i1, for future cpu the corresponding wacc instruction
11433 // dmxx[inst|extf]dmr512 is always generated for type v512i1, negating
11434 // the need to produce the xxm[t|f]acc.
11435 SDValue WideVec = Op.getOperand(i: 1);
11436 DAG.ReplaceAllUsesWith(From: Op, To: WideVec);
11437 return SDValue();
11438 }
11439
11440 case Intrinsic::ppc_unpack_longdouble: {
11441 auto *Idx = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
11442 assert(Idx && (Idx->getSExtValue() == 0 || Idx->getSExtValue() == 1) &&
11443 "Argument of long double unpack must be 0 or 1!");
11444 return DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: dl, VT: MVT::f64, N1: Op.getOperand(i: 1),
11445 N2: DAG.getConstant(Val: !!(Idx->getSExtValue()), DL: dl,
11446 VT: Idx->getValueType(ResNo: 0)));
11447 }
11448
11449 case Intrinsic::ppc_compare_exp_lt:
11450 case Intrinsic::ppc_compare_exp_gt:
11451 case Intrinsic::ppc_compare_exp_eq:
11452 case Intrinsic::ppc_compare_exp_uo: {
11453 unsigned Pred;
11454 switch (IntrinsicID) {
11455 case Intrinsic::ppc_compare_exp_lt:
11456 Pred = PPC::PRED_LT;
11457 break;
11458 case Intrinsic::ppc_compare_exp_gt:
11459 Pred = PPC::PRED_GT;
11460 break;
11461 case Intrinsic::ppc_compare_exp_eq:
11462 Pred = PPC::PRED_EQ;
11463 break;
11464 case Intrinsic::ppc_compare_exp_uo:
11465 Pred = PPC::PRED_UN;
11466 break;
11467 }
11468 return SDValue(
11469 DAG.getMachineNode(
11470 Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11471 Ops: {SDValue(DAG.getMachineNode(Opcode: PPC::XSCMPEXPDP, dl, VT: MVT::i32,
11472 Op1: Op.getOperand(i: 1), Op2: Op.getOperand(i: 2)),
11473 0),
11474 DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32), DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11475 DAG.getTargetConstant(Val: Pred, DL: dl, VT: MVT::i32)}),
11476 0);
11477 }
11478 case Intrinsic::ppc_test_data_class: {
11479 EVT OpVT = Op.getOperand(i: 1).getValueType();
11480 unsigned CmprOpc = OpVT == MVT::f128 ? PPC::XSTSTDCQP
11481 : (OpVT == MVT::f64 ? PPC::XSTSTDCDP
11482 : PPC::XSTSTDCSP);
11483 // Lower __builtin_ppc_test_data_class(value, mask) to XSTSTDC* instruction.
11484 // The XSTSTDC* instructions test if a floating-point value matches any of
11485 // the data classes specified in the mask, setting CR field bits
11486 // accordingly. We need to extract the EQ bit (bit 2) from the CR field and
11487 // convert it to an integer result (1 if match, 0 if no match).
11488 //
11489 // Note: Operands are swapped because XSTSTDC* expects (mask, value) but the
11490 // intrinsic provides (value, mask) as Op.getOperand(1) and
11491 // Op.getOperand(2).
11492 SDValue TestDataClass =
11493 SDValue(DAG.getMachineNode(Opcode: CmprOpc, dl, VT: MVT::i32,
11494 Ops: {Op.getOperand(i: 2), Op.getOperand(i: 1)}),
11495 0);
11496 if (Subtarget.isISA3_1()) {
11497 // ISA 3.1+: Use SETBC instruction to directly convert CR bit to integer.
11498 // This is more efficient than the SELECT_CC approach used in earlier
11499 // ISAs.
11500 SDValue SubRegIdx = DAG.getTargetConstant(Val: PPC::sub_eq, DL: dl, VT: MVT::i32);
11501 SDValue CRBit =
11502 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11503 Op1: TestDataClass, Op2: SubRegIdx),
11504 0);
11505
11506 return DAG.getNode(Opcode: PPCISD::SETBC, DL: dl, VT: MVT::i32, Operand: CRBit);
11507 }
11508
11509 // Pre-ISA 3.1: Use SELECT_CC to convert CR field to integer (1 or 0).
11510 return SDValue(
11511 DAG.getMachineNode(Opcode: PPC::SELECT_CC_I4, dl, VT: MVT::i32,
11512 Ops: {TestDataClass, DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32),
11513 DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32),
11514 DAG.getTargetConstant(Val: PPC::PRED_EQ, DL: dl, VT: MVT::i32)}),
11515 0);
11516 }
11517 case Intrinsic::ppc_fnmsub: {
11518 EVT VT = Op.getOperand(i: 1).getValueType();
11519 if (!Subtarget.hasVSX() || (!Subtarget.hasFloat128() && VT == MVT::f128))
11520 return DAG.getNode(
11521 Opcode: ISD::FNEG, DL: dl, VT,
11522 Operand: DAG.getNode(Opcode: ISD::FMA, DL: dl, VT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11523 N3: DAG.getNode(Opcode: ISD::FNEG, DL: dl, VT, Operand: Op.getOperand(i: 3))));
11524 return DAG.getNode(Opcode: PPCISD::FNMSUB, DL: dl, VT, N1: Op.getOperand(i: 1),
11525 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11526 }
11527 case Intrinsic::ppc_convert_f128_to_ppcf128:
11528 case Intrinsic::ppc_convert_ppcf128_to_f128: {
11529 RTLIB::Libcall LC = IntrinsicID == Intrinsic::ppc_convert_ppcf128_to_f128
11530 ? RTLIB::CONVERT_PPCF128_F128
11531 : RTLIB::CONVERT_F128_PPCF128;
11532 MakeLibCallOptions CallOptions;
11533 std::pair<SDValue, SDValue> Result =
11534 makeLibCall(DAG, LC, RetVT: Op.getValueType(), Ops: Op.getOperand(i: 1), CallOptions,
11535 dl, Chain: SDValue());
11536 return Result.first;
11537 }
11538 case Intrinsic::ppc_maxfe:
11539 case Intrinsic::ppc_maxfl:
11540 case Intrinsic::ppc_maxfs:
11541 case Intrinsic::ppc_minfe:
11542 case Intrinsic::ppc_minfl:
11543 case Intrinsic::ppc_minfs: {
11544 EVT VT = Op.getValueType();
11545 assert(
11546 all_of(Op->ops().drop_front(4),
11547 [VT](const SDUse &Use) { return Use.getValueType() == VT; }) &&
11548 "ppc_[max|min]f[e|l|s] must have uniform type arguments");
11549 (void)VT;
11550 ISD::CondCode CC = ISD::SETGT;
11551 if (IntrinsicID == Intrinsic::ppc_minfe ||
11552 IntrinsicID == Intrinsic::ppc_minfl ||
11553 IntrinsicID == Intrinsic::ppc_minfs)
11554 CC = ISD::SETLT;
11555 unsigned I = Op.getNumOperands() - 2, Cnt = I;
11556 SDValue Res = Op.getOperand(i: I);
11557 for (--I; Cnt != 0; --Cnt, I = (--I == 0 ? (Op.getNumOperands() - 1) : I)) {
11558 Res =
11559 DAG.getSelectCC(DL: dl, LHS: Res, RHS: Op.getOperand(i: I), True: Res, False: Op.getOperand(i: I), Cond: CC);
11560 }
11561 return Res;
11562 }
11563 }
11564
11565 // If this is a lowered altivec predicate compare, CompareOpc is set to the
11566 // opcode number of the comparison.
11567 int CompareOpc;
11568 bool isDot;
11569 if (!getVectorCompareInfo(Intrin: Op, CompareOpc, isDot, Subtarget))
11570 return SDValue(); // Don't custom lower most intrinsics.
11571
11572 // If this is a non-dot comparison, make the VCMP node and we are done.
11573 if (!isDot) {
11574 SDValue Tmp = DAG.getNode(Opcode: PPCISD::VCMP, DL: dl, VT: Op.getOperand(i: 2).getValueType(),
11575 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11576 N3: DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32));
11577 return DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: Op.getValueType(), Operand: Tmp);
11578 }
11579
11580 // Create the PPCISD altivec 'dot' comparison node.
11581 SDValue Ops[] = {
11582 Op.getOperand(i: 2), // LHS
11583 Op.getOperand(i: 3), // RHS
11584 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
11585 };
11586 EVT VTs[] = { Op.getOperand(i: 2).getValueType(), MVT::Glue };
11587 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
11588
11589 // Unpack the result based on how the target uses it.
11590 unsigned BitNo; // Bit # of CR6.
11591 bool InvertBit; // Invert result?
11592 unsigned Bitx;
11593 unsigned SetOp;
11594 switch (Op.getConstantOperandVal(i: 1)) {
11595 default: // Can't happen, don't crash on invalid number though.
11596 case 0: // Return the value of the EQ bit of CR6.
11597 BitNo = 0;
11598 InvertBit = false;
11599 Bitx = PPC::sub_eq;
11600 SetOp = PPCISD::SETBC;
11601 break;
11602 case 1: // Return the inverted value of the EQ bit of CR6.
11603 BitNo = 0;
11604 InvertBit = true;
11605 Bitx = PPC::sub_eq;
11606 SetOp = PPCISD::SETBCR;
11607 break;
11608 case 2: // Return the value of the LT bit of CR6.
11609 BitNo = 2;
11610 InvertBit = false;
11611 Bitx = PPC::sub_lt;
11612 SetOp = PPCISD::SETBC;
11613 break;
11614 case 3: // Return the inverted value of the LT bit of CR6.
11615 BitNo = 2;
11616 InvertBit = true;
11617 Bitx = PPC::sub_lt;
11618 SetOp = PPCISD::SETBCR;
11619 break;
11620 }
11621
11622 SDValue GlueOp = CompNode.getValue(R: 1);
11623 if (Subtarget.isISA3_1()) {
11624 SDValue SubRegIdx = DAG.getTargetConstant(Val: Bitx, DL: dl, VT: MVT::i32);
11625 SDValue CR6Reg = DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32);
11626 SDValue CRBit =
11627 SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::i1,
11628 Op1: CR6Reg, Op2: SubRegIdx, Op3: GlueOp),
11629 0);
11630 return DAG.getNode(Opcode: SetOp, DL: dl, VT: MVT::i32, Operand: CRBit);
11631 }
11632
11633 // Now that we have the comparison, emit a copy from the CR to a GPR.
11634 // This is flagged to the above dot comparison.
11635 SDValue Flags = DAG.getNode(Opcode: PPCISD::MFOCRF, DL: dl, VT: MVT::i32,
11636 N1: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32), N2: GlueOp);
11637
11638 // Shift the bit into the low position.
11639 Flags = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i32, N1: Flags,
11640 N2: DAG.getConstant(Val: 8 - (3 - BitNo), DL: dl, VT: MVT::i32));
11641 // Isolate the bit.
11642 Flags = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: Flags,
11643 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11644
11645 // If we are supposed to, toggle the bit.
11646 if (InvertBit)
11647 Flags = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT: MVT::i32, N1: Flags,
11648 N2: DAG.getConstant(Val: 1, DL: dl, VT: MVT::i32));
11649 return Flags;
11650}
11651
11652SDValue PPCTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
11653 SelectionDAG &DAG) const {
11654 // SelectionDAGBuilder::visitTargetIntrinsic may insert one extra chain to
11655 // the beginning of the argument list.
11656 int ArgStart = isa<ConstantSDNode>(Val: Op.getOperand(i: 0)) ? 0 : 1;
11657 SDLoc DL(Op);
11658 switch (Op.getConstantOperandVal(i: ArgStart)) {
11659 case Intrinsic::ppc_cfence: {
11660 assert(ArgStart == 1 && "llvm.ppc.cfence must carry a chain argument.");
11661 SDValue Val = Op.getOperand(i: ArgStart + 1);
11662 EVT Ty = Val.getValueType();
11663 if (Ty == MVT::i128) {
11664 // FIXME: Testing one of two paired registers is sufficient to guarantee
11665 // ordering?
11666 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i64, Operand: Val);
11667 }
11668 unsigned Opcode = Subtarget.isPPC64() ? PPC::CFENCE8 : PPC::CFENCE;
11669 return SDValue(
11670 DAG.getMachineNode(
11671 Opcode, dl: DL, VT: MVT::Other,
11672 Op1: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getScalarIntVT(), Operand: Val),
11673 Op2: Op.getOperand(i: 0)),
11674 0);
11675 }
11676 case Intrinsic::ppc_disassemble_dmr: {
11677 assert(ArgStart == 1 &&
11678 "llvm.ppc.disassemble.dmr must carry a chain argument.");
11679 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: Op.getOperand(i: ArgStart + 2),
11680 Ptr: Op.getOperand(i: ArgStart + 1), PtrInfo: MachinePointerInfo());
11681 }
11682 default:
11683 break;
11684 }
11685 return SDValue();
11686}
11687
11688// Lower scalar BSWAP64 to xxbrd.
11689SDValue PPCTargetLowering::LowerBSWAP(SDValue Op, SelectionDAG &DAG) const {
11690 SDLoc dl(Op);
11691 if (!Subtarget.isPPC64())
11692 return Op;
11693
11694 if (Subtarget.hasP9Vector()) {
11695 // MTVSRDD
11696 Op = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL: dl, VT: MVT::v2i64, N1: Op.getOperand(i: 0),
11697 N2: Op.getOperand(i: 0));
11698 // XXBRD
11699 Op = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::v2i64, Operand: Op);
11700 // MFVSRD
11701 int VectorIndex = 0;
11702 if (Subtarget.isLittleEndian())
11703 VectorIndex = 1;
11704 Op = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Op,
11705 N2: DAG.getTargetConstant(Val: VectorIndex, DL: dl, VT: MVT::i32));
11706 return Op;
11707 }
11708
11709 // For Power8, use parallel rotate instructions for faster bswap64.
11710 SDValue Input = Op.getOperand(i: 0);
11711 // Helper to create rotate-and-insert operations (RLWIMI/RLDIMI).
11712 auto CreateRotateInsert =
11713 [&](unsigned Opcode, MVT VT, SDValue Dest, SDValue Src, unsigned RotAmt,
11714 unsigned MaskBegin,
11715 std::optional<unsigned> MaskEnd = std::nullopt) -> SDValue {
11716 SmallVector<SDValue, 5> Ops = {
11717 Dest, Src, DAG.getTargetConstant(Val: RotAmt, DL: dl, VT: MVT::i32),
11718 DAG.getTargetConstant(Val: MaskBegin, DL: dl, VT: MVT::i32)};
11719 if (MaskEnd.has_value())
11720 Ops.push_back(Elt: DAG.getTargetConstant(Val: *MaskEnd, DL: dl, VT: MVT::i32));
11721
11722 return SDValue(DAG.getMachineNode(Opcode, dl, VT, Ops), 0);
11723 };
11724
11725 // Helper to perform 32-bit byte swap using rotl(8) + 2x rlwimi.
11726 auto Swap32 = [&](SDValue Val32) -> SDValue {
11727 SDValue Rot = DAG.getNode(Opcode: ISD::ROTL, DL: dl, VT: MVT::i32, N1: Val32,
11728 N2: DAG.getConstant(Val: 8, DL: dl, VT: MVT::i32));
11729 // Insert bits [24:31] from Val32 into Rot at position [0:7].
11730 SDValue Swap =
11731 CreateRotateInsert(PPC::RLWIMI, MVT::i32, Rot, Val32, 24, 0, 7);
11732 // Insert bits [16:23] from Val32 into Swap at position [16:23].
11733 return CreateRotateInsert(PPC::RLWIMI, MVT::i32, Swap, Val32, 24, 16, 23);
11734 };
11735 // Extract and swap high and low 32-bit halves independently for parallelism.
11736 SDValue Hi32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32,
11737 Operand: DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i64, N1: Input,
11738 N2: DAG.getConstant(Val: 32, DL: dl, VT: MVT::i64)));
11739 SDValue Lo32 = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Input);
11740
11741 // Combine swapped halves: rotate LoSwap left by 32 bits and insert into
11742 // HiSwap to swap their positions, completing the 64-bit byte reversal.
11743 SDValue HiSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Hi32));
11744 SDValue LoSwap = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i64, Operand: Swap32(Lo32));
11745
11746 return CreateRotateInsert(PPC::RLDIMI, MVT::i64, HiSwap, LoSwap, 32, 0);
11747}
11748
11749SDValue PPCTargetLowering::LowerATOMIC_LOAD_STORE(SDValue Op,
11750 SelectionDAG &DAG) const {
11751 AtomicSDNode *N = cast<AtomicSDNode>(Val: Op.getNode());
11752 EVT MemVT = N->getMemoryVT();
11753 assert(MemVT.getSimpleVT() == MVT::i128 &&
11754 "Expect quadword atomic operations");
11755 SDLoc dl(N);
11756 unsigned Opc = N->getOpcode();
11757 switch (Opc) {
11758 case ISD::ATOMIC_LOAD: {
11759 // Lower quadword atomic load to int_ppc_atomic_load_i128 which will be
11760 // lowered to ppc instructions by pattern matching instruction selector.
11761 SDVTList Tys = DAG.getVTList(VT1: MVT::i64, VT2: MVT::i64, VT3: MVT::Other);
11762 SmallVector<SDValue, 4> Ops{
11763 N->getOperand(Num: 0),
11764 DAG.getConstant(Val: Intrinsic::ppc_atomic_load_i128, DL: dl, VT: MVT::i32)};
11765 for (int I = 1, E = N->getNumOperands(); I < E; ++I)
11766 Ops.push_back(Elt: N->getOperand(Num: I));
11767 SDValue LoadedVal = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys,
11768 Ops, MemVT, MMO: N->getMemOperand());
11769 SDValue ValLo = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal);
11770 SDValue ValHi =
11771 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: MVT::i128, Operand: LoadedVal.getValue(R: 1));
11772 ValHi = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i128, N1: ValHi,
11773 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11774 SDValue Val =
11775 DAG.getNode(Opcode: ISD::OR, DL: dl, ResultTys: {MVT::i128, MVT::Other}, Ops: {ValLo, ValHi});
11776 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, ResultTys: {MVT::i128, MVT::Other},
11777 Ops: {Val, LoadedVal.getValue(R: 2)});
11778 }
11779 case ISD::ATOMIC_STORE: {
11780 // Lower quadword atomic store to int_ppc_atomic_store_i128 which will be
11781 // lowered to ppc instructions by pattern matching instruction selector.
11782 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
11783 SmallVector<SDValue, 4> Ops{
11784 N->getOperand(Num: 0),
11785 DAG.getConstant(Val: Intrinsic::ppc_atomic_store_i128, DL: dl, VT: MVT::i32)};
11786 SDValue Val = N->getOperand(Num: 1);
11787 SDValue ValLo = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: Val);
11788 SDValue ValHi = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: MVT::i128, N1: Val,
11789 N2: DAG.getConstant(Val: 64, DL: dl, VT: MVT::i32));
11790 ValHi = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i64, Operand: ValHi);
11791 Ops.push_back(Elt: ValLo);
11792 Ops.push_back(Elt: ValHi);
11793 Ops.push_back(Elt: N->getOperand(Num: 2));
11794 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops, MemVT,
11795 MMO: N->getMemOperand());
11796 }
11797 default:
11798 llvm_unreachable("Unexpected atomic opcode");
11799 }
11800}
11801
11802static SDValue getDataClassTest(SDValue Op, FPClassTest Mask, const SDLoc &Dl,
11803 SelectionDAG &DAG,
11804 const PPCSubtarget &Subtarget) {
11805 assert(Mask <= fcAllFlags && "Invalid fp_class flags!");
11806
11807 enum DataClassMask {
11808 DC_NAN = 1 << 6,
11809 DC_NEG_INF = 1 << 4,
11810 DC_POS_INF = 1 << 5,
11811 DC_NEG_ZERO = 1 << 2,
11812 DC_POS_ZERO = 1 << 3,
11813 DC_NEG_SUBNORM = 1,
11814 DC_POS_SUBNORM = 1 << 1,
11815 };
11816
11817 EVT VT = Op.getValueType();
11818
11819 unsigned TestOp = VT == MVT::f128 ? PPC::XSTSTDCQP
11820 : VT == MVT::f64 ? PPC::XSTSTDCDP
11821 : PPC::XSTSTDCSP;
11822
11823 if (Mask == fcAllFlags)
11824 return DAG.getBoolConstant(V: true, DL: Dl, VT: MVT::i1, OpVT: VT);
11825 if (Mask == 0)
11826 return DAG.getBoolConstant(V: false, DL: Dl, VT: MVT::i1, OpVT: VT);
11827
11828 // When it's cheaper or necessary to test reverse flags.
11829 if ((Mask & fcNormal) == fcNormal || Mask == ~fcQNan || Mask == ~fcSNan) {
11830 SDValue Rev = getDataClassTest(Op, Mask: ~Mask, Dl, DAG, Subtarget);
11831 return DAG.getNOT(DL: Dl, Val: Rev, VT: MVT::i1);
11832 }
11833
11834 // Power doesn't support testing whether a value is 'normal'. Test the rest
11835 // first, and test if it's 'not not-normal' with expected sign.
11836 if (Mask & fcNormal) {
11837 SDValue Rev(DAG.getMachineNode(
11838 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11839 Op1: DAG.getTargetConstant(Val: DC_NAN | DC_NEG_INF | DC_POS_INF |
11840 DC_NEG_ZERO | DC_POS_ZERO |
11841 DC_NEG_SUBNORM | DC_POS_SUBNORM,
11842 DL: Dl, VT: MVT::i32),
11843 Op2: Op),
11844 0);
11845 // Sign are stored in CR bit 0, result are in CR bit 2.
11846 SDValue Sign(
11847 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11848 Op2: DAG.getTargetConstant(Val: PPC::sub_lt, DL: Dl, VT: MVT::i32)),
11849 0);
11850 SDValue Normal(DAG.getNOT(
11851 DL: Dl,
11852 Val: SDValue(DAG.getMachineNode(
11853 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1, Op1: Rev,
11854 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11855 0),
11856 VT: MVT::i1));
11857 if (Mask & fcPosNormal)
11858 Sign = DAG.getNOT(DL: Dl, Val: Sign, VT: MVT::i1);
11859 SDValue Result = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: Sign, N2: Normal);
11860 if (Mask == fcPosNormal || Mask == fcNegNormal)
11861 return Result;
11862
11863 return DAG.getNode(
11864 Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11865 N1: getDataClassTest(Op, Mask: Mask & ~fcNormal, Dl, DAG, Subtarget), N2: Result);
11866 }
11867
11868 // The instruction doesn't differentiate between signaling or quiet NaN. Test
11869 // the rest first, and test if it 'is NaN and is signaling/quiet'.
11870 if ((Mask & fcNan) == fcQNan || (Mask & fcNan) == fcSNan) {
11871 bool IsQuiet = Mask & fcQNan;
11872 SDValue NanCheck = getDataClassTest(Op, Mask: fcNan, Dl, DAG, Subtarget);
11873
11874 // Quietness is determined by the first bit in fraction field.
11875 uint64_t QuietMask = 0;
11876 SDValue HighWord;
11877 if (VT == MVT::f128) {
11878 HighWord = DAG.getNode(
11879 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: DAG.getBitcast(VT: MVT::v4i32, V: Op),
11880 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 3 : 0, DL: Dl));
11881 QuietMask = 0x8000;
11882 } else if (VT == MVT::f64) {
11883 if (Subtarget.isPPC64()) {
11884 HighWord = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::i32,
11885 N1: DAG.getBitcast(VT: MVT::i64, V: Op),
11886 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11887 } else {
11888 SDValue Vec = DAG.getBitcast(
11889 VT: MVT::v4i32, V: DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL: Dl, VT: MVT::v2f64, Operand: Op));
11890 HighWord = DAG.getNode(
11891 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: Dl, VT: MVT::i32, N1: Vec,
11892 N2: DAG.getVectorIdxConstant(Val: Subtarget.isLittleEndian() ? 1 : 0, DL: Dl));
11893 }
11894 QuietMask = 0x80000;
11895 } else if (VT == MVT::f32) {
11896 HighWord = DAG.getBitcast(VT: MVT::i32, V: Op);
11897 QuietMask = 0x400000;
11898 }
11899 SDValue NanRes = DAG.getSetCC(
11900 DL: Dl, VT: MVT::i1,
11901 LHS: DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i32, N1: HighWord,
11902 N2: DAG.getConstant(Val: QuietMask, DL: Dl, VT: MVT::i32)),
11903 RHS: DAG.getConstant(Val: 0, DL: Dl, VT: MVT::i32), Cond: IsQuiet ? ISD::SETNE : ISD::SETEQ);
11904 NanRes = DAG.getNode(Opcode: ISD::AND, DL: Dl, VT: MVT::i1, N1: NanCheck, N2: NanRes);
11905 if (Mask == fcQNan || Mask == fcSNan)
11906 return NanRes;
11907
11908 return DAG.getNode(Opcode: ISD::OR, DL: Dl, VT: MVT::i1,
11909 N1: getDataClassTest(Op, Mask: Mask & ~fcNan, Dl, DAG, Subtarget),
11910 N2: NanRes);
11911 }
11912
11913 unsigned NativeMask = 0;
11914 if ((Mask & fcNan) == fcNan)
11915 NativeMask |= DC_NAN;
11916 if (Mask & fcNegInf)
11917 NativeMask |= DC_NEG_INF;
11918 if (Mask & fcPosInf)
11919 NativeMask |= DC_POS_INF;
11920 if (Mask & fcNegZero)
11921 NativeMask |= DC_NEG_ZERO;
11922 if (Mask & fcPosZero)
11923 NativeMask |= DC_POS_ZERO;
11924 if (Mask & fcNegSubnormal)
11925 NativeMask |= DC_NEG_SUBNORM;
11926 if (Mask & fcPosSubnormal)
11927 NativeMask |= DC_POS_SUBNORM;
11928 return SDValue(
11929 DAG.getMachineNode(
11930 Opcode: TargetOpcode::EXTRACT_SUBREG, dl: Dl, VT: MVT::i1,
11931 Op1: SDValue(DAG.getMachineNode(
11932 Opcode: TestOp, dl: Dl, VT: MVT::i32,
11933 Op1: DAG.getTargetConstant(Val: NativeMask, DL: Dl, VT: MVT::i32), Op2: Op),
11934 0),
11935 Op2: DAG.getTargetConstant(Val: PPC::sub_eq, DL: Dl, VT: MVT::i32)),
11936 0);
11937}
11938
11939SDValue PPCTargetLowering::LowerIS_FPCLASS(SDValue Op,
11940 SelectionDAG &DAG) const {
11941 assert(Subtarget.hasP9Vector() && "Test data class requires Power9");
11942 SDValue LHS = Op.getOperand(i: 0);
11943 uint64_t RHSC = Op.getConstantOperandVal(i: 1);
11944 SDLoc Dl(Op);
11945 FPClassTest Category = static_cast<FPClassTest>(RHSC);
11946 if (LHS.getValueType() == MVT::ppcf128) {
11947 // The higher part determines the value class.
11948 LHS = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL: Dl, VT: MVT::f64, N1: LHS,
11949 N2: DAG.getConstant(Val: 1, DL: Dl, VT: MVT::i32));
11950 }
11951
11952 return getDataClassTest(Op: LHS, Mask: Category, Dl, DAG, Subtarget);
11953}
11954
11955// Adjust the length value for a load/store with length to account for the
11956// instructions requiring a left justified length, and for non-byte element
11957// types requiring scaling by element size.
11958static SDValue AdjustLength(SDValue Val, unsigned Bits, bool Left,
11959 SelectionDAG &DAG) {
11960 SDLoc dl(Val);
11961 EVT VT = Val->getValueType(ResNo: 0);
11962 unsigned LeftAdj = Left ? VT.getSizeInBits() - 8 : 0;
11963 unsigned TypeAdj = llvm::countr_zero<uint32_t>(Val: Bits / 8);
11964 SDValue SHLAmt = DAG.getConstant(Val: LeftAdj + TypeAdj, DL: dl, VT);
11965 return DAG.getNode(Opcode: ISD::SHL, DL: dl, VT, N1: Val, N2: SHLAmt);
11966}
11967
11968SDValue PPCTargetLowering::LowerVP_LOAD(SDValue Op, SelectionDAG &DAG) const {
11969 auto VPLD = cast<VPLoadSDNode>(Val&: Op);
11970 bool Future = Subtarget.isISAFuture();
11971 SDLoc dl(Op);
11972 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(3), true) &&
11973 "Mask predication not supported");
11974 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
11975 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPLD->getOperand(Num: 4));
11976 unsigned IID = Future ? Intrinsic::ppc_vsx_lxvrl : Intrinsic::ppc_vsx_lxvl;
11977 unsigned EltBits = Op->getValueType(ResNo: 0).getScalarType().getSizeInBits();
11978 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
11979 SDValue Ops[] = {VPLD->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
11980 VPLD->getOperand(Num: 1), Len};
11981 SDVTList Tys = DAG.getVTList(VT1: Op->getValueType(ResNo: 0), VT2: MVT::Other);
11982 SDValue VPL =
11983 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl, VTList: Tys, Ops,
11984 MemVT: VPLD->getMemoryVT(), MMO: VPLD->getMemOperand());
11985 return VPL;
11986}
11987
11988SDValue PPCTargetLowering::LowerVP_STORE(SDValue Op, SelectionDAG &DAG) const {
11989 auto VPST = cast<VPStoreSDNode>(Val&: Op);
11990 assert(ISD::isConstantSplatVectorAllOnes(Op->getOperand(4), true) &&
11991 "Mask predication not supported");
11992 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
11993 SDLoc dl(Op);
11994 SDValue Len = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: PtrVT, Operand: VPST->getOperand(Num: 5));
11995 unsigned EltBits =
11996 Op->getOperand(Num: 1).getValueType().getScalarType().getSizeInBits();
11997 bool Future = Subtarget.isISAFuture();
11998 unsigned IID = Future ? Intrinsic::ppc_vsx_stxvrl : Intrinsic::ppc_vsx_stxvl;
11999 Len = AdjustLength(Val: Len, Bits: EltBits, Left: !Future, DAG);
12000 SDValue Ops[] = {
12001 VPST->getChain(), DAG.getConstant(Val: IID, DL: dl, VT: MVT::i32),
12002 DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v4i32, Operand: VPST->getOperand(Num: 1)),
12003 VPST->getOperand(Num: 2), Len};
12004 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12005 SDValue VPS =
12006 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12007 MemVT: VPST->getMemoryVT(), MMO: VPST->getMemOperand());
12008 return VPS;
12009}
12010
12011SDValue PPCTargetLowering::LowerPartialReduce(SDValue Op,
12012 SelectionDAG &DAG) const {
12013 SDValue Acc = Op.getOperand(i: 0);
12014 SDValue Op1 = Op.getOperand(i: 1);
12015 SDValue Op2 = Op.getOperand(i: 2);
12016
12017 assert(Op.getOpcode() == ISD::PARTIAL_REDUCE_UMLA &&
12018 "Unexpected partial reduction");
12019
12020 if (Acc.getValueType() != MVT::v4i32)
12021 return SDValue();
12022 if (Op1.getValueType() != MVT::v16i32 || Op1.getOpcode() != ISD::SIGN_EXTEND)
12023 return SDValue();
12024 SDValue Op1Input = Op1.getOperand(i: 0);
12025 if (Op1Input.getValueType() != MVT::v16i8 || !llvm::isOneOrOneSplat(V: Op2))
12026 return SDValue();
12027
12028 SDLoc dl(Op);
12029 SDValue Ones = DAG.getConstant(Val: 1, DL: dl, VT: MVT::v16i8);
12030 return DAG.getNode(Opcode: ISD::PARTIAL_REDUCE_SUMLA, DL: dl, VT: MVT::v4i32, N1: Acc, N2: Op1Input,
12031 N3: Ones);
12032}
12033
12034SDValue PPCTargetLowering::LowerSCALAR_TO_VECTOR(SDValue Op,
12035 SelectionDAG &DAG) const {
12036 SDLoc dl(Op);
12037
12038 MachineFunction &MF = DAG.getMachineFunction();
12039 SDValue Op0 = Op.getOperand(i: 0);
12040 EVT ValVT = Op0.getValueType();
12041 unsigned EltSize = Op.getValueType().getScalarSizeInBits();
12042 if (isa<ConstantSDNode>(Val: Op0) && EltSize <= 32) {
12043 int64_t IntVal = Op.getConstantOperandVal(i: 0);
12044 if (IntVal >= -16 && IntVal <= 15)
12045 return getCanonicalConstSplat(Val: IntVal, SplatSize: EltSize / 8, VT: Op.getValueType(), DAG,
12046 dl);
12047 }
12048
12049 ReuseLoadInfo RLI;
12050 if (Subtarget.hasLFIWAX() && Subtarget.hasVSX() &&
12051 Op.getValueType() == MVT::v4i32 && Op0.getOpcode() == ISD::LOAD &&
12052 Op0.getValueType() == MVT::i32 && Op0.hasOneUse() &&
12053 canReuseLoadAddress(Op: Op0, MemVT: MVT::i32, RLI, DAG, ET: ISD::NON_EXTLOAD)) {
12054
12055 MachineMemOperand *MMO = MF.getMachineMemOperand(
12056 PtrInfo: RLI.MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: RLI.Alignment,
12057 Metadata: MMOMetadata(RLI.AAInfo, RLI.Ranges));
12058 SDValue Ops[] = {RLI.Chain, RLI.Ptr, DAG.getValueType(Op.getValueType())};
12059 SDValue Bits = DAG.getMemIntrinsicNode(
12060 Opcode: PPCISD::LD_SPLAT, dl, VTList: DAG.getVTList(VT1: MVT::v4i32, VT2: MVT::Other), Ops,
12061 MemVT: MVT::i32, MMO);
12062 if (RLI.ResChain)
12063 DAG.makeEquivalentMemoryOrdering(OldChain: RLI.ResChain, NewMemOpChain: Bits.getValue(R: 1));
12064 return Bits.getValue(R: 0);
12065 }
12066
12067 // Create a stack slot that is 16-byte aligned.
12068 MachineFrameInfo &MFI = MF.getFrameInfo();
12069 int FrameIdx = MFI.CreateStackObject(Size: 16, Alignment: Align(16), isSpillSlot: false);
12070 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12071 SDValue FIdx = DAG.getFrameIndex(FI: FrameIdx, VT: PtrVT);
12072
12073 SDValue Val = Op0;
12074 // P10 hardware store forwarding requires that a single store contains all
12075 // the data for the load. P10 is able to merge a pair of adjacent stores. Try
12076 // to avoid load hit store on P10 when running binaries compiled for older
12077 // processors by generating two mergeable scalar stores to forward with the
12078 // vector load.
12079 if (EnableP10StoreForward && Subtarget.isPPC64() &&
12080 !Subtarget.isLittleEndian() && ValVT.isInteger() &&
12081 ValVT.getSizeInBits() <= 64) {
12082 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i64, Operand: Val);
12083 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: MVT::i64, DL: DAG.getDataLayout());
12084 SDValue ShiftBy = DAG.getConstant(
12085 Val: 64 - Op.getValueType().getScalarSizeInBits(), DL: dl, VT: ShiftAmountTy);
12086 Val = DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: MVT::i64, N1: Val, N2: ShiftBy);
12087 SDValue Plus8 =
12088 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FIdx, N2: DAG.getConstant(Val: 8, DL: dl, VT: PtrVT));
12089 SDValue Store2 =
12090 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: Plus8, PtrInfo: MachinePointerInfo());
12091 SDValue Store = DAG.getStore(Chain: Store2, dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12092 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx,
12093 PtrInfo: MachinePointerInfo());
12094 }
12095
12096 // Store the input value into Value#0 of the stack slot.
12097 SDValue Store =
12098 DAG.getStore(Chain: DAG.getEntryNode(), dl, Val, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12099 // Load it out.
12100 return DAG.getLoad(VT: Op.getValueType(), dl, Chain: Store, Ptr: FIdx, PtrInfo: MachinePointerInfo());
12101}
12102
12103SDValue PPCTargetLowering::LowerINSERT_VECTOR_ELT(SDValue Op,
12104 SelectionDAG &DAG) const {
12105 assert(Op.getOpcode() == ISD::INSERT_VECTOR_ELT &&
12106 "Should only be called for ISD::INSERT_VECTOR_ELT");
12107
12108 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 2));
12109
12110 EVT VT = Op.getValueType();
12111 SDLoc dl(Op);
12112 SDValue V1 = Op.getOperand(i: 0);
12113 SDValue V2 = Op.getOperand(i: 1);
12114
12115 if (VT == MVT::v2f64 && C)
12116 return Op;
12117
12118 if (Subtarget.hasP9Vector()) {
12119 // A f32 load feeding into a v4f32 insert_vector_elt is handled in this way
12120 // because on P10, it allows this specific insert_vector_elt load pattern to
12121 // utilize the refactored load and store infrastructure in order to exploit
12122 // prefixed loads.
12123 // On targets with inexpensive direct moves (Power9 and up), a
12124 // (insert_vector_elt v4f32:$vec, (f32 load)) is always better as an integer
12125 // load since a single precision load will involve conversion to double
12126 // precision on the load followed by another conversion to single precision.
12127 if ((VT == MVT::v4f32) && (V2.getValueType() == MVT::f32) &&
12128 (isa<LoadSDNode>(Val: V2))) {
12129 SDValue BitcastVector = DAG.getBitcast(VT: MVT::v4i32, V: V1);
12130 SDValue BitcastLoad = DAG.getBitcast(VT: MVT::i32, V: V2);
12131 SDValue InsVecElt =
12132 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL: dl, VT: MVT::v4i32, N1: BitcastVector,
12133 N2: BitcastLoad, N3: Op.getOperand(i: 2));
12134 return DAG.getBitcast(VT: MVT::v4f32, V: InsVecElt);
12135 }
12136 }
12137
12138 if (Subtarget.isISA3_1()) {
12139 if ((VT == MVT::v2i64 || VT == MVT::v2f64) && !Subtarget.isPPC64())
12140 return SDValue();
12141 // On P10, we have legal lowering for constant and variable indices for
12142 // all vectors.
12143 if (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
12144 VT == MVT::v2i64 || VT == MVT::v4f32 || VT == MVT::v2f64)
12145 return Op;
12146 }
12147
12148 // Before P10, we have legal lowering for constant indices but not for
12149 // variable ones.
12150 if (!C)
12151 return SDValue();
12152
12153 // We can use MTVSRZ + VECINSERT for v8i16 and v16i8 types.
12154 if (VT == MVT::v8i16 || VT == MVT::v16i8) {
12155 SDValue Mtvsrz = DAG.getNode(Opcode: PPCISD::MTVSRZ, DL: dl, VT, Operand: V2);
12156 unsigned BytesInEachElement = VT.getVectorElementType().getSizeInBits() / 8;
12157 unsigned InsertAtElement = C->getZExtValue();
12158 unsigned InsertAtByte = InsertAtElement * BytesInEachElement;
12159 if (Subtarget.isLittleEndian()) {
12160 InsertAtByte = (16 - BytesInEachElement) - InsertAtByte;
12161 }
12162 return DAG.getNode(Opcode: PPCISD::VECINSERT, DL: dl, VT, N1: V1, N2: Mtvsrz,
12163 N3: DAG.getConstant(Val: InsertAtByte, DL: dl, VT: MVT::i32));
12164 }
12165 return Op;
12166}
12167
12168SDValue PPCTargetLowering::LowerDMFVectorLoad(SDValue Op,
12169 SelectionDAG &DAG) const {
12170 SDLoc dl(Op);
12171 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12172 SDValue LoadChain = LN->getChain();
12173 SDValue BasePtr = LN->getBasePtr();
12174 EVT VT = Op.getValueType();
12175 bool IsV1024i1 = VT == MVT::v1024i1;
12176 bool IsV2048i1 = VT == MVT::v2048i1;
12177
12178 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12179 // Dense Math dmr pair registers, respectively.
12180 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12181 (void)IsV2048i1;
12182 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12183 "Dense Math support required.");
12184 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12185
12186 SmallVector<SDValue, 8> Loads;
12187 SmallVector<SDValue, 8> LoadChains;
12188
12189 SDValue IntrinID = DAG.getConstant(Val: Intrinsic::ppc_vsx_lxvp, DL: dl, VT: MVT::i32);
12190 SDValue LoadOps[] = {LoadChain, IntrinID, BasePtr};
12191 MachineMemOperand *MMO = LN->getMemOperand();
12192 unsigned NumVecs = VT.getSizeInBits() / 256;
12193 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12194 MachineMemOperand *NewMMO =
12195 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12196 if (Idx > 0) {
12197 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12198 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12199 LoadOps[2] = BasePtr;
12200 }
12201 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
12202 VTList: DAG.getVTList(VT1: MVT::v256i1, VT2: MVT::Other),
12203 Ops: LoadOps, MemVT: MVT::v256i1, MMO: NewMMO);
12204 LoadChains.push_back(Elt: Ld.getValue(R: 1));
12205 Loads.push_back(Elt: Ld);
12206 }
12207
12208 if (Subtarget.isLittleEndian()) {
12209 std::reverse(first: Loads.begin(), last: Loads.end());
12210 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12211 }
12212
12213 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12214 SDValue Value = DMFInsert1024(Pairs: Loads, dl, DAG);
12215
12216 if (IsV1024i1) {
12217 return DAG.getMergeValues(Ops: {Value, TF}, dl);
12218 }
12219
12220 // Handle Loads for V2048i1 which represents a dmr pair.
12221 SmallVector<SDValue, 4> MoreLoads{Loads[4], Loads[5], Loads[6], Loads[7]};
12222 SDValue Dmr1Value = DMFInsert1024(Pairs: MoreLoads, dl, DAG);
12223
12224 SDValue Dmr0Sub = DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32);
12225 SDValue Dmr1Sub = DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32);
12226
12227 SDValue DmrPRC = DAG.getTargetConstant(Val: PPC::DMRpRCRegClassID, DL: dl, VT: MVT::i32);
12228 const SDValue DmrPOps[] = {DmrPRC, Value, Dmr0Sub, Dmr1Value, Dmr1Sub};
12229
12230 SDValue DmrPValue = SDValue(
12231 DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v2048i1, Ops: DmrPOps), 0);
12232
12233 return DAG.getMergeValues(Ops: {DmrPValue, TF}, dl);
12234}
12235
12236SDValue PPCTargetLowering::DMFInsert1024(const SmallVectorImpl<SDValue> &Pairs,
12237 const SDLoc &dl,
12238 SelectionDAG &DAG) const {
12239 SDValue Lo =
12240 DAG.getNode(Opcode: PPCISD::INST512, DL: dl, VT: MVT::v512i1, N1: Pairs[0], N2: Pairs[1]);
12241 SDValue LoSub = DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32);
12242 SDValue Hi =
12243 DAG.getNode(Opcode: PPCISD::INST512HI, DL: dl, VT: MVT::v512i1, N1: Pairs[2], N2: Pairs[3]);
12244 SDValue HiSub = DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32);
12245 SDValue RC = DAG.getTargetConstant(Val: PPC::DMRRCRegClassID, DL: dl, VT: MVT::i32);
12246
12247 return SDValue(DAG.getMachineNode(Opcode: PPC::REG_SEQUENCE, dl, VT: MVT::v1024i1,
12248 Ops: {RC, Lo, LoSub, Hi, HiSub}),
12249 0);
12250}
12251
12252SDValue PPCTargetLowering::LowerVectorLoad(SDValue Op,
12253 SelectionDAG &DAG) const {
12254 SDLoc dl(Op);
12255 LoadSDNode *LN = cast<LoadSDNode>(Val: Op.getNode());
12256 SDValue LoadChain = LN->getChain();
12257 SDValue BasePtr = LN->getBasePtr();
12258 EVT VT = Op.getValueType();
12259
12260 if (VT == MVT::v1024i1 || VT == MVT::v2048i1)
12261 return LowerDMFVectorLoad(Op, DAG);
12262
12263 if (VT != MVT::v256i1 && VT != MVT::v512i1)
12264 return Op;
12265
12266 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12267 assert((VT != MVT::v512i1 || Subtarget.hasMMA()) &&
12268 "Type unsupported without MMA");
12269 assert((VT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12270 "Type unsupported without paired vector support");
12271
12272 // For v256i1 on ISA Future, let the load go through to instruction selection
12273 // where it will be matched to lxvp/plxvp by the instruction patterns.
12274 if (VT == MVT::v256i1 && Subtarget.isISAFuture())
12275 return Op;
12276
12277 // For other cases, create 2 or 4 v16i8 loads to load the pair or accumulator
12278 // value in 2 or 4 vsx registers.
12279 Align Alignment = LN->getAlign();
12280 SmallVector<SDValue, 4> Loads;
12281 SmallVector<SDValue, 4> LoadChains;
12282 unsigned NumVecs = VT.getSizeInBits() / 128;
12283 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12284 SDValue Load =
12285 DAG.getLoad(VT: MVT::v16i8, dl, Chain: LoadChain, Ptr: BasePtr,
12286 PtrInfo: LN->getPointerInfo().getWithOffset(O: Idx * 16),
12287 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12288 MMOFlags: LN->getMemOperand()->getFlags(), Metadata: LN->getAAInfo());
12289 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12290 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12291 Loads.push_back(Elt: Load);
12292 LoadChains.push_back(Elt: Load.getValue(R: 1));
12293 }
12294 if (Subtarget.isLittleEndian()) {
12295 std::reverse(first: Loads.begin(), last: Loads.end());
12296 std::reverse(first: LoadChains.begin(), last: LoadChains.end());
12297 }
12298 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: LoadChains);
12299 SDValue Value =
12300 DAG.getNode(Opcode: VT == MVT::v512i1 ? PPCISD::ACC_BUILD : PPCISD::PAIR_BUILD,
12301 DL: dl, VT, Ops: Loads);
12302 SDValue RetOps[] = {Value, TF};
12303 return DAG.getMergeValues(Ops: RetOps, dl);
12304}
12305
12306SDValue PPCTargetLowering::LowerDMFVectorStore(SDValue Op,
12307 SelectionDAG &DAG) const {
12308
12309 SDLoc dl(Op);
12310 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12311 SDValue StoreChain = SN->getChain();
12312 SDValue BasePtr = SN->getBasePtr();
12313 SmallVector<SDValue, 8> Values;
12314 SmallVector<SDValue, 8> Stores;
12315 EVT VT = SN->getValue().getValueType();
12316 bool IsV1024i1 = VT == MVT::v1024i1;
12317 bool IsV2048i1 = VT == MVT::v2048i1;
12318
12319 // The types v1024i1 and v2048i1 are used for Dense Math dmr registers and
12320 // Dense Math dmr pair registers, respectively.
12321 assert((IsV1024i1 || IsV2048i1) && "Unsupported type.");
12322 (void)IsV2048i1;
12323 assert((Subtarget.hasMMA() && Subtarget.isISAFuture()) &&
12324 "Dense Math support required.");
12325 assert(Subtarget.pairedVectorMemops() && "Vector pair support required.");
12326
12327 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12328 if (IsV1024i1) {
12329 SDValue Lo(DAG.getMachineNode(
12330 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12331 Op1: Op.getOperand(i: 1),
12332 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12333 0);
12334 SDValue Hi(DAG.getMachineNode(
12335 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1,
12336 Op1: Op.getOperand(i: 1),
12337 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12338 0);
12339 MachineSDNode *ExtNode =
12340 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Lo);
12341 Values.push_back(Elt: SDValue(ExtNode, 0));
12342 Values.push_back(Elt: SDValue(ExtNode, 1));
12343 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Hi);
12344 Values.push_back(Elt: SDValue(ExtNode, 0));
12345 Values.push_back(Elt: SDValue(ExtNode, 1));
12346 } else {
12347 // This corresponds to v2048i1 which represents a dmr pair.
12348 SDValue Dmr0(
12349 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12350 Op1: Op.getOperand(i: 1),
12351 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr0, DL: dl, VT: MVT::i32)),
12352 0);
12353
12354 SDValue Dmr1(
12355 DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v1024i1,
12356 Op1: Op.getOperand(i: 1),
12357 Op2: DAG.getTargetConstant(Val: PPC::sub_dmr1, DL: dl, VT: MVT::i32)),
12358 0);
12359
12360 SDValue Dmr0Lo(DAG.getMachineNode(
12361 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12362 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12363 0);
12364
12365 SDValue Dmr0Hi(DAG.getMachineNode(
12366 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr0,
12367 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12368 0);
12369
12370 SDValue Dmr1Lo(DAG.getMachineNode(
12371 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12372 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_lo, DL: dl, VT: MVT::i32)),
12373 0);
12374
12375 SDValue Dmr1Hi(DAG.getMachineNode(
12376 Opcode: TargetOpcode::EXTRACT_SUBREG, dl, VT: MVT::v512i1, Op1: Dmr1,
12377 Op2: DAG.getTargetConstant(Val: PPC::sub_wacc_hi, DL: dl, VT: MVT::i32)),
12378 0);
12379
12380 MachineSDNode *ExtNode =
12381 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr0Lo);
12382 Values.push_back(Elt: SDValue(ExtNode, 0));
12383 Values.push_back(Elt: SDValue(ExtNode, 1));
12384 ExtNode =
12385 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr0Hi);
12386 Values.push_back(Elt: SDValue(ExtNode, 0));
12387 Values.push_back(Elt: SDValue(ExtNode, 1));
12388 ExtNode = DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Dmr1Lo);
12389 Values.push_back(Elt: SDValue(ExtNode, 0));
12390 Values.push_back(Elt: SDValue(ExtNode, 1));
12391 ExtNode =
12392 DAG.getMachineNode(Opcode: PPC::DMXXEXTFDMR512_HI, dl, ResultTys: ReturnTypes, Ops: Dmr1Hi);
12393 Values.push_back(Elt: SDValue(ExtNode, 0));
12394 Values.push_back(Elt: SDValue(ExtNode, 1));
12395 }
12396
12397 if (Subtarget.isLittleEndian())
12398 std::reverse(first: Values.begin(), last: Values.end());
12399
12400 SDVTList Tys = DAG.getVTList(VT: MVT::Other);
12401 SmallVector<SDValue, 4> Ops{
12402 StoreChain, DAG.getConstant(Val: Intrinsic::ppc_vsx_stxvp, DL: dl, VT: MVT::i32),
12403 Values[0], BasePtr};
12404 MachineMemOperand *MMO = SN->getMemOperand();
12405 unsigned NumVecs = VT.getSizeInBits() / 256;
12406 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12407 MachineMemOperand *NewMMO =
12408 DAG.getMachineFunction().getMachineMemOperand(MMO, Offset: Idx * 32, Size: 32);
12409 if (Idx > 0) {
12410 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12411 N2: DAG.getConstant(Val: 32, DL: dl, VT: BasePtr.getValueType()));
12412 Ops[3] = BasePtr;
12413 }
12414 Ops[2] = Values[Idx];
12415 SDValue St = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl, VTList: Tys, Ops,
12416 MemVT: MVT::v256i1, MMO: NewMMO);
12417 Stores.push_back(Elt: St);
12418 }
12419
12420 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12421 return TF;
12422}
12423
12424SDValue PPCTargetLowering::LowerVectorStore(SDValue Op,
12425 SelectionDAG &DAG) const {
12426 SDLoc dl(Op);
12427 StoreSDNode *SN = cast<StoreSDNode>(Val: Op.getNode());
12428 SDValue StoreChain = SN->getChain();
12429 SDValue BasePtr = SN->getBasePtr();
12430 SDValue Value = SN->getValue();
12431 SDValue Value2 = SN->getValue();
12432 EVT StoreVT = Value.getValueType();
12433
12434 if (StoreVT == MVT::v1024i1 || StoreVT == MVT::v2048i1)
12435 return LowerDMFVectorStore(Op, DAG);
12436
12437 if (StoreVT != MVT::v256i1 && StoreVT != MVT::v512i1)
12438 return Op;
12439
12440 // Type v256i1 is used for pairs and v512i1 is used for accumulators.
12441 assert((StoreVT != MVT::v512i1 || Subtarget.hasMMA()) &&
12442 "Type unsupported without MMA");
12443 assert((StoreVT != MVT::v256i1 || Subtarget.pairedVectorMemops()) &&
12444 "Type unsupported without paired vector support");
12445
12446 // For v256i1 on ISA Future, let the store go through to instruction selection
12447 // where it will be matched to stxvp/pstxvp by the instruction patterns.
12448 if (StoreVT == MVT::v256i1 && Subtarget.isISAFuture() &&
12449 EnableAutoPairedVecSt)
12450 return Op;
12451
12452 // For other cases, create 2 or 4 v16i8 stores to store the pair or
12453 // accumulator underlying registers individually.
12454 Align Alignment = SN->getAlign();
12455 SmallVector<SDValue, 4> Stores;
12456 unsigned NumVecs = 2;
12457 if (StoreVT == MVT::v512i1) {
12458 if (Subtarget.isISAFuture()) {
12459 EVT ReturnTypes[] = {MVT::v256i1, MVT::v256i1};
12460 MachineSDNode *ExtNode = DAG.getMachineNode(
12461 Opcode: PPC::DMXXEXTFDMR512, dl, ResultTys: ReturnTypes, Ops: Op.getOperand(i: 1));
12462
12463 Value = SDValue(ExtNode, 0);
12464 Value2 = SDValue(ExtNode, 1);
12465 } else
12466 Value = DAG.getNode(Opcode: PPCISD::XXMFACC, DL: dl, VT: MVT::v512i1, Operand: Value);
12467 NumVecs = 4;
12468 }
12469 for (unsigned Idx = 0; Idx < NumVecs; ++Idx) {
12470 unsigned VecNum = Subtarget.isLittleEndian() ? NumVecs - 1 - Idx : Idx;
12471 SDValue Elt;
12472 if (Subtarget.isISAFuture()) {
12473 VecNum = Subtarget.isLittleEndian() ? 1 - (Idx % 2) : (Idx % 2);
12474 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8,
12475 N1: Idx > 1 ? Value2 : Value,
12476 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12477 } else
12478 Elt = DAG.getNode(Opcode: PPCISD::EXTRACT_VSX_REG, DL: dl, VT: MVT::v16i8, N1: Value,
12479 N2: DAG.getConstant(Val: VecNum, DL: dl, VT: getPointerTy(DL: DAG.getDataLayout())));
12480
12481 SDValue Store =
12482 DAG.getStore(Chain: StoreChain, dl, Val: Elt, Ptr: BasePtr,
12483 PtrInfo: SN->getPointerInfo().getWithOffset(O: Idx * 16),
12484 Alignment: commonAlignment(A: Alignment, Offset: Idx * 16),
12485 MMOFlags: SN->getMemOperand()->getFlags(), Metadata: SN->getAAInfo());
12486 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
12487 N2: DAG.getConstant(Val: 16, DL: dl, VT: BasePtr.getValueType()));
12488 Stores.push_back(Elt: Store);
12489 }
12490 SDValue TF = DAG.getTokenFactor(DL: dl, Vals&: Stores);
12491 return TF;
12492}
12493
12494SDValue PPCTargetLowering::LowerMUL(SDValue Op, SelectionDAG &DAG) const {
12495 SDLoc dl(Op);
12496 if (Op.getValueType() == MVT::v4i32) {
12497 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12498
12499 SDValue Zero = getCanonicalConstSplat(Val: 0, SplatSize: 1, VT: MVT::v4i32, DAG, dl);
12500 // +16 as shift amt.
12501 SDValue Neg16 = getCanonicalConstSplat(Val: -16, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
12502 SDValue RHSSwap = // = vrlw RHS, 16
12503 BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vrlw, LHS: RHS, RHS: Neg16, DAG, dl);
12504
12505 // Shrinkify inputs to v8i16.
12506 LHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: LHS);
12507 RHS = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHS);
12508 RHSSwap = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v8i16, Operand: RHSSwap);
12509
12510 // Low parts multiplied together, generating 32-bit results (we ignore the
12511 // top parts).
12512 SDValue LoProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmulouh,
12513 LHS, RHS, DAG, dl, DestVT: MVT::v4i32);
12514
12515 SDValue HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmsumuhm,
12516 Op0: LHS, Op1: RHSSwap, Op2: Zero, DAG, dl, DestVT: MVT::v4i32);
12517 // Shift the high parts up 16 bits.
12518 HiProd = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vslw, LHS: HiProd,
12519 RHS: Neg16, DAG, dl);
12520 return DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::v4i32, N1: LoProd, N2: HiProd);
12521 } else if (Op.getValueType() == MVT::v16i8) {
12522 SDValue LHS = Op.getOperand(i: 0), RHS = Op.getOperand(i: 1);
12523 bool isLittleEndian = Subtarget.isLittleEndian();
12524
12525 // Multiply the even 8-bit parts, producing 16-bit sums.
12526 SDValue EvenParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuleub,
12527 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12528 EvenParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: EvenParts);
12529
12530 // Multiply the odd 8-bit parts, producing 16-bit sums.
12531 SDValue OddParts = BuildIntrinsicOp(IID: Intrinsic::ppc_altivec_vmuloub,
12532 LHS, RHS, DAG, dl, DestVT: MVT::v8i16);
12533 OddParts = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v16i8, Operand: OddParts);
12534
12535 // Merge the results together. Because vmuleub and vmuloub are
12536 // instructions with a big-endian bias, we must reverse the
12537 // element numbering and reverse the meaning of "odd" and "even"
12538 // when generating little endian code.
12539 int Ops[16];
12540 for (unsigned i = 0; i != 8; ++i) {
12541 if (isLittleEndian) {
12542 Ops[i*2 ] = 2*i;
12543 Ops[i*2+1] = 2*i+16;
12544 } else {
12545 Ops[i*2 ] = 2*i+1;
12546 Ops[i*2+1] = 2*i+1+16;
12547 }
12548 }
12549 if (isLittleEndian)
12550 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: OddParts, N2: EvenParts, Mask: Ops);
12551 else
12552 return DAG.getVectorShuffle(VT: MVT::v16i8, dl, N1: EvenParts, N2: OddParts, Mask: Ops);
12553 } else {
12554 llvm_unreachable("Unknown mul to lower!");
12555 }
12556}
12557
12558SDValue PPCTargetLowering::LowerFP_ROUND(SDValue Op, SelectionDAG &DAG) const {
12559 bool IsStrict = Op->isStrictFPOpcode();
12560 if (Op.getOperand(i: IsStrict ? 1 : 0).getValueType() == MVT::f128 &&
12561 !Subtarget.hasP9Vector())
12562 return SDValue();
12563
12564 return Op;
12565}
12566
12567// Custom lowering for fpext vf32 to v2f64
12568SDValue PPCTargetLowering::LowerFP_EXTEND(SDValue Op, SelectionDAG &DAG) const {
12569
12570 assert(Op.getOpcode() == ISD::FP_EXTEND &&
12571 "Should only be called for ISD::FP_EXTEND");
12572
12573 // FIXME: handle extends from half precision float vectors on P9.
12574 // We only want to custom lower an extend from v2f32 to v2f64.
12575 if (Op.getValueType() != MVT::v2f64 ||
12576 Op.getOperand(i: 0).getValueType() != MVT::v2f32)
12577 return SDValue();
12578
12579 SDLoc dl(Op);
12580 SDValue Op0 = Op.getOperand(i: 0);
12581
12582 switch (Op0.getOpcode()) {
12583 default:
12584 return SDValue();
12585 case ISD::EXTRACT_SUBVECTOR: {
12586 assert(Op0.getNumOperands() == 2 &&
12587 isa<ConstantSDNode>(Op0->getOperand(1)) &&
12588 "Node should have 2 operands with second one being a constant!");
12589
12590 if (Op0.getOperand(i: 0).getValueType() != MVT::v4f32)
12591 return SDValue();
12592
12593 // Custom lower is only done for high or low doubleword.
12594 int Idx = Op0.getConstantOperandVal(i: 1);
12595 if (Idx % 2 != 0)
12596 return SDValue();
12597
12598 // Since input is v4f32, at this point Idx is either 0 or 2.
12599 // Shift to get the doubleword position we want.
12600 int DWord = Idx >> 1;
12601
12602 // High and low word positions are different on little endian.
12603 if (Subtarget.isLittleEndian())
12604 DWord ^= 0x1;
12605
12606 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64,
12607 N1: Op0.getOperand(i: 0), N2: DAG.getConstant(Val: DWord, DL: dl, VT: MVT::i32));
12608 }
12609 case ISD::FADD:
12610 case ISD::FMUL:
12611 case ISD::FSUB: {
12612 SDValue NewLoad[2];
12613 for (unsigned i = 0, ie = Op0.getNumOperands(); i != ie; ++i) {
12614 // Ensure both input are loads.
12615 SDValue LdOp = Op0.getOperand(i);
12616 if (LdOp.getOpcode() != ISD::LOAD)
12617 return SDValue();
12618 // Generate new load node.
12619 LoadSDNode *LD = cast<LoadSDNode>(Val&: LdOp);
12620 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12621 NewLoad[i] = DAG.getMemIntrinsicNode(
12622 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12623 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12624 }
12625 SDValue NewOp =
12626 DAG.getNode(Opcode: Op0.getOpcode(), DL: SDLoc(Op0), VT: MVT::v4f32, N1: NewLoad[0],
12627 N2: NewLoad[1], Flags: Op0.getNode()->getFlags());
12628 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewOp,
12629 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12630 }
12631 case ISD::LOAD: {
12632 LoadSDNode *LD = cast<LoadSDNode>(Val&: Op0);
12633 SDValue LoadOps[] = {LD->getChain(), LD->getBasePtr()};
12634 SDValue NewLd = DAG.getMemIntrinsicNode(
12635 Opcode: PPCISD::LD_VSX_LH, dl, VTList: DAG.getVTList(VT1: MVT::v4f32, VT2: MVT::Other), Ops: LoadOps,
12636 MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
12637 return DAG.getNode(Opcode: PPCISD::FP_EXTEND_HALF, DL: dl, VT: MVT::v2f64, N1: NewLd,
12638 N2: DAG.getConstant(Val: 0, DL: dl, VT: MVT::i32));
12639 }
12640 }
12641 llvm_unreachable("ERROR:Should return for all cases within swtich.");
12642}
12643
12644static SDValue ConvertCarryValueToCarryFlag(EVT SumType, SDValue Value,
12645 SelectionDAG &DAG,
12646 const PPCSubtarget &STI) {
12647 SDLoc DL(Value);
12648 if (STI.useCRBits())
12649 Value = DAG.getNode(Opcode: ISD::SELECT, DL, VT: SumType, N1: Value,
12650 N2: DAG.getConstant(Val: 1, DL, VT: SumType),
12651 N3: DAG.getConstant(Val: 0, DL, VT: SumType));
12652 else
12653 Value = DAG.getZExtOrTrunc(Op: Value, DL, VT: SumType);
12654 SDValue Sum = DAG.getNode(Opcode: PPCISD::ADDC, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32),
12655 N1: Value, N2: DAG.getAllOnesConstant(DL, VT: SumType));
12656 return Sum.getValue(R: 1);
12657}
12658
12659static SDValue ConvertCarryFlagToCarryValue(EVT SumType, SDValue Flag,
12660 EVT CarryType, SelectionDAG &DAG,
12661 const PPCSubtarget &STI) {
12662 SDLoc DL(Flag);
12663 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: SumType);
12664 SDValue Carry = DAG.getNode(
12665 Opcode: PPCISD::ADDE, DL, VTList: DAG.getVTList(VT1: SumType, VT2: MVT::i32), N1: Zero, N2: Zero, N3: Flag);
12666 if (STI.useCRBits())
12667 return DAG.getSetCC(DL, VT: CarryType, LHS: Carry, RHS: Zero, Cond: ISD::SETNE);
12668 return DAG.getZExtOrTrunc(Op: Carry, DL, VT: CarryType);
12669}
12670
12671SDValue PPCTargetLowering::LowerADDSUBO(SDValue Op, SelectionDAG &DAG) const {
12672
12673 SDLoc DL(Op);
12674 SDNode *N = Op.getNode();
12675 EVT VT = N->getValueType(ResNo: 0);
12676 EVT CarryType = N->getValueType(ResNo: 1);
12677 unsigned Opc = N->getOpcode();
12678 bool IsAdd = Opc == ISD::UADDO;
12679 Opc = IsAdd ? PPCISD::ADDC : PPCISD::SUBC;
12680 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12681 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
12682 SDValue Carry = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType,
12683 DAG, STI: Subtarget);
12684 if (!IsAdd)
12685 Carry = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Carry,
12686 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
12687 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: Carry);
12688}
12689
12690SDValue PPCTargetLowering::LowerADDSUBO_CARRY(SDValue Op,
12691 SelectionDAG &DAG) const {
12692 SDLoc DL(Op);
12693 SDNode *N = Op.getNode();
12694 unsigned Opc = N->getOpcode();
12695 EVT VT = N->getValueType(ResNo: 0);
12696 EVT CarryType = N->getValueType(ResNo: 1);
12697 SDValue CarryOp = N->getOperand(Num: 2);
12698 bool IsAdd = Opc == ISD::UADDO_CARRY;
12699 Opc = IsAdd ? PPCISD::ADDE : PPCISD::SUBE;
12700 if (!IsAdd)
12701 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12702 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12703 CarryOp = ConvertCarryValueToCarryFlag(SumType: VT, Value: CarryOp, DAG, STI: Subtarget);
12704 SDValue Sum = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: VT, VT2: MVT::i32),
12705 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1), N3: CarryOp);
12706 CarryOp = ConvertCarryFlagToCarryValue(SumType: VT, Flag: Sum.getValue(R: 1), CarryType, DAG,
12707 STI: Subtarget);
12708 if (!IsAdd)
12709 CarryOp = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryOp.getValueType(), N1: CarryOp,
12710 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryOp.getValueType()));
12711 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL, VTList: N->getVTList(), N1: Sum, N2: CarryOp);
12712}
12713
12714SDValue PPCTargetLowering::LowerSSUBO(SDValue Op, SelectionDAG &DAG) const {
12715
12716 SDLoc dl(Op);
12717 SDValue LHS = Op.getOperand(i: 0);
12718 SDValue RHS = Op.getOperand(i: 1);
12719 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12720
12721 SDValue Sub = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: LHS, N2: RHS);
12722
12723 SDValue Xor1 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: RHS, N2: LHS);
12724 SDValue Xor2 = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sub, N2: LHS);
12725
12726 SDValue And = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: Xor1, N2: Xor2);
12727
12728 SDValue Overflow =
12729 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: And,
12730 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12731
12732 SDValue OverflowTrunc =
12733 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12734
12735 return DAG.getMergeValues(Ops: {Sub, OverflowTrunc}, dl);
12736}
12737
12738/// Implements signed add with overflow detection using the rule:
12739/// (x eqv y) & (sum xor x), where the overflow bit is extracted from the sign
12740SDValue PPCTargetLowering::LowerSADDO(SDValue Op, SelectionDAG &DAG) const {
12741
12742 SDLoc dl(Op);
12743 SDValue LHS = Op.getOperand(i: 0);
12744 SDValue RHS = Op.getOperand(i: 1);
12745 EVT VT = Op.getNode()->getValueType(ResNo: 0);
12746
12747 SDValue Sum = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT, N1: LHS, N2: RHS);
12748
12749 // Compute ~(x xor y)
12750 SDValue XorXY = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: LHS, N2: RHS);
12751 SDValue EqvXY = DAG.getNOT(DL: dl, Val: XorXY, VT);
12752 // Compute (s xor x)
12753 SDValue SumXorX = DAG.getNode(Opcode: ISD::XOR, DL: dl, VT, N1: Sum, N2: LHS);
12754
12755 // overflow = (x eqv y) & (s xor x)
12756 SDValue OverflowInSign = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: EqvXY, N2: SumXorX);
12757
12758 // Shift sign bit down to LSB
12759 SDValue Overflow =
12760 DAG.getNode(Opcode: ISD::SRL, DL: dl, VT, N1: OverflowInSign,
12761 N2: DAG.getConstant(Val: VT.getSizeInBits() - 1, DL: dl, VT: MVT::i32));
12762 // Truncate to the overflow type (i1)
12763 SDValue OverflowTrunc =
12764 DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: Op.getNode()->getValueType(ResNo: 1), Operand: Overflow);
12765
12766 return DAG.getMergeValues(Ops: {Sum, OverflowTrunc}, dl);
12767}
12768
12769/// Lower ABDU with negation pattern using branchless carry arithmetic.
12770/// Recognizes: abdu(a, sub(0, x)) and transforms to:
12771/// a - (0 - x) = a + x (mod 2^n)
12772/// Uses SUBC to compute result without branches.
12773SDValue PPCTargetLowering::LowerABDU(SDValue Op, SelectionDAG &DAG) const {
12774 SDLoc DL(Op);
12775 SDValue LHS = Op.getOperand(i: 0);
12776 SDValue RHS = Op.getOperand(i: 1);
12777 EVT OpVT = LHS.getValueType();
12778 EVT VT = Op.getValueType();
12779 bool IsNonNegative = DAG.SignBitIsZero(Op: LHS) && DAG.SignBitIsZero(Op: RHS);
12780
12781 // If the subtract doesn't overflow then just use abs(sub()).
12782 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: LHS, N1: RHS))
12783 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12784 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: RHS));
12785
12786 if (DAG.willNotOverflowSub(IsSigned: IsNonNegative, N0: RHS, N1: LHS))
12787 return DAG.getNode(Opcode: ISD::ABS, DL, VT,
12788 Operand: DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: RHS, N2: LHS));
12789
12790 // General path: use SUBC (or ADDC when RHS is 0-X) to get
12791 // subtract-with-flags, then CMOV to select a-b or b-a. ADDC/SUBC produce the
12792 // flags we need.
12793 unsigned Opcode = PPCISD::SUBC;
12794
12795 // Check if RHS is a negation (0 - X). If so, we can use ADDC instead of SUBC:
12796 // a - (0 - x) = a + x (mod 2^n)
12797 // Same semantics as in LowerCMP; apply same safety checks.
12798 if (RHS.getOpcode() == ISD::SUB) {
12799 SDValue SubLHS = RHS.getOperand(i: 0);
12800 SDValue SubRHS = RHS.getOperand(i: 1);
12801
12802 if (isNullConstant(V: SubLHS) && DAG.isKnownNeverZero(Op: SubRHS)) {
12803 Opcode = PPCISD::ADDC;
12804 RHS = SubRHS;
12805 }
12806 }
12807
12808 // On PPC64, carry ops use the full 64-bit register. Operands are type-legal
12809 // i32 here; widen only for the carry path (fast abs(sub) stays at VT).
12810 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12811 LHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: LHS);
12812 RHS = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: RHS);
12813 OpVT = MVT::i64;
12814 }
12815
12816 // Calculate LHS - RHS and capture the carry (CA)
12817 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12818 SDValue Res = DAG.getNode(Opcode, DL, VTList: VTs, N1: LHS, N2: RHS);
12819 SDValue CA0 = Res.getValue(R: 1);
12820
12821 // t2 = A - B + CA0 using SUBE.
12822 SDValue ZeroOrNeg1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Res, N2: Res, N3: CA0);
12823
12824 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT: OpVT, N1: Res, N2: ZeroOrNeg1);
12825
12826 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: Xor, N2: ZeroOrNeg1);
12827
12828 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
12829 return Res;
12830}
12831
12832// Lower unsigned 3-way compare producing -1/0/1.
12833SDValue PPCTargetLowering::LowerUCMP(SDValue Op, SelectionDAG &DAG) const {
12834 SDLoc DL(Op);
12835 SDValue A = DAG.getFreeze(V: Op.getOperand(i: 0));
12836 SDValue B = DAG.getFreeze(V: Op.getOperand(i: 1));
12837 EVT OpVT = A.getValueType();
12838 EVT ResVT = Op.getValueType();
12839
12840 // On PPC64, i32 carries are affected by the upper 32 bits of the registers.
12841 // We must zero-extend to i64 to ensure the carry reflects the 32-bit unsigned
12842 // comparison.
12843 if (Subtarget.isPPC64() && OpVT != MVT::i64) {
12844 A = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: A);
12845 B = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: B);
12846 OpVT = MVT::i64;
12847 }
12848
12849 // First compute diff = A - B.
12850 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: OpVT, N1: A, N2: B);
12851
12852 // Generate B - A using SUBC to capture carry.
12853 SDVTList VTs = DAG.getVTList(VT1: OpVT, VT2: MVT::i32);
12854 SDValue SubC = DAG.getNode(Opcode: PPCISD::SUBC, DL, VTList: VTs, N1: B, N2: A);
12855 SDValue CA0 = SubC.getValue(R: 1);
12856
12857 // t2 = A - B + CA0 using SUBE.
12858 SDValue SubE1 = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: A, N2: B, N3: CA0);
12859 SDValue CA1 = SubE1.getValue(R: 1);
12860
12861 // res = diff - t2 + CA1 using SUBE (produces desired -1/0/1).
12862 SDValue ResPair = DAG.getNode(Opcode: PPCISD::SUBE, DL, VTList: VTs, N1: Diff, N2: SubE1, N3: CA1);
12863
12864 // Extract the first result and truncate to result type if needed.
12865 return DAG.getSExtOrTrunc(Op: ResPair.getValue(R: 0), DL, VT: ResVT);
12866}
12867
12868// Lower signed 3-way compare producing -1/0/1.
12869// For PowerPC with SETB (P9+), use the default expansion.
12870// For pre-P9, we generate a custom sequence using comparison and bit
12871// extraction.
12872// cmpw cr7, LHS, RHS
12873// mfocrf r, CR7
12874// rlwinm LT, r, 29, 31, 31 (extract LT bit from CR7)
12875// rlwinm GT, r, 30, 31, 31 (extract GT bit from CR7)
12876// subf result, LT, GT (GT - LT = -1/0/1)
12877SDValue PPCTargetLowering::LowerSCMP(SDValue Op, SelectionDAG &DAG) const {
12878 SDLoc DL(Op);
12879 SDValue LHS = Op.getOperand(i: 0);
12880 SDValue RHS = Op.getOperand(i: 1);
12881 EVT VT = LHS.getValueType();
12882 EVT ResVT = Op.getValueType();
12883
12884 // For P9+, use the default expansion which will use SETB instruction
12885 if (Subtarget.isISA3_0())
12886 return SDValue();
12887
12888 // For pre-P9, generate custom sequence without SETB
12889 bool Is64BitCmp = (VT == MVT::i64);
12890
12891 // On PPC64, always use 64-bit operations to avoid extra sign-extension
12892 // Even if result type is i32, it will likely be extended to i64 by calling
12893 // convention
12894 bool Use64BitOps = Subtarget.isPPC64();
12895
12896 // Perform comparison
12897 unsigned CmpOpc = Is64BitCmp ? PPC::CMPD : PPC::CMPW;
12898 SDValue Cmp = SDValue(DAG.getMachineNode(Opcode: CmpOpc, dl: DL, VT: MVT::i32, Op1: LHS, Op2: RHS), 0);
12899
12900 // Move comparison result to CR7 to have known bit positions
12901 SDValue CR7Reg = DAG.getRegister(Reg: PPC::CR7, VT: MVT::i32);
12902 SDValue InGlue; // Null incoming glue
12903 SDValue CopyToReg =
12904 DAG.getCopyToReg(Chain: DAG.getEntryNode(), dl: DL, Reg: CR7Reg, N: Cmp, Glue: InGlue);
12905 SDValue Glue = CopyToReg.getValue(R: 1);
12906
12907 // Use MFOCRF to read CR7
12908 unsigned MFOCRFOpc = Use64BitOps ? PPC::MFOCRF8 : PPC::MFOCRF;
12909 EVT MFVT = Use64BitOps ? MVT::i64 : MVT::i32;
12910 SDValue MFOCRF =
12911 SDValue(DAG.getMachineNode(Opcode: MFOCRFOpc, dl: DL, VT: MFVT, Op1: CR7Reg, Op2: Glue), 0);
12912
12913 // CR7 bits in the result (after MFOCRF):
12914 // Bit 28: LT
12915 // Bit 29: GT
12916 // Bit 30: EQ
12917 // Bit 31: SO
12918
12919 // Extract LT and GT bits using RLWINM
12920 unsigned RLWinmOpc = Use64BitOps ? PPC::RLWINM8 : PPC::RLWINM;
12921
12922 // Extract LT bit (Bit 28 -> Rotate Left 29 to move to bit 31, then mask)
12923 SDValue LTOps[] = {MFOCRF, DAG.getTargetConstant(Val: 29, DL, VT: MVT::i32),
12924 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32),
12925 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32)};
12926 SDValue LTBit = SDValue(DAG.getMachineNode(Opcode: RLWinmOpc, dl: DL, VT: MFVT, Ops: LTOps), 0);
12927
12928 // Extract GT bit (Bit 29 -> Rotate Left 30 to move to bit 31, then mask)
12929 SDValue GTOps[] = {MFOCRF, DAG.getTargetConstant(Val: 30, DL, VT: MVT::i32),
12930 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32),
12931 DAG.getTargetConstant(Val: 31, DL, VT: MVT::i32)};
12932 SDValue GTBit = SDValue(DAG.getMachineNode(Opcode: RLWinmOpc, dl: DL, VT: MFVT, Ops: GTOps), 0);
12933
12934 // Compute result: GT - LT
12935 // If LT: 0 - 1 = -1
12936 // If GT: 1 - 0 = 1
12937 // If EQ: 0 - 0 = 0
12938 unsigned SubOpc = Use64BitOps ? PPC::SUBF8 : PPC::SUBF;
12939 SDValue Result =
12940 SDValue(DAG.getMachineNode(Opcode: SubOpc, dl: DL, VT: MFVT, Op1: LTBit, Op2: GTBit), 0);
12941
12942 if (ResVT == MFVT)
12943 return Result;
12944
12945 // Adjust result to match expected type (sign-extend or truncate as needed)
12946 return DAG.getSExtOrTrunc(Op: Result, DL, VT: ResVT);
12947}
12948
12949/// LowerOperation - Provide custom lowering hooks for some operations.
12950///
12951SDValue PPCTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
12952 switch (Op.getOpcode()) {
12953 default:
12954 llvm_unreachable("Wasn't expecting to be able to lower this!");
12955 case ISD::FPOW: return lowerPow(Op, DAG);
12956 case ISD::FSIN: return lowerSin(Op, DAG);
12957 case ISD::FCOS: return lowerCos(Op, DAG);
12958 case ISD::FLOG: return lowerLog(Op, DAG);
12959 case ISD::FLOG10: return lowerLog10(Op, DAG);
12960 case ISD::FEXP: return lowerExp(Op, DAG);
12961 case ISD::ConstantPool: return LowerConstantPool(Op, DAG);
12962 case ISD::BlockAddress: return LowerBlockAddress(Op, DAG);
12963 case ISD::GlobalAddress: return LowerGlobalAddress(Op, DAG);
12964 case ISD::GlobalTLSAddress: return LowerGlobalTLSAddress(Op, DAG);
12965 case ISD::JumpTable: return LowerJumpTable(Op, DAG);
12966 case ISD::STRICT_FSETCC:
12967 case ISD::STRICT_FSETCCS:
12968 case ISD::SETCC: return LowerSETCC(Op, DAG);
12969 case ISD::BR_CC: return LowerBR_CC(Op, DAG);
12970 case ISD::INIT_TRAMPOLINE: return LowerINIT_TRAMPOLINE(Op, DAG);
12971 case ISD::ADJUST_TRAMPOLINE: return LowerADJUST_TRAMPOLINE(Op, DAG);
12972 case ISD::SSUBO:
12973 return LowerSSUBO(Op, DAG);
12974 case ISD::SADDO:
12975 return LowerSADDO(Op, DAG);
12976
12977 case ISD::INLINEASM:
12978 case ISD::INLINEASM_BR: return LowerINLINEASM(Op, DAG);
12979 // Variable argument lowering.
12980 case ISD::VASTART: return LowerVASTART(Op, DAG);
12981 case ISD::VAARG: return LowerVAARG(Op, DAG);
12982 case ISD::VACOPY: return LowerVACOPY(Op, DAG);
12983
12984 case ISD::STACKRESTORE: return LowerSTACKRESTORE(Op, DAG);
12985 case ISD::DYNAMIC_STACKALLOC: return LowerDYNAMIC_STACKALLOC(Op, DAG);
12986 case ISD::GET_DYNAMIC_AREA_OFFSET:
12987 return LowerGET_DYNAMIC_AREA_OFFSET(Op, DAG);
12988
12989 // Exception handling lowering.
12990 case ISD::EH_DWARF_CFA: return LowerEH_DWARF_CFA(Op, DAG);
12991 case ISD::EH_SJLJ_SETJMP: return lowerEH_SJLJ_SETJMP(Op, DAG);
12992 case ISD::EH_SJLJ_LONGJMP: return lowerEH_SJLJ_LONGJMP(Op, DAG);
12993
12994 case ISD::LOAD: return LowerLOAD(Op, DAG);
12995 case ISD::STORE: return LowerSTORE(Op, DAG);
12996 case ISD::TRUNCATE: return LowerTRUNCATE(Op, DAG);
12997 case ISD::SELECT_CC: return LowerSELECT_CC(Op, DAG);
12998 case ISD::STRICT_FP_TO_UINT:
12999 case ISD::STRICT_FP_TO_SINT:
13000 case ISD::FP_TO_UINT:
13001 case ISD::FP_TO_SINT: return LowerFP_TO_INT(Op, DAG, dl: SDLoc(Op));
13002 case ISD::STRICT_UINT_TO_FP:
13003 case ISD::STRICT_SINT_TO_FP:
13004 case ISD::UINT_TO_FP:
13005 case ISD::SINT_TO_FP: return LowerINT_TO_FP(Op, DAG);
13006 case ISD::GET_ROUNDING: return LowerGET_ROUNDING(Op, DAG);
13007 case ISD::SET_ROUNDING:
13008 return LowerSET_ROUNDING(Op, DAG);
13009
13010 // Lower 64-bit shifts.
13011 case ISD::SHL_PARTS: return LowerSHL_PARTS(Op, DAG);
13012 case ISD::SRL_PARTS: return LowerSRL_PARTS(Op, DAG);
13013 case ISD::SRA_PARTS: return LowerSRA_PARTS(Op, DAG);
13014
13015 case ISD::FSHL: return LowerFunnelShift(Op, DAG);
13016 case ISD::FSHR: return LowerFunnelShift(Op, DAG);
13017
13018 // Vector-related lowering.
13019 case ISD::BUILD_VECTOR: return LowerBUILD_VECTOR(Op, DAG);
13020 case ISD::VECTOR_SHUFFLE: return LowerVECTOR_SHUFFLE(Op, DAG);
13021 case ISD::INTRINSIC_WO_CHAIN: return LowerINTRINSIC_WO_CHAIN(Op, DAG);
13022 case ISD::SCALAR_TO_VECTOR: return LowerSCALAR_TO_VECTOR(Op, DAG);
13023 case ISD::INSERT_VECTOR_ELT: return LowerINSERT_VECTOR_ELT(Op, DAG);
13024 case ISD::MUL: return LowerMUL(Op, DAG);
13025 case ISD::FP_EXTEND: return LowerFP_EXTEND(Op, DAG);
13026 case ISD::STRICT_FP_ROUND:
13027 case ISD::FP_ROUND:
13028 return LowerFP_ROUND(Op, DAG);
13029 case ISD::ROTL: return LowerROTL(Op, DAG);
13030
13031 // For counter-based loop handling.
13032 case ISD::INTRINSIC_W_CHAIN:
13033 return SDValue();
13034
13035 case ISD::BITCAST: return LowerBITCAST(Op, DAG);
13036
13037 // Frame & Return address.
13038 case ISD::RETURNADDR: return LowerRETURNADDR(Op, DAG);
13039 case ISD::FRAMEADDR: return LowerFRAMEADDR(Op, DAG);
13040
13041 case ISD::INTRINSIC_VOID:
13042 return LowerINTRINSIC_VOID(Op, DAG);
13043 case ISD::BSWAP:
13044 return LowerBSWAP(Op, DAG);
13045 case ISD::ATOMIC_STORE:
13046 return LowerATOMIC_LOAD_STORE(Op, DAG);
13047 case ISD::IS_FPCLASS:
13048 return LowerIS_FPCLASS(Op, DAG);
13049 case ISD::UADDO:
13050 case ISD::USUBO:
13051 return LowerADDSUBO(Op, DAG);
13052 case ISD::UADDO_CARRY:
13053 case ISD::USUBO_CARRY:
13054 return LowerADDSUBO_CARRY(Op, DAG);
13055 case ISD::UCMP:
13056 return LowerUCMP(Op, DAG);
13057 case ISD::ABDU:
13058 return LowerABDU(Op, DAG);
13059 case ISD::SCMP:
13060 return LowerSCMP(Op, DAG);
13061 case ISD::STRICT_LRINT:
13062 case ISD::STRICT_LLRINT:
13063 case ISD::STRICT_LROUND:
13064 case ISD::STRICT_LLROUND:
13065 case ISD::STRICT_FNEARBYINT:
13066 if (Op->getFlags().hasNoFPExcept())
13067 return Op;
13068 return SDValue();
13069 case ISD::VP_LOAD:
13070 return LowerVP_LOAD(Op, DAG);
13071 case ISD::VP_STORE:
13072 return LowerVP_STORE(Op, DAG);
13073 case ISD::PARTIAL_REDUCE_UMLA:
13074 return LowerPartialReduce(Op, DAG);
13075 }
13076}
13077
13078void PPCTargetLowering::ReplaceNodeResults(SDNode *N,
13079 SmallVectorImpl<SDValue>&Results,
13080 SelectionDAG &DAG) const {
13081 SDLoc dl(N);
13082 switch (N->getOpcode()) {
13083 default:
13084 llvm_unreachable("Do not know how to custom type legalize this operation!");
13085 case ISD::ATOMIC_LOAD: {
13086 SDValue Res = LowerATOMIC_LOAD_STORE(Op: SDValue(N, 0), DAG);
13087 Results.push_back(Elt: Res);
13088 Results.push_back(Elt: Res.getValue(R: 1));
13089 break;
13090 }
13091 case ISD::READCYCLECOUNTER: {
13092 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other);
13093 SDValue RTB = DAG.getNode(Opcode: PPCISD::READ_TIME_BASE, DL: dl, VTList: VTs, N: N->getOperand(Num: 0));
13094
13095 Results.push_back(
13096 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: RTB, N2: RTB.getValue(R: 1)));
13097 Results.push_back(Elt: RTB.getValue(R: 2));
13098 break;
13099 }
13100 case ISD::INTRINSIC_W_CHAIN: {
13101 if (N->getConstantOperandVal(Num: 1) != Intrinsic::loop_decrement)
13102 break;
13103
13104 assert(N->getValueType(0) == MVT::i1 &&
13105 "Unexpected result type for CTR decrement intrinsic");
13106 EVT SVT = getSetCCResultType(DL: DAG.getDataLayout(), C&: *DAG.getContext(),
13107 VT: N->getValueType(ResNo: 0));
13108 SDVTList VTs = DAG.getVTList(VT1: SVT, VT2: MVT::Other);
13109 SDValue NewInt = DAG.getNode(Opcode: N->getOpcode(), DL: dl, VTList: VTs, N1: N->getOperand(Num: 0),
13110 N2: N->getOperand(Num: 1));
13111
13112 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: NewInt));
13113 Results.push_back(Elt: NewInt.getValue(R: 1));
13114 break;
13115 }
13116 case ISD::INTRINSIC_WO_CHAIN: {
13117 switch (N->getConstantOperandVal(Num: 0)) {
13118 case Intrinsic::ppc_pack_longdouble:
13119 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::ppcf128,
13120 N1: N->getOperand(Num: 2), N2: N->getOperand(Num: 1)));
13121 break;
13122 case Intrinsic::ppc_maxfe:
13123 case Intrinsic::ppc_minfe:
13124 case Intrinsic::ppc_fnmsub:
13125 case Intrinsic::ppc_convert_f128_to_ppcf128:
13126 Results.push_back(Elt: LowerINTRINSIC_WO_CHAIN(Op: SDValue(N, 0), DAG));
13127 break;
13128 }
13129 break;
13130 }
13131 case ISD::VAARG: {
13132 if (!Subtarget.isSVR4ABI() || Subtarget.isPPC64())
13133 return;
13134
13135 EVT VT = N->getValueType(ResNo: 0);
13136
13137 if (VT == MVT::i64) {
13138 SDValue NewNode = LowerVAARG(Op: SDValue(N, 1), DAG);
13139
13140 Results.push_back(Elt: NewNode);
13141 Results.push_back(Elt: NewNode.getValue(R: 1));
13142 }
13143 return;
13144 }
13145 case ISD::STRICT_FP_TO_SINT:
13146 case ISD::STRICT_FP_TO_UINT:
13147 case ISD::FP_TO_SINT:
13148 case ISD::FP_TO_UINT: {
13149 // LowerFP_TO_INT() can only handle f32 and f64.
13150 if (N->getOperand(Num: N->isStrictFPOpcode() ? 1 : 0).getValueType() ==
13151 MVT::ppcf128)
13152 return;
13153 SDValue LoweredValue = LowerFP_TO_INT(Op: SDValue(N, 0), DAG, dl);
13154 Results.push_back(Elt: LoweredValue);
13155 if (N->isStrictFPOpcode())
13156 Results.push_back(Elt: LoweredValue.getValue(R: 1));
13157 return;
13158 }
13159 case ISD::TRUNCATE: {
13160 if (!N->getValueType(ResNo: 0).isVector())
13161 return;
13162 SDValue Lowered = LowerTRUNCATEVector(Op: SDValue(N, 0), DAG);
13163 if (Lowered)
13164 Results.push_back(Elt: Lowered);
13165 return;
13166 }
13167 case ISD::SCALAR_TO_VECTOR: {
13168 SDValue Lowered = LowerSCALAR_TO_VECTOR(Op: SDValue(N, 0), DAG);
13169 if (Lowered)
13170 Results.push_back(Elt: Lowered);
13171 return;
13172 }
13173 case ISD::FSHL:
13174 case ISD::FSHR:
13175 // Don't handle funnel shifts here.
13176 return;
13177 case ISD::BITCAST:
13178 // Don't handle bitcast here.
13179 return;
13180 case ISD::FP_EXTEND:
13181 SDValue Lowered = LowerFP_EXTEND(Op: SDValue(N, 0), DAG);
13182 if (Lowered)
13183 Results.push_back(Elt: Lowered);
13184 return;
13185 }
13186}
13187
13188//===----------------------------------------------------------------------===//
13189// Other Lowering Code
13190//===----------------------------------------------------------------------===//
13191
13192static CallInst *callIntrinsic(IRBuilderBase &Builder, Intrinsic::ID Id) {
13193 return Builder.CreateIntrinsicWithoutFolding(ID: Id, Args: {});
13194}
13195
13196Value *PPCTargetLowering::emitLoadLinked(IRBuilderBase &Builder, Type *ValueTy,
13197 Value *Addr,
13198 AtomicOrdering Ord) const {
13199 unsigned SZ = ValueTy->getPrimitiveSizeInBits();
13200
13201 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13202 "Only 8/16/32/64-bit atomic loads supported");
13203 Intrinsic::ID IntID;
13204 switch (SZ) {
13205 default:
13206 llvm_unreachable("Unexpected PrimitiveSize");
13207 case 8:
13208 IntID = Intrinsic::ppc_lbarx;
13209 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13210 break;
13211 case 16:
13212 IntID = Intrinsic::ppc_lharx;
13213 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13214 break;
13215 case 32:
13216 IntID = Intrinsic::ppc_lwarx;
13217 break;
13218 case 64:
13219 IntID = Intrinsic::ppc_ldarx;
13220 break;
13221 }
13222 Value *Call =
13223 Builder.CreateIntrinsic(ID: IntID, Args: Addr, /*FMFSource=*/nullptr, Name: "larx");
13224
13225 return Builder.CreateTruncOrBitCast(V: Call, DestTy: ValueTy);
13226}
13227
13228// Perform a store-conditional operation to Addr. Return the status of the
13229// store. This should be 0 if the store succeeded, non-zero otherwise.
13230Value *PPCTargetLowering::emitStoreConditional(IRBuilderBase &Builder,
13231 Value *Val, Value *Addr,
13232 AtomicOrdering Ord) const {
13233 Type *Ty = Val->getType();
13234 unsigned SZ = Ty->getPrimitiveSizeInBits();
13235
13236 assert((SZ == 8 || SZ == 16 || SZ == 32 || SZ == 64) &&
13237 "Only 8/16/32/64-bit atomic loads supported");
13238 Intrinsic::ID IntID;
13239 switch (SZ) {
13240 default:
13241 llvm_unreachable("Unexpected PrimitiveSize");
13242 case 8:
13243 IntID = Intrinsic::ppc_stbcx;
13244 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13245 break;
13246 case 16:
13247 IntID = Intrinsic::ppc_sthcx;
13248 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
13249 break;
13250 case 32:
13251 IntID = Intrinsic::ppc_stwcx;
13252 break;
13253 case 64:
13254 IntID = Intrinsic::ppc_stdcx;
13255 break;
13256 }
13257
13258 if (SZ == 8 || SZ == 16)
13259 Val = Builder.CreateZExt(V: Val, DestTy: Builder.getInt32Ty());
13260
13261 Value *Call = Builder.CreateIntrinsic(ID: IntID, Args: {Addr, Val},
13262 /*FMFSource=*/nullptr, Name: "stcx");
13263 return Builder.CreateXor(LHS: Call, RHS: Builder.getInt32(C: 1));
13264}
13265
13266// The mappings for emitLeading/TrailingFence is taken from
13267// http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html
13268Instruction *PPCTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
13269 Instruction *Inst,
13270 AtomicOrdering Ord) const {
13271 if (Ord == AtomicOrdering::SequentiallyConsistent)
13272 return callIntrinsic(Builder, Id: Intrinsic::ppc_sync);
13273 if (isReleaseOrStronger(AO: Ord))
13274 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13275 return nullptr;
13276}
13277
13278Instruction *PPCTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
13279 Instruction *Inst,
13280 AtomicOrdering Ord) const {
13281 if (Inst->hasAtomicLoad() && isAcquireOrStronger(AO: Ord)) {
13282 // See http://www.cl.cam.ac.uk/~pes20/cpp/cpp0xmappings.html and
13283 // http://www.rdrop.com/users/paulmck/scalability/paper/N2745r.2011.03.04a.html
13284 // and http://www.cl.cam.ac.uk/~pes20/cppppc/ for justification.
13285 if (isa<LoadInst>(Val: Inst))
13286 return Builder.CreateIntrinsicWithoutFolding(ID: Intrinsic::ppc_cfence,
13287 OverloadTypes: {Inst->getType()}, Args: {Inst});
13288 // FIXME: Can use isync for rmw operation.
13289 return callIntrinsic(Builder, Id: Intrinsic::ppc_lwsync);
13290 }
13291 return nullptr;
13292}
13293
13294bool PPCTargetLowering::fenceClearsLoadLinkedReservation() const {
13295 return !Subtarget.fenceKeepsReservation();
13296}
13297
13298MachineBasicBlock *PPCTargetLowering::EmitAtomicBinary(MachineInstr &MI,
13299 MachineBasicBlock *BB,
13300 unsigned BinOpcode,
13301 unsigned CmpOpcode,
13302 unsigned CmpPred) const {
13303 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13304 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13305 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13306 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13307 unsigned AtomicSize = MI.getOperand(i: 3).getImm();
13308
13309 auto LoadMnemonic = PPC::LDARX;
13310 auto StoreMnemonic = PPC::STDCX;
13311 switch (AtomicSize) {
13312 default:
13313 llvm_unreachable("Unexpected size of atomic entity");
13314 case 1:
13315 LoadMnemonic = PPC::LBARX;
13316 StoreMnemonic = PPC::STBCX;
13317 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13318 break;
13319 case 2:
13320 LoadMnemonic = PPC::LHARX;
13321 StoreMnemonic = PPC::STHCX;
13322 assert(Subtarget.hasPartwordAtomics() && "Call this only with size >=4");
13323 break;
13324 case 4:
13325 LoadMnemonic = PPC::LWARX;
13326 StoreMnemonic = PPC::STWCX;
13327 break;
13328 case 8:
13329 LoadMnemonic = PPC::LDARX;
13330 StoreMnemonic = PPC::STDCX;
13331 break;
13332 }
13333
13334 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13335 MachineFunction *F = BB->getParent();
13336 MachineFunction::iterator It = ++BB->getIterator();
13337
13338 if (CmpOpcode == PPC::CMPW && (AtomicSize == 1 || AtomicSize == 2))
13339 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, /*IsByte=*/AtomicSize == 1, TII);
13340
13341 Register dest = MI.getOperand(i: 0).getReg();
13342 Register ptrA = MI.getOperand(i: 1).getReg();
13343 Register ptrB = MI.getOperand(i: 2).getReg();
13344 Register incr = MI.getOperand(i: 4).getReg();
13345 DebugLoc dl = MI.getDebugLoc();
13346
13347 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13348 MachineBasicBlock *loop2MBB =
13349 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13350 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13351 F->insert(MBBI: It, MBB: loopMBB);
13352 if (CmpOpcode)
13353 F->insert(MBBI: It, MBB: loop2MBB);
13354 F->insert(MBBI: It, MBB: exitMBB);
13355 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13356 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13357 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13358
13359 MachineRegisterInfo &RegInfo = F->getRegInfo();
13360 Register TmpReg = (!BinOpcode) ? incr :
13361 RegInfo.createVirtualRegister( RegClass: AtomicSize == 8 ? &PPC::G8RCRegClass
13362 : &PPC::GPRCRegClass);
13363
13364 // thisMBB:
13365 // ...
13366 // fallthrough --> loopMBB
13367 BB->addSuccessor(Succ: loopMBB);
13368
13369 // loopMBB:
13370 // l[wd]arx dest, ptr
13371 // add r0, dest, incr
13372 // st[wd]cx. r0, ptr
13373 // bne- loopMBB
13374 // fallthrough --> exitMBB
13375
13376 // For max/min...
13377 // loopMBB:
13378 // l[wd]arx dest, ptr
13379 // cmpl?[wd] dest, incr
13380 // bgt exitMBB
13381 // loop2MBB:
13382 // st[wd]cx. dest, ptr
13383 // bne- loopMBB
13384 // fallthrough --> exitMBB
13385
13386 BB = loopMBB;
13387 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest)
13388 .addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13389 if (BinOpcode)
13390 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg).addReg(RegNo: incr).addReg(RegNo: dest);
13391 if (CmpOpcode) {
13392 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13393 // Signed comparisons of byte or halfword values must be sign-extended.
13394 if (CmpOpcode == PPC::CMPW && AtomicSize < 4) {
13395 Register ExtReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13396 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: AtomicSize == 1 ? PPC::EXTSB : PPC::EXTSH),
13397 DestReg: ExtReg).addReg(RegNo: dest);
13398 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ExtReg).addReg(RegNo: incr);
13399 } else
13400 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: dest).addReg(RegNo: incr);
13401
13402 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13403 .addImm(Val: CmpPred)
13404 .addReg(RegNo: CrReg)
13405 .addMBB(MBB: exitMBB);
13406 BB->addSuccessor(Succ: loop2MBB);
13407 BB->addSuccessor(Succ: exitMBB);
13408 BB = loop2MBB;
13409 }
13410 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
13411 .addReg(RegNo: TmpReg).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
13412 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13413 .addImm(Val: PPC::PRED_NE_MINUS)
13414 .addReg(RegNo: PPC::CR0)
13415 .addMBB(MBB: loopMBB);
13416 BB->addSuccessor(Succ: loopMBB);
13417 BB->addSuccessor(Succ: exitMBB);
13418
13419 // exitMBB:
13420 // ...
13421 BB = exitMBB;
13422 return BB;
13423}
13424
13425static bool isSignExtended(MachineInstr &MI, const PPCInstrInfo *TII) {
13426 switch(MI.getOpcode()) {
13427 default:
13428 return false;
13429 case PPC::COPY:
13430 return TII->isSignExtended(Reg: MI.getOperand(i: 1).getReg(),
13431 MRI: &MI.getMF()->getRegInfo());
13432 case PPC::LHA:
13433 case PPC::LHA8:
13434 case PPC::LHAU:
13435 case PPC::LHAU8:
13436 case PPC::LHAUX:
13437 case PPC::LHAUX8:
13438 case PPC::LHAX:
13439 case PPC::LHAX8:
13440 case PPC::LWA:
13441 case PPC::LWAUX:
13442 case PPC::LWAX:
13443 case PPC::LWAX_32:
13444 case PPC::LWA_32:
13445 case PPC::PLHA:
13446 case PPC::PLHA8:
13447 case PPC::PLHA8pc:
13448 case PPC::PLHApc:
13449 case PPC::PLWA:
13450 case PPC::PLWA8:
13451 case PPC::PLWA8pc:
13452 case PPC::PLWApc:
13453 case PPC::EXTSB:
13454 case PPC::EXTSB8:
13455 case PPC::EXTSB8_32_64:
13456 case PPC::EXTSB8_rec:
13457 case PPC::EXTSB_rec:
13458 case PPC::EXTSH:
13459 case PPC::EXTSH8:
13460 case PPC::EXTSH8_32_64:
13461 case PPC::EXTSH8_rec:
13462 case PPC::EXTSH_rec:
13463 case PPC::EXTSW:
13464 case PPC::EXTSWSLI:
13465 case PPC::EXTSWSLI_32_64:
13466 case PPC::EXTSWSLI_32_64_rec:
13467 case PPC::EXTSWSLI_rec:
13468 case PPC::EXTSW_32:
13469 case PPC::EXTSW_32_64:
13470 case PPC::EXTSW_32_64_rec:
13471 case PPC::EXTSW_rec:
13472 case PPC::SRAW:
13473 case PPC::SRAWI:
13474 case PPC::SRAWI_rec:
13475 case PPC::SRAW_rec:
13476 return true;
13477 }
13478 return false;
13479}
13480
13481// Sign extend operand OpIdx if the value is not known to be sign extended.
13482// Assumes the operand is a register. The flag IsByte controls which instruction
13483// is used for the sign extension.
13484static void signExtendOperandIfUnknown(MachineInstr &MI, MachineBasicBlock *BB,
13485 unsigned OpIdx, bool IsByte,
13486 const PPCInstrInfo *TII) {
13487 MachineFunction *F = MI.getMF();
13488 MachineRegisterInfo &RegInfo = F->getRegInfo();
13489 Register Reg = MI.getOperand(i: OpIdx).getReg();
13490 bool IsSignExtended =
13491 Reg.isVirtual() && isSignExtended(MI&: *RegInfo.getVRegDef(Reg), TII);
13492
13493 if (!IsSignExtended) {
13494 Register ValueReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
13495 BuildMI(BB&: *BB, I&: MI, MIMD: MI.getDebugLoc(),
13496 MCID: TII->get(Opcode: IsByte ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueReg)
13497 .addReg(RegNo: Reg);
13498 MI.getOperand(i: OpIdx).setReg(ValueReg);
13499 }
13500}
13501
13502MachineBasicBlock *PPCTargetLowering::EmitPartwordAtomicBinary(
13503 MachineInstr &MI, MachineBasicBlock *BB, unsigned BinOpcode,
13504 unsigned CmpOpcode, unsigned CmpPred) const {
13505 // BinOpcode != 0: Handles atomic load with binary operator, e.g. NAND.
13506 // CmpOpcode != 0: Handles atomic load with MIN/MAX etc.
13507 // BinOpcode == 0 && CmpOpcode == 0: Handles ATOMIC_SWAP.
13508 assert(!Subtarget.hasPartwordAtomics() &&
13509 "Assumes that part-word atomics are not available");
13510 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
13511
13512 // If this is a signed comparison and the value being compared is not known
13513 // to be sign extended, sign extend it here.
13514 DebugLoc dl = MI.getDebugLoc();
13515 MachineFunction *F = BB->getParent();
13516 MachineRegisterInfo &RegInfo = F->getRegInfo();
13517 const bool is8bit = MI.getOperand(i: 3).getImm() == 1;
13518 if (CmpOpcode == PPC::CMPW)
13519 signExtendOperandIfUnknown(MI, BB, OpIdx: 4, IsByte: is8bit, TII);
13520 Register incr = MI.getOperand(i: 4).getReg();
13521
13522 // In 64 bit mode we have to use 64 bits for addresses, even though the
13523 // lwarx/stwcx are 32 bits. With the 32-bit atomics we can use address
13524 // registers without caring whether they're 32 or 64, but here we're
13525 // doing actual arithmetic on the addresses.
13526 bool is64bit = Subtarget.isPPC64();
13527 bool isLittleEndian = Subtarget.isLittleEndian();
13528 unsigned ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
13529
13530 const BasicBlock *LLVM_BB = BB->getBasicBlock();
13531 MachineFunction::iterator It = ++BB->getIterator();
13532
13533 Register dest = MI.getOperand(i: 0).getReg();
13534 Register ptrA = MI.getOperand(i: 1).getReg();
13535 Register ptrB = MI.getOperand(i: 2).getReg();
13536
13537 MachineBasicBlock *loopMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13538 MachineBasicBlock *loop2MBB =
13539 CmpOpcode ? F->CreateMachineBasicBlock(BB: LLVM_BB) : nullptr;
13540 MachineBasicBlock *exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
13541 F->insert(MBBI: It, MBB: loopMBB);
13542 if (CmpOpcode)
13543 F->insert(MBBI: It, MBB: loop2MBB);
13544 F->insert(MBBI: It, MBB: exitMBB);
13545 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
13546 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
13547 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
13548
13549 const TargetRegisterClass *RC =
13550 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13551 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
13552
13553 Register PtrReg = RegInfo.createVirtualRegister(RegClass: RC);
13554 Register Shift1Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13555 Register ShiftReg =
13556 isLittleEndian ? Shift1Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13557 Register Incr2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13558 Register MaskReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13559 Register Mask2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13560 Register Mask3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13561 Register Tmp2Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13562 Register Tmp3Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13563 Register Tmp4Reg = RegInfo.createVirtualRegister(RegClass: GPRC);
13564 Register TmpDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13565 Register SrwDestReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13566 Register Ptr1Reg;
13567 Register TmpReg =
13568 (!BinOpcode) ? Incr2Reg : RegInfo.createVirtualRegister(RegClass: GPRC);
13569
13570 // thisMBB:
13571 // ...
13572 // fallthrough --> loopMBB
13573 BB->addSuccessor(Succ: loopMBB);
13574
13575 // The 4-byte load must be aligned, while a char or short may be
13576 // anywhere in the word. Hence all this nasty bookkeeping code.
13577 // add ptr1, ptrA, ptrB [copy if ptrA==0]
13578 // rlwinm shift1, ptr1, 3, 27, 28 [3, 27, 27]
13579 // xori shift, shift1, 24 [16]
13580 // rlwinm ptr, ptr1, 0, 0, 29
13581 // slw incr2, incr, shift
13582 // li mask2, 255 [li mask3, 0; ori mask2, mask3, 65535]
13583 // slw mask, mask2, shift
13584 // loopMBB:
13585 // lwarx tmpDest, ptr
13586 // add tmp, tmpDest, incr2
13587 // andc tmp2, tmpDest, mask
13588 // and tmp3, tmp, mask
13589 // or tmp4, tmp3, tmp2
13590 // stwcx. tmp4, ptr
13591 // bne- loopMBB
13592 // fallthrough --> exitMBB
13593 // srw SrwDest, tmpDest, shift
13594 // rlwinm SrwDest, SrwDest, 0, 24 [16], 31
13595 if (ptrA != ZeroReg) {
13596 Ptr1Reg = RegInfo.createVirtualRegister(RegClass: RC);
13597 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
13598 .addReg(RegNo: ptrA)
13599 .addReg(RegNo: ptrB);
13600 } else {
13601 Ptr1Reg = ptrB;
13602 }
13603 // We need use 32-bit subregister to avoid mismatch register class in 64-bit
13604 // mode.
13605 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
13606 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
13607 .addImm(Val: 3)
13608 .addImm(Val: 27)
13609 .addImm(Val: is8bit ? 28 : 27);
13610 if (!isLittleEndian)
13611 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
13612 .addReg(RegNo: Shift1Reg)
13613 .addImm(Val: is8bit ? 24 : 16);
13614 if (is64bit)
13615 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
13616 .addReg(RegNo: Ptr1Reg)
13617 .addImm(Val: 0)
13618 .addImm(Val: 61);
13619 else
13620 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
13621 .addReg(RegNo: Ptr1Reg)
13622 .addImm(Val: 0)
13623 .addImm(Val: 0)
13624 .addImm(Val: 29);
13625 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: Incr2Reg).addReg(RegNo: incr).addReg(RegNo: ShiftReg);
13626 if (is8bit)
13627 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
13628 else {
13629 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
13630 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
13631 .addReg(RegNo: Mask3Reg)
13632 .addImm(Val: 65535);
13633 }
13634 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
13635 .addReg(RegNo: Mask2Reg)
13636 .addReg(RegNo: ShiftReg);
13637
13638 BB = loopMBB;
13639 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
13640 .addReg(RegNo: ZeroReg)
13641 .addReg(RegNo: PtrReg);
13642 if (BinOpcode)
13643 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: BinOpcode), DestReg: TmpReg)
13644 .addReg(RegNo: Incr2Reg)
13645 .addReg(RegNo: TmpDestReg);
13646 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
13647 .addReg(RegNo: TmpDestReg)
13648 .addReg(RegNo: MaskReg);
13649 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: Tmp3Reg).addReg(RegNo: TmpReg).addReg(RegNo: MaskReg);
13650 if (CmpOpcode) {
13651 // For unsigned comparisons, we can directly compare the shifted values.
13652 // For signed comparisons we shift and sign extend.
13653 Register SReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13654 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
13655 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: SReg)
13656 .addReg(RegNo: TmpDestReg)
13657 .addReg(RegNo: MaskReg);
13658 unsigned ValueReg = SReg;
13659 unsigned CmpReg = Incr2Reg;
13660 if (CmpOpcode == PPC::CMPW) {
13661 ValueReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13662 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: ValueReg)
13663 .addReg(RegNo: SReg)
13664 .addReg(RegNo: ShiftReg);
13665 Register ValueSReg = RegInfo.createVirtualRegister(RegClass: GPRC);
13666 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is8bit ? PPC::EXTSB : PPC::EXTSH), DestReg: ValueSReg)
13667 .addReg(RegNo: ValueReg);
13668 ValueReg = ValueSReg;
13669 CmpReg = incr;
13670 }
13671 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: CmpOpcode), DestReg: CrReg).addReg(RegNo: ValueReg).addReg(RegNo: CmpReg);
13672 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13673 .addImm(Val: CmpPred)
13674 .addReg(RegNo: CrReg)
13675 .addMBB(MBB: exitMBB);
13676 BB->addSuccessor(Succ: loop2MBB);
13677 BB->addSuccessor(Succ: exitMBB);
13678 BB = loop2MBB;
13679 }
13680 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg).addReg(RegNo: Tmp3Reg).addReg(RegNo: Tmp2Reg);
13681 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
13682 .addReg(RegNo: Tmp4Reg)
13683 .addReg(RegNo: ZeroReg)
13684 .addReg(RegNo: PtrReg);
13685 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
13686 .addImm(Val: PPC::PRED_NE_MINUS)
13687 .addReg(RegNo: PPC::CR0)
13688 .addMBB(MBB: loopMBB);
13689 BB->addSuccessor(Succ: loopMBB);
13690 BB->addSuccessor(Succ: exitMBB);
13691
13692 // exitMBB:
13693 // ...
13694 BB = exitMBB;
13695 // Since the shift amount is not a constant, we need to clear
13696 // the upper bits with a separate RLWINM.
13697 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: dest)
13698 .addReg(RegNo: SrwDestReg)
13699 .addImm(Val: 0)
13700 .addImm(Val: is8bit ? 24 : 16)
13701 .addImm(Val: 31);
13702 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: SrwDestReg)
13703 .addReg(RegNo: TmpDestReg)
13704 .addReg(RegNo: ShiftReg);
13705 return BB;
13706}
13707
13708llvm::MachineBasicBlock *
13709PPCTargetLowering::emitEHSjLjSetJmp(MachineInstr &MI,
13710 MachineBasicBlock *MBB) const {
13711 DebugLoc DL = MI.getDebugLoc();
13712 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13713 const PPCRegisterInfo *TRI = Subtarget.getRegisterInfo();
13714
13715 MachineFunction *MF = MBB->getParent();
13716 MachineRegisterInfo &MRI = MF->getRegInfo();
13717
13718 const BasicBlock *BB = MBB->getBasicBlock();
13719 MachineFunction::iterator I = ++MBB->getIterator();
13720
13721 Register DstReg = MI.getOperand(i: 0).getReg();
13722 const TargetRegisterClass *RC = MRI.getRegClass(Reg: DstReg);
13723 assert(TRI->isTypeLegalForClass(*RC, MVT::i32) && "Invalid destination!");
13724 Register mainDstReg = MRI.createVirtualRegister(RegClass: RC);
13725 Register restoreDstReg = MRI.createVirtualRegister(RegClass: RC);
13726
13727 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13728 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13729 "Invalid Pointer Size!");
13730 // For v = setjmp(buf), we generate
13731 //
13732 // thisMBB:
13733 // SjLjSetup mainMBB
13734 // bl mainMBB
13735 // v_restore = 1
13736 // b sinkMBB
13737 //
13738 // mainMBB:
13739 // buf[LabelOffset] = LR
13740 // v_main = 0
13741 //
13742 // sinkMBB:
13743 // v = phi(main, restore)
13744 //
13745
13746 MachineBasicBlock *thisMBB = MBB;
13747 MachineBasicBlock *mainMBB = MF->CreateMachineBasicBlock(BB);
13748 MachineBasicBlock *sinkMBB = MF->CreateMachineBasicBlock(BB);
13749 MF->insert(MBBI: I, MBB: mainMBB);
13750 MF->insert(MBBI: I, MBB: sinkMBB);
13751
13752 MachineInstrBuilder MIB;
13753
13754 // Transfer the remainder of BB and its successor edges to sinkMBB.
13755 sinkMBB->splice(Where: sinkMBB->begin(), Other: MBB,
13756 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
13757 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
13758
13759 // Note that the structure of the jmp_buf used here is not compatible
13760 // with that used by libc, and is not designed to be. Specifically, it
13761 // stores only those 'reserved' registers that LLVM does not otherwise
13762 // understand how to spill. Also, by convention, by the time this
13763 // intrinsic is called, Clang has already stored the frame address in the
13764 // first slot of the buffer and stack address in the third. Following the
13765 // X86 target code, we'll store the jump address in the second slot. We also
13766 // need to save the TOC pointer (R2) to handle jumps between shared
13767 // libraries, and that will be stored in the fourth slot. The thread
13768 // identifier (R13) is not affected.
13769
13770 // thisMBB:
13771 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13772 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13773 const int64_t BPOffset = 4 * PVT.getStoreSize();
13774
13775 // Prepare IP either in reg.
13776 const TargetRegisterClass *PtrRC = getRegClassFor(VT: PVT);
13777 Register LabelReg = MRI.createVirtualRegister(RegClass: PtrRC);
13778 Register BufReg = MI.getOperand(i: 1).getReg();
13779
13780 if (Subtarget.is64BitELFABI()) {
13781 setUsesTOCBasePtr(*MBB->getParent());
13782 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13783 .addReg(RegNo: PPC::X2)
13784 .addImm(Val: TOCOffset)
13785 .addReg(RegNo: BufReg)
13786 .cloneMemRefs(OtherMI: MI);
13787 }
13788
13789 // Naked functions never have a base pointer, and so we use r1. For all
13790 // other functions, this decision must be delayed until during PEI.
13791 unsigned BaseReg;
13792 if (MF->getFunction().hasFnAttribute(Kind: Attribute::Naked))
13793 BaseReg = Subtarget.isPPC64() ? PPC::X1 : PPC::R1;
13794 else
13795 BaseReg = Subtarget.isPPC64() ? PPC::BP8 : PPC::BP;
13796
13797 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL,
13798 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::STD : PPC::STW))
13799 .addReg(RegNo: BaseReg)
13800 .addImm(Val: BPOffset)
13801 .addReg(RegNo: BufReg)
13802 .cloneMemRefs(OtherMI: MI);
13803
13804 // Setup
13805 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::BCLalways)).addMBB(MBB: mainMBB);
13806 MIB.addRegMask(Mask: TRI->getNoPreservedMask());
13807
13808 BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: restoreDstReg).addImm(Val: 1);
13809
13810 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::EH_SjLj_Setup))
13811 .addMBB(MBB: mainMBB);
13812 MIB = BuildMI(BB&: *thisMBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: sinkMBB);
13813
13814 thisMBB->addSuccessor(Succ: mainMBB, Prob: BranchProbability::getZero());
13815 thisMBB->addSuccessor(Succ: sinkMBB, Prob: BranchProbability::getOne());
13816
13817 // mainMBB:
13818 // mainDstReg = 0
13819 MIB =
13820 BuildMI(BB: mainMBB, MIMD: DL,
13821 MCID: TII->get(Opcode: Subtarget.isPPC64() ? PPC::MFLR8 : PPC::MFLR), DestReg: LabelReg);
13822
13823 // Store IP
13824 if (Subtarget.isPPC64()) {
13825 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STD))
13826 .addReg(RegNo: LabelReg)
13827 .addImm(Val: LabelOffset)
13828 .addReg(RegNo: BufReg);
13829 } else {
13830 MIB = BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::STW))
13831 .addReg(RegNo: LabelReg)
13832 .addImm(Val: LabelOffset)
13833 .addReg(RegNo: BufReg);
13834 }
13835 MIB.cloneMemRefs(OtherMI: MI);
13836
13837 BuildMI(BB: mainMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::LI), DestReg: mainDstReg).addImm(Val: 0);
13838 mainMBB->addSuccessor(Succ: sinkMBB);
13839
13840 // sinkMBB:
13841 BuildMI(BB&: *sinkMBB, I: sinkMBB->begin(), MIMD: DL,
13842 MCID: TII->get(Opcode: PPC::PHI), DestReg: DstReg)
13843 .addReg(RegNo: mainDstReg).addMBB(MBB: mainMBB)
13844 .addReg(RegNo: restoreDstReg).addMBB(MBB: thisMBB);
13845
13846 MI.eraseFromParent();
13847 return sinkMBB;
13848}
13849
13850MachineBasicBlock *
13851PPCTargetLowering::emitEHSjLjLongJmp(MachineInstr &MI,
13852 MachineBasicBlock *MBB) const {
13853 DebugLoc DL = MI.getDebugLoc();
13854 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13855
13856 MachineFunction *MF = MBB->getParent();
13857 MachineRegisterInfo &MRI = MF->getRegInfo();
13858
13859 MVT PVT = getPointerTy(DL: MF->getDataLayout());
13860 assert((PVT == MVT::i64 || PVT == MVT::i32) &&
13861 "Invalid Pointer Size!");
13862
13863 const TargetRegisterClass *RC =
13864 (PVT == MVT::i64) ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
13865 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
13866 // Since FP is only updated here but NOT referenced, it's treated as GPR.
13867 unsigned FP = (PVT == MVT::i64) ? PPC::X31 : PPC::R31;
13868 unsigned SP = (PVT == MVT::i64) ? PPC::X1 : PPC::R1;
13869 unsigned BP =
13870 (PVT == MVT::i64)
13871 ? PPC::X30
13872 : (Subtarget.isSVR4ABI() && isPositionIndependent() ? PPC::R29
13873 : PPC::R30);
13874
13875 MachineInstrBuilder MIB;
13876
13877 const int64_t LabelOffset = 1 * PVT.getStoreSize();
13878 const int64_t SPOffset = 2 * PVT.getStoreSize();
13879 const int64_t TOCOffset = 3 * PVT.getStoreSize();
13880 const int64_t BPOffset = 4 * PVT.getStoreSize();
13881
13882 Register BufReg = MI.getOperand(i: 0).getReg();
13883
13884 // Reload FP (the jumped-to function may not have had a
13885 // frame pointer, and if so, then its r31 will be restored
13886 // as necessary).
13887 if (PVT == MVT::i64) {
13888 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: FP)
13889 .addImm(Val: 0)
13890 .addReg(RegNo: BufReg);
13891 } else {
13892 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: FP)
13893 .addImm(Val: 0)
13894 .addReg(RegNo: BufReg);
13895 }
13896 MIB.cloneMemRefs(OtherMI: MI);
13897
13898 // Reload IP
13899 if (PVT == MVT::i64) {
13900 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: Tmp)
13901 .addImm(Val: LabelOffset)
13902 .addReg(RegNo: BufReg);
13903 } else {
13904 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: Tmp)
13905 .addImm(Val: LabelOffset)
13906 .addReg(RegNo: BufReg);
13907 }
13908 MIB.cloneMemRefs(OtherMI: MI);
13909
13910 // Reload SP
13911 if (PVT == MVT::i64) {
13912 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: SP)
13913 .addImm(Val: SPOffset)
13914 .addReg(RegNo: BufReg);
13915 } else {
13916 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: SP)
13917 .addImm(Val: SPOffset)
13918 .addReg(RegNo: BufReg);
13919 }
13920 MIB.cloneMemRefs(OtherMI: MI);
13921
13922 // Reload BP
13923 if (PVT == MVT::i64) {
13924 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: BP)
13925 .addImm(Val: BPOffset)
13926 .addReg(RegNo: BufReg);
13927 } else {
13928 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LWZ), DestReg: BP)
13929 .addImm(Val: BPOffset)
13930 .addReg(RegNo: BufReg);
13931 }
13932 MIB.cloneMemRefs(OtherMI: MI);
13933
13934 // Reload TOC
13935 if (PVT == MVT::i64 && Subtarget.isSVR4ABI()) {
13936 setUsesTOCBasePtr(*MBB->getParent());
13937 MIB = BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::LD), DestReg: PPC::X2)
13938 .addImm(Val: TOCOffset)
13939 .addReg(RegNo: BufReg)
13940 .cloneMemRefs(OtherMI: MI);
13941 }
13942
13943 // Jump
13944 BuildMI(BB&: *MBB, I&: MI, MIMD: DL,
13945 MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::MTCTR8 : PPC::MTCTR)).addReg(RegNo: Tmp);
13946 BuildMI(BB&: *MBB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PVT == MVT::i64 ? PPC::BCTR8 : PPC::BCTR));
13947
13948 MI.eraseFromParent();
13949 return MBB;
13950}
13951
13952bool PPCTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
13953 // If the function specifically requests inline stack probes, emit them.
13954 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
13955 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
13956 "inline-asm";
13957 return false;
13958}
13959
13960unsigned PPCTargetLowering::getStackProbeSize(const MachineFunction &MF) const {
13961 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
13962 unsigned StackAlign = TFI->getStackAlignment();
13963 assert(StackAlign >= 1 && isPowerOf2_32(StackAlign) &&
13964 "Unexpected stack alignment");
13965 // The default stack probe size is 4096 if the function has no
13966 // stack-probe-size attribute.
13967 const Function &Fn = MF.getFunction();
13968 unsigned StackProbeSize =
13969 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
13970 // Round down to the stack alignment.
13971 StackProbeSize &= ~(StackAlign - 1);
13972 return StackProbeSize ? StackProbeSize : StackAlign;
13973}
13974
13975// Lower dynamic stack allocation with probing. `emitProbedAlloca` is splitted
13976// into three phases. In the first phase, it uses pseudo instruction
13977// PREPARE_PROBED_ALLOCA to get the future result of actual FramePointer and
13978// FinalStackPtr. In the second phase, it generates a loop for probing blocks.
13979// At last, it uses pseudo instruction DYNAREAOFFSET to get the future result of
13980// MaxCallFrameSize so that it can calculate correct data area pointer.
13981MachineBasicBlock *
13982PPCTargetLowering::emitProbedAlloca(MachineInstr &MI,
13983 MachineBasicBlock *MBB) const {
13984 const bool isPPC64 = Subtarget.isPPC64();
13985 MachineFunction *MF = MBB->getParent();
13986 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
13987 DebugLoc DL = MI.getDebugLoc();
13988 const unsigned ProbeSize = getStackProbeSize(MF: *MF);
13989 const BasicBlock *ProbedBB = MBB->getBasicBlock();
13990 MachineRegisterInfo &MRI = MF->getRegInfo();
13991 // The CFG of probing stack looks as
13992 // +-----+
13993 // | MBB |
13994 // +--+--+
13995 // |
13996 // +----v----+
13997 // +--->+ TestMBB +---+
13998 // | +----+----+ |
13999 // | | |
14000 // | +-----v----+ |
14001 // +---+ BlockMBB | |
14002 // +----------+ |
14003 // |
14004 // +---------+ |
14005 // | TailMBB +<--+
14006 // +---------+
14007 // In MBB, calculate previous frame pointer and final stack pointer.
14008 // In TestMBB, test if sp is equal to final stack pointer, if so, jump to
14009 // TailMBB. In BlockMBB, update the sp atomically and jump back to TestMBB.
14010 // TailMBB is spliced via \p MI.
14011 MachineBasicBlock *TestMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14012 MachineBasicBlock *TailMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14013 MachineBasicBlock *BlockMBB = MF->CreateMachineBasicBlock(BB: ProbedBB);
14014
14015 MachineFunction::iterator MBBIter = ++MBB->getIterator();
14016 MF->insert(MBBI: MBBIter, MBB: TestMBB);
14017 MF->insert(MBBI: MBBIter, MBB: BlockMBB);
14018 MF->insert(MBBI: MBBIter, MBB: TailMBB);
14019
14020 const TargetRegisterClass *G8RC = &PPC::G8RCRegClass;
14021 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
14022
14023 Register DstReg = MI.getOperand(i: 0).getReg();
14024 Register NegSizeReg = MI.getOperand(i: 1).getReg();
14025 Register SPReg = isPPC64 ? PPC::X1 : PPC::R1;
14026 Register FinalStackPtr = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14027 Register FramePointer = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14028 Register ActualNegSizeReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14029
14030 // Since value of NegSizeReg might be realigned in prologepilog, insert a
14031 // PREPARE_PROBED_ALLOCA pseudo instruction to get actual FramePointer and
14032 // NegSize.
14033 unsigned ProbeOpc;
14034 if (!MRI.hasOneNonDBGUse(RegNo: NegSizeReg))
14035 ProbeOpc =
14036 isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_64 : PPC::PREPARE_PROBED_ALLOCA_32;
14037 else
14038 // By introducing PREPARE_PROBED_ALLOCA_NEGSIZE_OPT, ActualNegSizeReg
14039 // and NegSizeReg will be allocated in the same phyreg to avoid
14040 // redundant copy when NegSizeReg has only one use which is current MI and
14041 // will be replaced by PREPARE_PROBED_ALLOCA then.
14042 ProbeOpc = isPPC64 ? PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_64
14043 : PPC::PREPARE_PROBED_ALLOCA_NEGSIZE_SAME_REG_32;
14044 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: ProbeOpc), DestReg: FramePointer)
14045 .addDef(RegNo: ActualNegSizeReg)
14046 .addReg(RegNo: NegSizeReg)
14047 .add(MO: MI.getOperand(i: 2))
14048 .add(MO: MI.getOperand(i: 3));
14049
14050 // Calculate final stack pointer, which equals to SP + ActualNegSize.
14051 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4),
14052 DestReg: FinalStackPtr)
14053 .addReg(RegNo: SPReg)
14054 .addReg(RegNo: ActualNegSizeReg);
14055
14056 // Materialize a scratch register for update.
14057 int64_t NegProbeSize = -(int64_t)ProbeSize;
14058 assert(isInt<32>(NegProbeSize) && "Unhandled probe size!");
14059 Register ScratchReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14060 if (!isInt<16>(x: NegProbeSize)) {
14061 Register TempReg = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14062 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LIS8 : PPC::LIS), DestReg: TempReg)
14063 .addImm(Val: NegProbeSize >> 16);
14064 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ORI8 : PPC::ORI),
14065 DestReg: ScratchReg)
14066 .addReg(RegNo: TempReg)
14067 .addImm(Val: NegProbeSize & 0xFFFF);
14068 } else
14069 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::LI8 : PPC::LI), DestReg: ScratchReg)
14070 .addImm(Val: NegProbeSize);
14071
14072 {
14073 // Probing leading residual part.
14074 Register Div = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14075 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::DIVD : PPC::DIVW), DestReg: Div)
14076 .addReg(RegNo: ActualNegSizeReg)
14077 .addReg(RegNo: ScratchReg);
14078 Register Mul = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14079 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::MULLD : PPC::MULLW), DestReg: Mul)
14080 .addReg(RegNo: Div)
14081 .addReg(RegNo: ScratchReg);
14082 Register NegMod = MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14083 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::SUBF8 : PPC::SUBF), DestReg: NegMod)
14084 .addReg(RegNo: Mul)
14085 .addReg(RegNo: ActualNegSizeReg);
14086 BuildMI(BB&: *MBB, I&: {MI}, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14087 .addReg(RegNo: FramePointer)
14088 .addReg(RegNo: SPReg)
14089 .addReg(RegNo: NegMod);
14090 }
14091
14092 {
14093 // Remaining part should be multiple of ProbeSize.
14094 Register CmpResult = MRI.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14095 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::CMPD : PPC::CMPW), DestReg: CmpResult)
14096 .addReg(RegNo: SPReg)
14097 .addReg(RegNo: FinalStackPtr);
14098 BuildMI(BB: TestMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::BCC))
14099 .addImm(Val: PPC::PRED_EQ)
14100 .addReg(RegNo: CmpResult)
14101 .addMBB(MBB: TailMBB);
14102 TestMBB->addSuccessor(Succ: BlockMBB);
14103 TestMBB->addSuccessor(Succ: TailMBB);
14104 }
14105
14106 {
14107 // Touch the block.
14108 // |P...|P...|P...
14109 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::STDUX : PPC::STWUX), DestReg: SPReg)
14110 .addReg(RegNo: FramePointer)
14111 .addReg(RegNo: SPReg)
14112 .addReg(RegNo: ScratchReg);
14113 BuildMI(BB: BlockMBB, MIMD: DL, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: TestMBB);
14114 BlockMBB->addSuccessor(Succ: TestMBB);
14115 }
14116
14117 // Calculation of MaxCallFrameSize is deferred to prologepilog, use
14118 // DYNAREAOFFSET pseudo instruction to get the future result.
14119 Register MaxCallFrameSizeReg =
14120 MRI.createVirtualRegister(RegClass: isPPC64 ? G8RC : GPRC);
14121 BuildMI(BB: TailMBB, MIMD: DL,
14122 MCID: TII->get(Opcode: isPPC64 ? PPC::DYNAREAOFFSET8 : PPC::DYNAREAOFFSET),
14123 DestReg: MaxCallFrameSizeReg)
14124 .add(MO: MI.getOperand(i: 2))
14125 .add(MO: MI.getOperand(i: 3));
14126 BuildMI(BB: TailMBB, MIMD: DL, MCID: TII->get(Opcode: isPPC64 ? PPC::ADD8 : PPC::ADD4), DestReg: DstReg)
14127 .addReg(RegNo: SPReg)
14128 .addReg(RegNo: MaxCallFrameSizeReg);
14129
14130 // Splice instructions after MI to TailMBB.
14131 TailMBB->splice(Where: TailMBB->end(), Other: MBB,
14132 From: std::next(x: MachineBasicBlock::iterator(MI)), To: MBB->end());
14133 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
14134 MBB->addSuccessor(Succ: TestMBB);
14135
14136 // Delete the pseudo instruction.
14137 MI.eraseFromParent();
14138
14139 ++NumDynamicAllocaProbed;
14140 return TailMBB;
14141}
14142
14143/// Check if the opcode is a SELECT or SELECT_CC variant.
14144/// @param Opcode The opcode to check
14145/// @param CheckOnlyCC If true, only return true for SELECT_CC variants;
14146/// if false, return true for both SELECT and SELECT_CC
14147static bool IsSelect(unsigned Opcode, bool CheckOnlyCC = false) {
14148 switch (Opcode) {
14149 // SELECT_CC variants - always return true
14150 case PPC::SELECT_CC_I4:
14151 case PPC::SELECT_CC_I8:
14152 case PPC::SELECT_CC_F4:
14153 case PPC::SELECT_CC_F8:
14154 case PPC::SELECT_CC_F16:
14155 case PPC::SELECT_CC_VRRC:
14156 case PPC::SELECT_CC_VSFRC:
14157 case PPC::SELECT_CC_VSSRC:
14158 case PPC::SELECT_CC_VSRC:
14159 case PPC::SELECT_CC_SPE4:
14160 case PPC::SELECT_CC_SPE:
14161 return true;
14162 // SELECT variants - only return true if CheckOnlyCC is false
14163 case PPC::SELECT_I4:
14164 case PPC::SELECT_I8:
14165 case PPC::SELECT_F4:
14166 case PPC::SELECT_F8:
14167 case PPC::SELECT_F16:
14168 case PPC::SELECT_SPE:
14169 case PPC::SELECT_SPE4:
14170 case PPC::SELECT_VRRC:
14171 case PPC::SELECT_VSFRC:
14172 case PPC::SELECT_VSSRC:
14173 case PPC::SELECT_VSRC:
14174 return !CheckOnlyCC; // true if checking all SELECTs, false if only CC
14175 default:
14176 return false;
14177 }
14178}
14179static bool IsSelectCC(unsigned Opcode) { return IsSelect(Opcode, CheckOnlyCC: true); }
14180
14181/// Emit SELECT instruction, using ISEL if available, otherwise use
14182/// branch-based control flow.
14183///
14184/// For targets with ISEL support (SELECT_CC_I4/I8, SELECT_I4/I8), this
14185/// generates a single ISEL instruction. Otherwise, it creates a
14186/// branch-based control flow pattern with PHI nodes.
14187static MachineBasicBlock *emitSelect(MachineInstr &MI, MachineBasicBlock *BB,
14188 const TargetInstrInfo *TII,
14189 const PPCSubtarget &Subtarget) {
14190 assert(IsSelect(MI.getOpcode()) && "Instruction must be a SELECT variant");
14191
14192 // Check if we can use ISEL for this SELECT
14193 if (Subtarget.hasISEL() &&
14194 (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14195 MI.getOpcode() == PPC::SELECT_CC_I8 ||
14196 MI.getOpcode() == PPC::SELECT_I4 || MI.getOpcode() == PPC::SELECT_I8)) {
14197 SmallVector<MachineOperand, 2> Cond;
14198 if (MI.getOpcode() == PPC::SELECT_CC_I4 ||
14199 MI.getOpcode() == PPC::SELECT_CC_I8)
14200 Cond.push_back(Elt: MI.getOperand(i: 4));
14201 else
14202 Cond.push_back(Elt: MachineOperand::CreateImm(Val: PPC::PRED_BIT_SET));
14203 Cond.push_back(Elt: MI.getOperand(i: 1));
14204
14205 DebugLoc dl = MI.getDebugLoc();
14206 TII->insertSelect(MBB&: *BB, I: MI, DL: dl, DstReg: MI.getOperand(i: 0).getReg(), Cond,
14207 TrueReg: MI.getOperand(i: 2).getReg(), FalseReg: MI.getOperand(i: 3).getReg());
14208 MI.eraseFromParent();
14209 return BB;
14210 }
14211
14212 // Fall back to branch-based SELECT implementation
14213 MachineFunction *F = BB->getParent();
14214 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14215 MachineFunction::iterator It = ++BB->getIterator();
14216 DebugLoc dl = MI.getDebugLoc();
14217
14218 MachineBasicBlock *thisMBB = BB;
14219 MachineBasicBlock *copy0MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14220 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14221 F->insert(MBBI: It, MBB: copy0MBB);
14222 F->insert(MBBI: It, MBB: sinkMBB);
14223
14224 if (isPhysRegUsedAfter(Reg: PPC::CARRY, MBI: MI.getIterator())) {
14225 copy0MBB->addLiveIn(PhysReg: PPC::CARRY);
14226 sinkMBB->addLiveIn(PhysReg: PPC::CARRY);
14227 }
14228
14229 // Set the call frame size on entry to the new basic blocks.
14230 unsigned CallFrameSize = TII->getCallFrameSizeAt(MI);
14231 copy0MBB->setCallFrameSize(CallFrameSize);
14232 sinkMBB->setCallFrameSize(CallFrameSize);
14233
14234 // Transfer the remainder of BB and its successor edges to sinkMBB.
14235 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14236 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14237 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14238
14239 // Add successors
14240 BB->addSuccessor(Succ: copy0MBB);
14241 BB->addSuccessor(Succ: sinkMBB);
14242
14243 // Build branch instruction
14244 if (IsSelectCC(Opcode: MI.getOpcode()))
14245 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14246 .addImm(Val: MI.getOperand(i: 4).getImm())
14247 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14248 .addMBB(MBB: sinkMBB);
14249 else
14250 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BC))
14251 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14252 .addMBB(MBB: sinkMBB);
14253
14254 // copy0MBB: fallthrough to sinkMBB
14255 BB = copy0MBB;
14256 BB->addSuccessor(Succ: sinkMBB);
14257
14258 // sinkMBB: PHI instruction
14259 BB = sinkMBB;
14260 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::PHI), DestReg: MI.getOperand(i: 0).getReg())
14261 .addReg(RegNo: MI.getOperand(i: 3).getReg())
14262 .addMBB(MBB: copy0MBB)
14263 .addReg(RegNo: MI.getOperand(i: 2).getReg())
14264 .addMBB(MBB: thisMBB);
14265 MI.eraseFromParent();
14266 return BB;
14267}
14268
14269/// Helper function to create basic blocks for atomic compare-and-swap.
14270/// Creates three basic blocks (loop1MBB, loop2MBB, exitMBB) and sets up
14271/// the control flow structure common to both hardware and software
14272/// implementations of atomic compare-and-swap operations.
14273static void createAtomicLoopBlocks(MachineFunction *F, MachineBasicBlock *BB,
14274 MachineBasicBlock *&loop1MBB,
14275 MachineBasicBlock *&loop2MBB,
14276 MachineBasicBlock *&exitMBB,
14277 MachineInstr &MI,
14278 MachineFunction::iterator It) {
14279 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14280 loop1MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14281 loop2MBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14282 exitMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14283 F->insert(MBBI: It, MBB: loop1MBB);
14284 F->insert(MBBI: It, MBB: loop2MBB);
14285 F->insert(MBBI: It, MBB: exitMBB);
14286 exitMBB->splice(Where: exitMBB->begin(), Other: BB,
14287 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14288 exitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14289 BB->addSuccessor(Succ: loop1MBB);
14290}
14291
14292/// Emit hardware-supported atomic compare-and-swap for I32/I64 and I8/I16
14293/// with partword atomic support.
14294///
14295/// This uses native PowerPC atomic instructions (LBARX/LHARX/LWARX/LDARX for
14296/// load-and-reserve, STBCX/STHCX/STWCX/STDCX for store-conditional) to
14297/// implement atomic compare-and-swap at byte, halfword, word, or doubleword
14298/// granularity.
14299///
14300/// Control flow:
14301/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14302/// | |
14303/// +------------+
14304///
14305/// loop1MBB:
14306/// - Load-and-reserve from memory
14307/// - Compare loaded value with expected old value
14308/// - Branch to exitMBB if not equal (CAS failed)
14309/// loop2MBB:
14310/// - Store-conditional new value to memory
14311/// - Branch back to loop1MBB if store failed (retry)
14312/// - Fall through to exitMBB on success
14313static MachineBasicBlock *
14314emitAtomicCmpSwapHardware(MachineInstr &MI, MachineBasicBlock *BB,
14315 const TargetInstrInfo *TII,
14316 const PPCSubtarget &Subtarget) {
14317 MachineFunction *F = BB->getParent();
14318 MachineFunction::iterator It = ++BB->getIterator();
14319
14320 bool is64bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64;
14321
14322 unsigned LoadMnemonic = PPC::LDARX;
14323 unsigned StoreMnemonic = PPC::STDCX;
14324 switch (MI.getOpcode()) {
14325 default:
14326 llvm_unreachable("Compare and swap of unknown size");
14327 case PPC::ATOMIC_CMP_SWAP_I8:
14328 LoadMnemonic = PPC::LBARX;
14329 StoreMnemonic = PPC::STBCX;
14330 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14331 break;
14332 case PPC::ATOMIC_CMP_SWAP_I16:
14333 LoadMnemonic = PPC::LHARX;
14334 StoreMnemonic = PPC::STHCX;
14335 assert(Subtarget.hasPartwordAtomics() && "No support partword atomics.");
14336 break;
14337 case PPC::ATOMIC_CMP_SWAP_I32:
14338 LoadMnemonic = PPC::LWARX;
14339 StoreMnemonic = PPC::STWCX;
14340 break;
14341 case PPC::ATOMIC_CMP_SWAP_I64:
14342 LoadMnemonic = PPC::LDARX;
14343 StoreMnemonic = PPC::STDCX;
14344 break;
14345 }
14346
14347 MachineRegisterInfo &RegInfo = F->getRegInfo();
14348 Register dest = MI.getOperand(i: 0).getReg();
14349 Register ptrA = MI.getOperand(i: 1).getReg();
14350 Register ptrB = MI.getOperand(i: 2).getReg();
14351 Register oldval = MI.getOperand(i: 3).getReg();
14352 Register newval = MI.getOperand(i: 4).getReg();
14353 DebugLoc dl = MI.getDebugLoc();
14354
14355 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14356 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14357
14358 Register CrReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14359
14360 // loop1MBB:
14361 // l[bhwd]arx dest, ptr
14362 // cmp[wd] dest, oldval
14363 // bne- exitBB
14364 BB = loop1MBB;
14365 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: LoadMnemonic), DestReg: dest).addReg(RegNo: ptrA).addReg(RegNo: ptrB);
14366 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::CMPD : PPC::CMPW), DestReg: CrReg)
14367 .addReg(RegNo: dest)
14368 .addReg(RegNo: oldval);
14369 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14370 .addImm(Val: PPC::PRED_NE_MINUS)
14371 .addReg(RegNo: CrReg)
14372 .addMBB(MBB: exitMBB);
14373 BB->addSuccessor(Succ: loop2MBB);
14374 BB->addSuccessor(Succ: exitMBB);
14375
14376 // loop2MBB:
14377 // st[bhwd]cx. newval, ptr
14378 // bne- loopMBB
14379 // b exitBB
14380 BB = loop2MBB;
14381 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: StoreMnemonic))
14382 .addReg(RegNo: newval)
14383 .addReg(RegNo: ptrA)
14384 .addReg(RegNo: ptrB);
14385 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14386 .addImm(Val: PPC::PRED_NE_MINUS)
14387 .addReg(RegNo: PPC::CR0)
14388 .addMBB(MBB: loop1MBB);
14389 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14390 BB->addSuccessor(Succ: loop1MBB);
14391 BB->addSuccessor(Succ: exitMBB);
14392
14393 return exitMBB;
14394}
14395
14396/// Emit software-emulated atomic compare-and-swap for I8/I16 without
14397/// hardware partword atomic support.
14398///
14399/// This emulates byte/halfword atomic operations using word (32-bit) atomic
14400/// instructions. Since PowerPC atomic instructions work at word granularity,
14401/// we must:
14402/// 1. Align the pointer to a word boundary
14403/// 2. Calculate the bit shift for the target byte/halfword within the word
14404/// 3. Create masks to isolate the target byte/halfword
14405/// 4. Shift old/new values into the correct bit position
14406/// 5. Use LWARX/STWCX on the full word
14407/// 6. Mask and merge to preserve other bytes in the word
14408/// 7. Extract and shift the result back
14409///
14410/// Control flow:
14411/// thisMBB -> loop1MBB -> loop2MBB -> exitMBB
14412/// | |
14413/// +------------+
14414///
14415/// loop1MBB:
14416/// - LWARX: Load-and-reserve full word
14417/// - Mask to extract target byte/halfword
14418/// - Compare with expected old value
14419/// - Branch to exitMBB if not equal (CAS failed)
14420/// loop2MBB:
14421/// - Merge new value with other bytes in the word
14422/// - STWCX: Store-conditional full word
14423/// - Branch back to loop1MBB if store failed (retry)
14424/// - Fall through to exitMBB on success
14425/// exitMBB:
14426/// - Extract and return the loaded value
14427static MachineBasicBlock *
14428emitAtomicCmpSwapSoftware(MachineInstr &MI, MachineBasicBlock *BB,
14429 const TargetInstrInfo *TII,
14430 const PPCSubtarget &Subtarget) {
14431 MachineFunction *F = BB->getParent();
14432 MachineFunction::iterator It = ++BB->getIterator();
14433
14434 bool is64bit = Subtarget.isPPC64();
14435 bool isLittleEndian = Subtarget.isLittleEndian();
14436 bool is8bit = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8;
14437
14438 Register dest = MI.getOperand(i: 0).getReg();
14439 Register ptrA = MI.getOperand(i: 1).getReg();
14440 Register ptrB = MI.getOperand(i: 2).getReg();
14441 Register oldval = MI.getOperand(i: 3).getReg();
14442 Register newval = MI.getOperand(i: 4).getReg();
14443 DebugLoc dl = MI.getDebugLoc();
14444
14445 MachineBasicBlock *loop1MBB, *loop2MBB, *exitMBB;
14446 createAtomicLoopBlocks(F, BB, loop1MBB, loop2MBB, exitMBB, MI, It);
14447
14448 MachineRegisterInfo &RegInfo = F->getRegInfo();
14449 const TargetRegisterClass *RC =
14450 is64bit ? &PPC::G8RCRegClass : &PPC::GPRCRegClass;
14451 const TargetRegisterClass *GPRC = &PPC::GPRCRegClass;
14452
14453 // Lambda to create virtual registers
14454 auto createVReg = [&](const TargetRegisterClass *RC) {
14455 return RegInfo.createVirtualRegister(RegClass: RC);
14456 };
14457
14458 Register PtrReg = createVReg(RC);
14459 Register Shift1Reg = createVReg(GPRC);
14460 Register ShiftReg = isLittleEndian ? Shift1Reg : createVReg(GPRC);
14461 Register NewVal2Reg = createVReg(GPRC);
14462 Register NewVal3Reg = createVReg(GPRC);
14463 Register OldVal2Reg = createVReg(GPRC);
14464 Register OldVal3Reg = createVReg(GPRC);
14465 Register MaskReg = createVReg(GPRC);
14466 Register Mask2Reg = createVReg(GPRC);
14467 Register Mask3Reg = createVReg(GPRC);
14468 Register Tmp2Reg = createVReg(GPRC);
14469 Register Tmp4Reg = createVReg(GPRC);
14470 Register TmpDestReg = createVReg(GPRC);
14471 Register TmpReg = createVReg(GPRC);
14472 Register ZeroReg = is64bit ? PPC::ZERO8 : PPC::ZERO;
14473 Register CrReg = createVReg(&PPC::CRRCRegClass);
14474
14475 // Compute aligned pointer and shift amount
14476 Register Ptr1Reg;
14477 if (ptrA != ZeroReg) {
14478 Ptr1Reg = createVReg(RC);
14479 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: is64bit ? PPC::ADD8 : PPC::ADD4), DestReg: Ptr1Reg)
14480 .addReg(RegNo: ptrA)
14481 .addReg(RegNo: ptrB);
14482 } else {
14483 Ptr1Reg = ptrB;
14484 }
14485
14486 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: Shift1Reg)
14487 .addReg(RegNo: Ptr1Reg, Flags: {}, SubReg: is64bit ? PPC::sub_32 : 0)
14488 .addImm(Val: 3)
14489 .addImm(Val: 27)
14490 .addImm(Val: is8bit ? 28 : 27);
14491 if (!isLittleEndian)
14492 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::XORI), DestReg: ShiftReg)
14493 .addReg(RegNo: Shift1Reg)
14494 .addImm(Val: is8bit ? 24 : 16);
14495 if (is64bit)
14496 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDICR), DestReg: PtrReg)
14497 .addReg(RegNo: Ptr1Reg)
14498 .addImm(Val: 0)
14499 .addImm(Val: 61);
14500 else
14501 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::RLWINM), DestReg: PtrReg)
14502 .addReg(RegNo: Ptr1Reg)
14503 .addImm(Val: 0)
14504 .addImm(Val: 0)
14505 .addImm(Val: 29);
14506
14507 // Prepare masked values
14508 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: NewVal2Reg)
14509 .addReg(RegNo: newval)
14510 .addReg(RegNo: ShiftReg);
14511 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: OldVal2Reg)
14512 .addReg(RegNo: oldval)
14513 .addReg(RegNo: ShiftReg);
14514 if (is8bit)
14515 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask2Reg).addImm(Val: 255);
14516 else {
14517 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LI), DestReg: Mask3Reg).addImm(Val: 0);
14518 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ORI), DestReg: Mask2Reg)
14519 .addReg(RegNo: Mask3Reg)
14520 .addImm(Val: 65535);
14521 }
14522 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::SLW), DestReg: MaskReg)
14523 .addReg(RegNo: Mask2Reg)
14524 .addReg(RegNo: ShiftReg);
14525 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: NewVal3Reg)
14526 .addReg(RegNo: NewVal2Reg)
14527 .addReg(RegNo: MaskReg);
14528 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: OldVal3Reg)
14529 .addReg(RegNo: OldVal2Reg)
14530 .addReg(RegNo: MaskReg);
14531
14532 // loop1MBB:
14533 // lwarx tmpDest, ptr
14534 // and tmp, tmpDest, mask
14535 // cmpw tmp, oldval3
14536 // bne- exitBB
14537 BB = loop1MBB;
14538 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::LWARX), DestReg: TmpDestReg)
14539 .addReg(RegNo: ZeroReg)
14540 .addReg(RegNo: PtrReg);
14541 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::AND), DestReg: TmpReg)
14542 .addReg(RegNo: TmpDestReg)
14543 .addReg(RegNo: MaskReg);
14544 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CrReg).addReg(RegNo: TmpReg).addReg(RegNo: OldVal3Reg);
14545 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14546 .addImm(Val: PPC::PRED_NE)
14547 .addReg(RegNo: CrReg)
14548 .addMBB(MBB: exitMBB);
14549 BB->addSuccessor(Succ: loop2MBB);
14550 BB->addSuccessor(Succ: exitMBB);
14551
14552 // loop2MBB:
14553 // andc tmp2, tmpDest, mask
14554 // or tmp4, tmp2, newval3
14555 // stwcx. tmp4, ptr
14556 // bne- loop1MBB
14557 // b exitBB
14558 BB = loop2MBB;
14559 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::ANDC), DestReg: Tmp2Reg)
14560 .addReg(RegNo: TmpDestReg)
14561 .addReg(RegNo: MaskReg);
14562 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::OR), DestReg: Tmp4Reg)
14563 .addReg(RegNo: Tmp2Reg)
14564 .addReg(RegNo: NewVal3Reg);
14565 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::STWCX))
14566 .addReg(RegNo: Tmp4Reg)
14567 .addReg(RegNo: ZeroReg)
14568 .addReg(RegNo: PtrReg);
14569 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14570 .addImm(Val: PPC::PRED_NE)
14571 .addReg(RegNo: PPC::CR0)
14572 .addMBB(MBB: loop1MBB);
14573 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::B)).addMBB(MBB: exitMBB);
14574 BB->addSuccessor(Succ: loop1MBB);
14575 BB->addSuccessor(Succ: exitMBB);
14576
14577 // exitMBB:
14578 // srw dest, tmpDest, shift
14579 BB = exitMBB;
14580 BuildMI(BB&: *BB, I: BB->begin(), MIMD: dl, MCID: TII->get(Opcode: PPC::SRW), DestReg: dest)
14581 .addReg(RegNo: TmpReg)
14582 .addReg(RegNo: ShiftReg);
14583
14584 return BB;
14585}
14586
14587MachineBasicBlock *
14588PPCTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
14589 MachineBasicBlock *BB) const {
14590 const TargetInstrInfo *TII = Subtarget.getInstrInfo();
14591
14592 // To "insert" these instructions we actually have to insert their
14593 // control-flow patterns.
14594 const BasicBlock *LLVM_BB = BB->getBasicBlock();
14595 MachineFunction::iterator It = ++BB->getIterator();
14596
14597 MachineFunction *F = BB->getParent();
14598 MachineRegisterInfo &MRI = F->getRegInfo();
14599
14600 // Handle SELECT with ISEL support first (before generic SELECT handling)
14601 if (IsSelect(Opcode: MI.getOpcode()))
14602 return emitSelect(MI, BB, TII, Subtarget);
14603
14604 switch (MI.getOpcode()) {
14605 case TargetOpcode::STACKMAP:
14606 return emitPatchPoint(MI, MBB: BB);
14607 case TargetOpcode::PATCHPOINT:
14608 // Call lowering should have added an r2 operand to indicate a dependence
14609 // on the TOC base pointer value. It can't however, because there is no
14610 // way to mark the dependence as implicit there, and so the stackmap code
14611 // will confuse it with a regular operand. Instead, add the dependence
14612 // here.
14613 if (Subtarget.is64BitELFABI() && !Subtarget.isUsingPCRelativeCalls())
14614 MI.addOperand(Op: MachineOperand::CreateReg(Reg: PPC::X2, isDef: false, isImp: true));
14615 return emitPatchPoint(MI, MBB: BB);
14616
14617 case PPC::EH_SjLj_SetJmp32:
14618 case PPC::EH_SjLj_SetJmp64:
14619 return emitEHSjLjSetJmp(MI, MBB: BB);
14620
14621 case PPC::EH_SjLj_LongJmp32:
14622 case PPC::EH_SjLj_LongJmp64:
14623 return emitEHSjLjLongJmp(MI, MBB: BB);
14624
14625 case PPC::ReadTB: {
14626 // To read the 64-bit time-base register on a 32-bit target, we read the
14627 // two halves. Should the counter have wrapped while it was being read, we
14628 // need to try again.
14629 // ...
14630 // readLoop:
14631 // mfspr Rx,TBU # load from TBU
14632 // mfspr Ry,TB # load from TB
14633 // mfspr Rz,TBU # load from TBU
14634 // cmpw crX,Rx,Rz # check if 'old'='new'
14635 // bne readLoop # branch if they're not equal
14636 // ...
14637
14638 MachineBasicBlock *readMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14639 MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
14640 DebugLoc dl = MI.getDebugLoc();
14641 F->insert(MBBI: It, MBB: readMBB);
14642 F->insert(MBBI: It, MBB: sinkMBB);
14643
14644 // Transfer the remainder of BB and its successor edges to sinkMBB.
14645 sinkMBB->splice(Where: sinkMBB->begin(), Other: BB,
14646 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
14647 sinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
14648
14649 BB->addSuccessor(Succ: readMBB);
14650 BB = readMBB;
14651
14652 MachineRegisterInfo &RegInfo = F->getRegInfo();
14653 Register ReadAgainReg = RegInfo.createVirtualRegister(RegClass: &PPC::GPRCRegClass);
14654 Register LoReg = MI.getOperand(i: 0).getReg();
14655 Register HiReg = MI.getOperand(i: 1).getReg();
14656
14657 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: HiReg).addImm(Val: 269);
14658 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: LoReg).addImm(Val: 268);
14659 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::MFSPR), DestReg: ReadAgainReg).addImm(Val: 269);
14660
14661 Register CmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14662
14663 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::CMPW), DestReg: CmpReg)
14664 .addReg(RegNo: HiReg)
14665 .addReg(RegNo: ReadAgainReg);
14666 BuildMI(BB, MIMD: dl, MCID: TII->get(Opcode: PPC::BCC))
14667 .addImm(Val: PPC::PRED_NE)
14668 .addReg(RegNo: CmpReg)
14669 .addMBB(MBB: readMBB);
14670
14671 BB->addSuccessor(Succ: readMBB);
14672 BB->addSuccessor(Succ: sinkMBB);
14673 break;
14674 }
14675 case PPC::ATOMIC_LOAD_ADD_NOWP:
14676 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14677 break;
14678 case PPC::ATOMIC_LOAD_ADD:
14679 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD4);
14680 break;
14681 case PPC::ATOMIC_LOAD_ADD_I64:
14682 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::ADD8);
14683 break;
14684 case PPC::ATOMIC_LOAD_AND_NOWP:
14685 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14686 break;
14687 case PPC::ATOMIC_LOAD_AND:
14688 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND);
14689 break;
14690 case PPC::ATOMIC_LOAD_AND_I64:
14691 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::AND8);
14692 break;
14693 case PPC::ATOMIC_LOAD_OR_NOWP:
14694 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14695 break;
14696 case PPC::ATOMIC_LOAD_OR:
14697 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR);
14698 break;
14699 case PPC::ATOMIC_LOAD_OR_I64:
14700 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::OR8);
14701 break;
14702 case PPC::ATOMIC_LOAD_XOR_NOWP:
14703 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14704 break;
14705 case PPC::ATOMIC_LOAD_XOR:
14706 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR);
14707 break;
14708 case PPC::ATOMIC_LOAD_XOR_I64:
14709 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::XOR8);
14710 break;
14711 case PPC::ATOMIC_LOAD_NAND_NOWP:
14712 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14713 break;
14714 case PPC::ATOMIC_LOAD_NAND:
14715 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND);
14716 break;
14717 case PPC::ATOMIC_LOAD_NAND_I64:
14718 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::NAND8);
14719 break;
14720 case PPC::ATOMIC_LOAD_SUB_NOWP:
14721 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14722 break;
14723 case PPC::ATOMIC_LOAD_SUB:
14724 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF);
14725 break;
14726 case PPC::ATOMIC_LOAD_SUB_I64:
14727 BB = EmitAtomicBinary(MI, BB, BinOpcode: PPC::SUBF8);
14728 break;
14729 case PPC::ATOMIC_LOAD_MIN_NOWP:
14730 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14731 break;
14732 case PPC::ATOMIC_LOAD_MIN:
14733 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_LT);
14734 break;
14735 case PPC::ATOMIC_LOAD_MIN_I64:
14736 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_LT);
14737 break;
14738 case PPC::ATOMIC_LOAD_MAX_NOWP:
14739 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14740 break;
14741 case PPC::ATOMIC_LOAD_MAX:
14742 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPW, CmpPred: PPC::PRED_GT);
14743 break;
14744 case PPC::ATOMIC_LOAD_MAX_I64:
14745 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPD, CmpPred: PPC::PRED_GT);
14746 break;
14747 case PPC::ATOMIC_LOAD_UMIN_NOWP:
14748 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14749 break;
14750 case PPC::ATOMIC_LOAD_UMIN:
14751 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_LT);
14752 break;
14753 case PPC::ATOMIC_LOAD_UMIN_I64:
14754 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_LT);
14755 break;
14756 case PPC::ATOMIC_LOAD_UMAX_NOWP:
14757 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14758 break;
14759 case PPC::ATOMIC_LOAD_UMAX:
14760 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLW, CmpPred: PPC::PRED_GT);
14761 break;
14762 case PPC::ATOMIC_LOAD_UMAX_I64:
14763 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0, CmpOpcode: PPC::CMPLD, CmpPred: PPC::PRED_GT);
14764 break;
14765 case PPC::ATOMIC_SWAP_NOWP:
14766 BB = EmitPartwordAtomicBinary(MI, BB, BinOpcode: 0);
14767 break;
14768 case PPC::ATOMIC_SWAP:
14769 case PPC::ATOMIC_SWAP_I64:
14770 BB = EmitAtomicBinary(MI, BB, BinOpcode: 0);
14771 break;
14772 case PPC::ATOMIC_CMP_SWAP_I32:
14773 case PPC::ATOMIC_CMP_SWAP_I64:
14774 case PPC::ATOMIC_CMP_SWAP_I8:
14775 case PPC::ATOMIC_CMP_SWAP_I16: {
14776 // Use hardware-supported atomic operations if available
14777 bool useHardware = MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I32 ||
14778 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I64 ||
14779 (Subtarget.hasPartwordAtomics() &&
14780 (MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I8 ||
14781 MI.getOpcode() == PPC::ATOMIC_CMP_SWAP_I16));
14782
14783 if (useHardware)
14784 BB = emitAtomicCmpSwapHardware(MI, BB, TII, Subtarget);
14785 else
14786 BB = emitAtomicCmpSwapSoftware(MI, BB, TII, Subtarget);
14787 break;
14788 }
14789 case PPC::FADDrtz: {
14790 // This pseudo performs an FADD with rounding mode temporarily forced
14791 // to round-to-zero. We emit this via custom inserter since the FPSCR
14792 // is not modeled at the SelectionDAG level.
14793 Register Dest = MI.getOperand(i: 0).getReg();
14794 Register Src1 = MI.getOperand(i: 1).getReg();
14795 Register Src2 = MI.getOperand(i: 2).getReg();
14796 DebugLoc dl = MI.getDebugLoc();
14797
14798 MachineRegisterInfo &RegInfo = F->getRegInfo();
14799 Register MFFSReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
14800
14801 // Save FPSCR value.
14802 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: MFFSReg);
14803
14804 // Set rounding mode to round-to-zero.
14805 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB1))
14806 .addImm(Val: 31)
14807 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14808
14809 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSB0))
14810 .addImm(Val: 30)
14811 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14812
14813 // Perform addition.
14814 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::FADD), DestReg: Dest)
14815 .addReg(RegNo: Src1)
14816 .addReg(RegNo: Src2);
14817 if (MI.getFlag(Flag: MachineInstr::NoFPExcept))
14818 MIB.setMIFlag(MachineInstr::NoFPExcept);
14819
14820 // Restore FPSCR value.
14821 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSFb)).addImm(Val: 1).addReg(RegNo: MFFSReg);
14822 break;
14823 }
14824 case PPC::ANDI_rec_1_EQ_BIT:
14825 case PPC::ANDI_rec_1_GT_BIT:
14826 case PPC::ANDI_rec_1_EQ_BIT8:
14827 case PPC::ANDI_rec_1_GT_BIT8: {
14828 unsigned Opcode = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8 ||
14829 MI.getOpcode() == PPC::ANDI_rec_1_GT_BIT8)
14830 ? PPC::ANDI8_rec
14831 : PPC::ANDI_rec;
14832 bool IsEQ = (MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT ||
14833 MI.getOpcode() == PPC::ANDI_rec_1_EQ_BIT8);
14834
14835 MachineRegisterInfo &RegInfo = F->getRegInfo();
14836 Register Dest = RegInfo.createVirtualRegister(
14837 RegClass: Opcode == PPC::ANDI_rec ? &PPC::GPRCRegClass : &PPC::G8RCRegClass);
14838
14839 DebugLoc Dl = MI.getDebugLoc();
14840 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode), DestReg: Dest)
14841 .addReg(RegNo: MI.getOperand(i: 1).getReg())
14842 .addImm(Val: 1);
14843 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14844 DestReg: MI.getOperand(i: 0).getReg())
14845 .addReg(RegNo: IsEQ ? PPC::CR0EQ : PPC::CR0GT);
14846 break;
14847 }
14848 case PPC::TCHECK_RET: {
14849 DebugLoc Dl = MI.getDebugLoc();
14850 MachineRegisterInfo &RegInfo = F->getRegInfo();
14851 Register CRReg = RegInfo.createVirtualRegister(RegClass: &PPC::CRRCRegClass);
14852 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TCHECK), DestReg: CRReg);
14853 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14854 DestReg: MI.getOperand(i: 0).getReg())
14855 .addReg(RegNo: CRReg);
14856 break;
14857 }
14858 case PPC::TBEGIN_RET: {
14859 DebugLoc Dl = MI.getDebugLoc();
14860 unsigned Imm = MI.getOperand(i: 1).getImm();
14861 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::TBEGIN)).addImm(Val: Imm);
14862 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::COPY),
14863 DestReg: MI.getOperand(i: 0).getReg())
14864 .addReg(RegNo: PPC::CR0EQ);
14865 break;
14866 }
14867 case PPC::SETRNDi: {
14868 DebugLoc dl = MI.getDebugLoc();
14869 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14870
14871 // Save FPSCR value.
14872 if (MRI.use_empty(RegNo: OldFPSCRReg))
14873 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
14874 else
14875 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14876
14877 // The floating point rounding mode is in the bits 62:63 of FPCSR, and has
14878 // the following settings:
14879 // 00 Round to nearest
14880 // 01 Round to 0
14881 // 10 Round to +inf
14882 // 11 Round to -inf
14883
14884 // When the operand is immediate, using the two least significant bits of
14885 // the immediate to set the bits 62:63 of FPSCR.
14886 unsigned Mode = MI.getOperand(i: 1).getImm();
14887 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 1) ? PPC::MTFSB1 : PPC::MTFSB0))
14888 .addImm(Val: 31)
14889 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14890
14891 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: (Mode & 2) ? PPC::MTFSB1 : PPC::MTFSB0))
14892 .addImm(Val: 30)
14893 .addReg(RegNo: PPC::RM, Flags: RegState::ImplicitDefine);
14894 break;
14895 }
14896 case PPC::SETRND: {
14897 DebugLoc dl = MI.getDebugLoc();
14898
14899 // Copy register from F8RCRegClass::SrcReg to G8RCRegClass::DestReg
14900 // or copy register from G8RCRegClass::SrcReg to F8RCRegClass::DestReg.
14901 // If the target doesn't have DirectMove, we should use stack to do the
14902 // conversion, because the target doesn't have the instructions like mtvsrd
14903 // or mfvsrd to do this conversion directly.
14904 auto copyRegFromG8RCOrF8RC = [&] (unsigned DestReg, unsigned SrcReg) {
14905 if (Subtarget.hasDirectMove()) {
14906 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::COPY), DestReg)
14907 .addReg(RegNo: SrcReg);
14908 } else {
14909 // Use stack to do the register copy.
14910 unsigned StoreOp = PPC::STD, LoadOp = PPC::LFD;
14911 MachineRegisterInfo &RegInfo = F->getRegInfo();
14912 const TargetRegisterClass *RC = RegInfo.getRegClass(Reg: SrcReg);
14913 if (RC == &PPC::F8RCRegClass) {
14914 // Copy register from F8RCRegClass to G8RCRegclass.
14915 assert((RegInfo.getRegClass(DestReg) == &PPC::G8RCRegClass) &&
14916 "Unsupported RegClass.");
14917
14918 StoreOp = PPC::STFD;
14919 LoadOp = PPC::LD;
14920 } else {
14921 // Copy register from G8RCRegClass to F8RCRegclass.
14922 assert((RegInfo.getRegClass(SrcReg) == &PPC::G8RCRegClass) &&
14923 (RegInfo.getRegClass(DestReg) == &PPC::F8RCRegClass) &&
14924 "Unsupported RegClass.");
14925 }
14926
14927 MachineFrameInfo &MFI = F->getFrameInfo();
14928 int FrameIdx = MFI.CreateStackObject(Size: 8, Alignment: Align(8), isSpillSlot: false);
14929
14930 MachineMemOperand *MMOStore = F->getMachineMemOperand(
14931 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14932 F: MachineMemOperand::MOStore, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14933 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14934
14935 // Store the SrcReg into the stack.
14936 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: StoreOp))
14937 .addReg(RegNo: SrcReg)
14938 .addImm(Val: 0)
14939 .addFrameIndex(Idx: FrameIdx)
14940 .addMemOperand(MMO: MMOStore);
14941
14942 MachineMemOperand *MMOLoad = F->getMachineMemOperand(
14943 PtrInfo: MachinePointerInfo::getFixedStack(MF&: *F, FI: FrameIdx, Offset: 0),
14944 F: MachineMemOperand::MOLoad, Size: MFI.getObjectSize(ObjectIdx: FrameIdx),
14945 BaseAlignment: MFI.getObjectAlign(ObjectIdx: FrameIdx));
14946
14947 // Load from the stack where SrcReg is stored, and save to DestReg,
14948 // so we have done the RegClass conversion from RegClass::SrcReg to
14949 // RegClass::DestReg.
14950 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: LoadOp), DestReg)
14951 .addImm(Val: 0)
14952 .addFrameIndex(Idx: FrameIdx)
14953 .addMemOperand(MMO: MMOLoad);
14954 }
14955 };
14956
14957 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
14958
14959 // Save FPSCR value.
14960 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
14961
14962 // When the operand is gprc register, use two least significant bits of the
14963 // register and mtfsf instruction to set the bits 62:63 of FPSCR.
14964 //
14965 // copy OldFPSCRTmpReg, OldFPSCRReg
14966 // (INSERT_SUBREG ExtSrcReg, (IMPLICIT_DEF ImDefReg), SrcOp, 1)
14967 // rldimi NewFPSCRTmpReg, ExtSrcReg, OldFPSCRReg, 0, 62
14968 // copy NewFPSCRReg, NewFPSCRTmpReg
14969 // mtfsf 255, NewFPSCRReg
14970 MachineOperand SrcOp = MI.getOperand(i: 1);
14971 MachineRegisterInfo &RegInfo = F->getRegInfo();
14972 Register OldFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14973
14974 copyRegFromG8RCOrF8RC(OldFPSCRTmpReg, OldFPSCRReg);
14975
14976 Register ImDefReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14977 Register ExtSrcReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14978
14979 // The first operand of INSERT_SUBREG should be a register which has
14980 // subregisters, we only care about its RegClass, so we should use an
14981 // IMPLICIT_DEF register.
14982 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: ImDefReg);
14983 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::INSERT_SUBREG), DestReg: ExtSrcReg)
14984 .addReg(RegNo: ImDefReg)
14985 .add(MO: SrcOp)
14986 .addImm(Val: PPC::sub_32);
14987
14988 Register NewFPSCRTmpReg = RegInfo.createVirtualRegister(RegClass: &PPC::G8RCRegClass);
14989 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::RLDIMI), DestReg: NewFPSCRTmpReg)
14990 .addReg(RegNo: OldFPSCRTmpReg)
14991 .addReg(RegNo: ExtSrcReg)
14992 .addImm(Val: 0)
14993 .addImm(Val: 62);
14994
14995 Register NewFPSCRReg = RegInfo.createVirtualRegister(RegClass: &PPC::F8RCRegClass);
14996 copyRegFromG8RCOrF8RC(NewFPSCRReg, NewFPSCRTmpReg);
14997
14998 // The mask 255 means that put the 32:63 bits of NewFPSCRReg to the 32:63
14999 // bits of FPSCR.
15000 BuildMI(BB&: *BB, I&: MI, MIMD: dl, MCID: TII->get(Opcode: PPC::MTFSF))
15001 .addImm(Val: 255)
15002 .addReg(RegNo: NewFPSCRReg)
15003 .addImm(Val: 0)
15004 .addImm(Val: 0);
15005 break;
15006 }
15007 case PPC::SETFLM: {
15008 DebugLoc Dl = MI.getDebugLoc();
15009
15010 // Result of setflm is previous FPSCR content, so we need to save it first.
15011 Register OldFPSCRReg = MI.getOperand(i: 0).getReg();
15012 if (MRI.use_empty(RegNo: OldFPSCRReg))
15013 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: TargetOpcode::IMPLICIT_DEF), DestReg: OldFPSCRReg);
15014 else
15015 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MFFS), DestReg: OldFPSCRReg);
15016
15017 // Put bits in 32:63 to FPSCR.
15018 Register NewFPSCRReg = MI.getOperand(i: 1).getReg();
15019 BuildMI(BB&: *BB, I&: MI, MIMD: Dl, MCID: TII->get(Opcode: PPC::MTFSF))
15020 .addImm(Val: 255)
15021 .addReg(RegNo: NewFPSCRReg)
15022 .addImm(Val: 0)
15023 .addImm(Val: 0);
15024 break;
15025 }
15026 case PPC::PROBED_ALLOCA_32:
15027 case PPC::PROBED_ALLOCA_64:
15028 return emitProbedAlloca(MI, MBB: BB);
15029
15030 case PPC::SPLIT_QUADWORD: {
15031 DebugLoc DL = MI.getDebugLoc();
15032 Register Src = MI.getOperand(i: 2).getReg();
15033 Register Lo = MI.getOperand(i: 0).getReg();
15034 Register Hi = MI.getOperand(i: 1).getReg();
15035 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
15036 .addDef(RegNo: Lo)
15037 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x1);
15038 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: TargetOpcode::COPY))
15039 .addDef(RegNo: Hi)
15040 .addUse(RegNo: Src, Flags: {}, SubReg: PPC::sub_gp8_x0);
15041 break;
15042 }
15043 case PPC::LQX_PSEUDO:
15044 case PPC::STQX_PSEUDO: {
15045 DebugLoc DL = MI.getDebugLoc();
15046 // Ptr is used as the ptr_rc_no_r0 part
15047 // of LQ/STQ's memory operand and adding result of RA and RB,
15048 // so it has to be g8rc_and_g8rc_nox0.
15049 Register Ptr =
15050 F->getRegInfo().createVirtualRegister(RegClass: &PPC::G8RC_and_G8RC_NOX0RegClass);
15051 Register Val = MI.getOperand(i: 0).getReg();
15052 Register RA = MI.getOperand(i: 1).getReg();
15053 Register RB = MI.getOperand(i: 2).getReg();
15054 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII->get(Opcode: PPC::ADD8), DestReg: Ptr).addReg(RegNo: RA).addReg(RegNo: RB);
15055 BuildMI(BB&: *BB, I&: MI, MIMD: DL,
15056 MCID: MI.getOpcode() == PPC::LQX_PSEUDO ? TII->get(Opcode: PPC::LQ)
15057 : TII->get(Opcode: PPC::STQ))
15058 .addReg(RegNo: Val, Flags: getDefRegState(B: MI.getOpcode() == PPC::LQX_PSEUDO))
15059 .addImm(Val: 0)
15060 .addReg(RegNo: Ptr);
15061 break;
15062 }
15063 default:
15064 llvm_unreachable("Unexpected instr type to insert");
15065 }
15066
15067 MI.eraseFromParent(); // The pseudo instruction is gone now.
15068 return BB;
15069}
15070
15071//===----------------------------------------------------------------------===//
15072// Target Optimization Hooks
15073//===----------------------------------------------------------------------===//
15074
15075static int getEstimateRefinementSteps(EVT VT, const PPCSubtarget &Subtarget) {
15076 // For the estimates, convergence is quadratic, so we essentially double the
15077 // number of digits correct after every iteration. For both FRE and FRSQRTE,
15078 // the minimum architected relative accuracy is 2^-5. When hasRecipPrec(),
15079 // this is 2^-14. IEEE float has 23 digits and double has 52 digits.
15080 int RefinementSteps = Subtarget.hasRecipPrec() ? 1 : 3;
15081 if (VT.getScalarType() == MVT::f64)
15082 RefinementSteps++;
15083 return RefinementSteps;
15084}
15085
15086SDValue PPCTargetLowering::getSqrtInputTest(SDValue Op, SelectionDAG &DAG,
15087 const DenormalMode &Mode,
15088 SDNodeFlags Flags) const {
15089 // We only have VSX Vector Test for software Square Root.
15090 EVT VT = Op.getValueType();
15091 if (!isTypeLegal(VT: MVT::i1) ||
15092 (VT != MVT::f64 &&
15093 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX())))
15094 return TargetLowering::getSqrtInputTest(Operand: Op, DAG, Mode, Flags);
15095
15096 SDLoc DL(Op);
15097 // The output register of FTSQRT is CR field.
15098 SDValue FTSQRT = DAG.getNode(Opcode: PPCISD::FTSQRT, DL, VT: MVT::i32, Operand: Op, Flags);
15099 // ftsqrt BF,FRB
15100 // Let e_b be the unbiased exponent of the double-precision
15101 // floating-point operand in register FRB.
15102 // fe_flag is set to 1 if either of the following conditions occurs.
15103 // - The double-precision floating-point operand in register FRB is a zero,
15104 // a NaN, or an infinity, or a negative value.
15105 // - e_b is less than or equal to -970.
15106 // Otherwise fe_flag is set to 0.
15107 // Both VSX and non-VSX versions would set EQ bit in the CR if the number is
15108 // not eligible for iteration. (zero/negative/infinity/nan or unbiased
15109 // exponent is less than -970)
15110 SDValue SRIdxVal = DAG.getTargetConstant(Val: PPC::sub_eq, DL, VT: MVT::i32);
15111 return SDValue(DAG.getMachineNode(Opcode: TargetOpcode::EXTRACT_SUBREG, dl: DL, VT: MVT::i1,
15112 Op1: FTSQRT, Op2: SRIdxVal),
15113 0);
15114}
15115
15116SDValue
15117PPCTargetLowering::getSqrtResultForDenormInput(SDValue Op,
15118 SelectionDAG &DAG) const {
15119 // We only have VSX Vector Square Root.
15120 EVT VT = Op.getValueType();
15121 if (VT != MVT::f64 &&
15122 ((VT != MVT::v2f64 && VT != MVT::v4f32) || !Subtarget.hasVSX()))
15123 return TargetLowering::getSqrtResultForDenormInput(Operand: Op, DAG);
15124
15125 return DAG.getNode(Opcode: PPCISD::FSQRT, DL: SDLoc(Op), VT, Operand: Op);
15126}
15127
15128SDValue PPCTargetLowering::getSqrtEstimate(SDValue Operand, SelectionDAG &DAG,
15129 int Enabled, int &RefinementSteps,
15130 bool &UseOneConstNR,
15131 bool Reciprocal) const {
15132 EVT VT = Operand.getValueType();
15133 if ((VT == MVT::f32 && Subtarget.hasFRSQRTES()) ||
15134 (VT == MVT::f64 && Subtarget.hasFRSQRTE()) ||
15135 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15136 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15137 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15138 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15139
15140 // The Newton-Raphson computation with a single constant does not provide
15141 // enough accuracy on some CPUs.
15142 UseOneConstNR = !Subtarget.needsTwoConstNR();
15143 return DAG.getNode(Opcode: PPCISD::FRSQRTE, DL: SDLoc(Operand), VT, Operand);
15144 }
15145 return SDValue();
15146}
15147
15148SDValue PPCTargetLowering::getRecipEstimate(SDValue Operand, SelectionDAG &DAG,
15149 int Enabled,
15150 int &RefinementSteps) const {
15151 EVT VT = Operand.getValueType();
15152 if ((VT == MVT::f32 && Subtarget.hasFRES()) ||
15153 (VT == MVT::f64 && Subtarget.hasFRE()) ||
15154 (VT == MVT::v4f32 && Subtarget.hasAltivec()) ||
15155 (VT == MVT::v2f64 && Subtarget.hasVSX())) {
15156 if (RefinementSteps == ReciprocalEstimate::Unspecified)
15157 RefinementSteps = getEstimateRefinementSteps(VT, Subtarget);
15158 return DAG.getNode(Opcode: PPCISD::FRE, DL: SDLoc(Operand), VT, Operand);
15159 }
15160 return SDValue();
15161}
15162
15163unsigned PPCTargetLowering::combineRepeatedFPDivisors() const {
15164 // Note: This functionality is used only when arcp is enabled, and
15165 // on cores with reciprocal estimates (which are used when arcp is
15166 // enabled for division), this functionality is redundant with the default
15167 // combiner logic (once the division -> reciprocal/multiply transformation
15168 // has taken place). As a result, this matters more for older cores than for
15169 // newer ones.
15170
15171 // Combine multiple FDIVs with the same divisor into multiple FMULs by the
15172 // reciprocal if there are two or more FDIVs (for embedded cores with only
15173 // one FP pipeline) for three or more FDIVs (for generic OOO cores).
15174 switch (Subtarget.getCPUDirective()) {
15175 default:
15176 return 3;
15177 case PPC::DIR_440:
15178 case PPC::DIR_A2:
15179 case PPC::DIR_E500:
15180 case PPC::DIR_E500mc:
15181 case PPC::DIR_E5500:
15182 return 2;
15183 }
15184}
15185
15186// isConsecutiveLSLoc needs to work even if all adds have not yet been
15187// collapsed, and so we need to look through chains of them.
15188static void getBaseWithConstantOffset(SDValue Loc, SDValue &Base,
15189 int64_t& Offset, SelectionDAG &DAG) {
15190 if (DAG.isBaseWithConstantOffset(Op: Loc)) {
15191 Base = Loc.getOperand(i: 0);
15192 Offset += cast<ConstantSDNode>(Val: Loc.getOperand(i: 1))->getSExtValue();
15193
15194 // The base might itself be a base plus an offset, and if so, accumulate
15195 // that as well.
15196 getBaseWithConstantOffset(Loc: Loc.getOperand(i: 0), Base, Offset, DAG);
15197 }
15198}
15199
15200static bool isConsecutiveLSLoc(SDValue Loc, EVT VT, LSBaseSDNode *Base,
15201 unsigned Bytes, int Dist,
15202 SelectionDAG &DAG) {
15203 if (VT.getSizeInBits() / 8 != Bytes)
15204 return false;
15205
15206 SDValue BaseLoc = Base->getBasePtr();
15207 if (Loc.getOpcode() == ISD::FrameIndex) {
15208 if (BaseLoc.getOpcode() != ISD::FrameIndex)
15209 return false;
15210 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
15211 int FI = cast<FrameIndexSDNode>(Val&: Loc)->getIndex();
15212 int BFI = cast<FrameIndexSDNode>(Val&: BaseLoc)->getIndex();
15213 int FS = MFI.getObjectSize(ObjectIdx: FI);
15214 int BFS = MFI.getObjectSize(ObjectIdx: BFI);
15215 if (FS != BFS || FS != (int)Bytes) return false;
15216 return MFI.getObjectOffset(ObjectIdx: FI) == (MFI.getObjectOffset(ObjectIdx: BFI) + Dist*Bytes);
15217 }
15218
15219 SDValue Base1 = Loc, Base2 = BaseLoc;
15220 int64_t Offset1 = 0, Offset2 = 0;
15221 getBaseWithConstantOffset(Loc, Base&: Base1, Offset&: Offset1, DAG);
15222 getBaseWithConstantOffset(Loc: BaseLoc, Base&: Base2, Offset&: Offset2, DAG);
15223 if (Base1 == Base2 && Offset1 == (Offset2 + Dist * Bytes))
15224 return true;
15225
15226 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
15227 const GlobalValue *GV1 = nullptr;
15228 const GlobalValue *GV2 = nullptr;
15229 Offset1 = 0;
15230 Offset2 = 0;
15231 bool isGA1 = TLI.isGAPlusOffset(N: Loc.getNode(), GA&: GV1, Offset&: Offset1);
15232 bool isGA2 = TLI.isGAPlusOffset(N: BaseLoc.getNode(), GA&: GV2, Offset&: Offset2);
15233 if (isGA1 && isGA2 && GV1 == GV2)
15234 return Offset1 == (Offset2 + Dist*Bytes);
15235 return false;
15236}
15237
15238// Like SelectionDAG::isConsecutiveLoad, but also works for stores, and does
15239// not enforce equality of the chain operands.
15240static bool isConsecutiveLS(SDNode *N, LSBaseSDNode *Base,
15241 unsigned Bytes, int Dist,
15242 SelectionDAG &DAG) {
15243 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Val: N)) {
15244 EVT VT = LS->getMemoryVT();
15245 SDValue Loc = LS->getBasePtr();
15246 return isConsecutiveLSLoc(Loc, VT, Base, Bytes, Dist, DAG);
15247 }
15248
15249 if (N->getOpcode() == ISD::INTRINSIC_W_CHAIN) {
15250 EVT VT;
15251 switch (N->getConstantOperandVal(Num: 1)) {
15252 default: return false;
15253 case Intrinsic::ppc_altivec_lvx:
15254 case Intrinsic::ppc_altivec_lvxl:
15255 case Intrinsic::ppc_vsx_lxvw4x:
15256 case Intrinsic::ppc_vsx_lxvw4x_be:
15257 VT = MVT::v4i32;
15258 break;
15259 case Intrinsic::ppc_vsx_lxvd2x:
15260 case Intrinsic::ppc_vsx_lxvd2x_be:
15261 VT = MVT::v2f64;
15262 break;
15263 case Intrinsic::ppc_altivec_lvebx:
15264 VT = MVT::i8;
15265 break;
15266 case Intrinsic::ppc_altivec_lvehx:
15267 VT = MVT::i16;
15268 break;
15269 case Intrinsic::ppc_altivec_lvewx:
15270 VT = MVT::i32;
15271 break;
15272 }
15273
15274 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 2), VT, Base, Bytes, Dist, DAG);
15275 }
15276
15277 if (N->getOpcode() == ISD::INTRINSIC_VOID) {
15278 EVT VT;
15279 switch (N->getConstantOperandVal(Num: 1)) {
15280 default: return false;
15281 case Intrinsic::ppc_altivec_stvx:
15282 case Intrinsic::ppc_altivec_stvxl:
15283 case Intrinsic::ppc_vsx_stxvw4x:
15284 VT = MVT::v4i32;
15285 break;
15286 case Intrinsic::ppc_vsx_stxvd2x:
15287 VT = MVT::v2f64;
15288 break;
15289 case Intrinsic::ppc_vsx_stxvw4x_be:
15290 VT = MVT::v4i32;
15291 break;
15292 case Intrinsic::ppc_vsx_stxvd2x_be:
15293 VT = MVT::v2f64;
15294 break;
15295 case Intrinsic::ppc_altivec_stvebx:
15296 VT = MVT::i8;
15297 break;
15298 case Intrinsic::ppc_altivec_stvehx:
15299 VT = MVT::i16;
15300 break;
15301 case Intrinsic::ppc_altivec_stvewx:
15302 VT = MVT::i32;
15303 break;
15304 }
15305
15306 return isConsecutiveLSLoc(Loc: N->getOperand(Num: 3), VT, Base, Bytes, Dist, DAG);
15307 }
15308
15309 return false;
15310}
15311
15312// Return true is there is a nearyby consecutive load to the one provided
15313// (regardless of alignment). We search up and down the chain, looking though
15314// token factors and other loads (but nothing else). As a result, a true result
15315// indicates that it is safe to create a new consecutive load adjacent to the
15316// load provided.
15317static bool findConsecutiveLoad(LoadSDNode *LD, SelectionDAG &DAG) {
15318 SDValue Chain = LD->getChain();
15319 EVT VT = LD->getMemoryVT();
15320
15321 SmallPtrSet<SDNode *, 16> LoadRoots;
15322 SmallVector<SDNode *, 8> Queue(1, Chain.getNode());
15323 SmallPtrSet<SDNode *, 16> Visited;
15324
15325 // First, search up the chain, branching to follow all token-factor operands.
15326 // If we find a consecutive load, then we're done, otherwise, record all
15327 // nodes just above the top-level loads and token factors.
15328 while (!Queue.empty()) {
15329 SDNode *ChainNext = Queue.pop_back_val();
15330 if (!Visited.insert(Ptr: ChainNext).second)
15331 continue;
15332
15333 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: ChainNext)) {
15334 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15335 return true;
15336
15337 if (!Visited.count(Ptr: ChainLD->getChain().getNode()))
15338 Queue.push_back(Elt: ChainLD->getChain().getNode());
15339 } else if (ChainNext->getOpcode() == ISD::TokenFactor) {
15340 for (const SDUse &O : ChainNext->ops())
15341 if (!Visited.count(Ptr: O.getNode()))
15342 Queue.push_back(Elt: O.getNode());
15343 } else
15344 LoadRoots.insert(Ptr: ChainNext);
15345 }
15346
15347 // Second, search down the chain, starting from the top-level nodes recorded
15348 // in the first phase. These top-level nodes are the nodes just above all
15349 // loads and token factors. Starting with their uses, recursively look though
15350 // all loads (just the chain uses) and token factors to find a consecutive
15351 // load.
15352 Visited.clear();
15353 Queue.clear();
15354
15355 for (SDNode *I : LoadRoots) {
15356 Queue.push_back(Elt: I);
15357
15358 while (!Queue.empty()) {
15359 SDNode *LoadRoot = Queue.pop_back_val();
15360 if (!Visited.insert(Ptr: LoadRoot).second)
15361 continue;
15362
15363 if (MemSDNode *ChainLD = dyn_cast<MemSDNode>(Val: LoadRoot))
15364 if (isConsecutiveLS(N: ChainLD, Base: LD, Bytes: VT.getStoreSize(), Dist: 1, DAG))
15365 return true;
15366
15367 for (SDNode *U : LoadRoot->users())
15368 if (((isa<MemSDNode>(Val: U) &&
15369 cast<MemSDNode>(Val: U)->getChain().getNode() == LoadRoot) ||
15370 U->getOpcode() == ISD::TokenFactor) &&
15371 !Visited.count(Ptr: U))
15372 Queue.push_back(Elt: U);
15373 }
15374 }
15375
15376 return false;
15377}
15378
15379/// This function is called when we have proved that a SETCC node can be replaced
15380/// by subtraction (and other supporting instructions) so that the result of
15381/// comparison is kept in a GPR instead of CR. This function is purely for
15382/// codegen purposes and has some flags to guide the codegen process.
15383static SDValue generateEquivalentSub(SDNode *N, int Size, bool Complement,
15384 bool Swap, SDLoc &DL, SelectionDAG &DAG) {
15385 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15386
15387 // Zero extend the operands to the largest legal integer. Originally, they
15388 // must be of a strictly smaller size.
15389 auto Op0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15390 auto Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15391
15392 // Swap if needed. Depends on the condition code.
15393 if (Swap)
15394 std::swap(a&: Op0, b&: Op1);
15395
15396 // Subtract extended integers.
15397 auto SubNode = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: Op0, N2: Op1);
15398
15399 // Move the sign bit to the least significant position and zero out the rest.
15400 // Now the least significant bit carries the result of original comparison.
15401 auto Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: SubNode,
15402 N2: DAG.getConstant(Val: Size - 1, DL, VT: MVT::i32));
15403 auto Final = Shifted;
15404
15405 // Complement the result if needed. Based on the condition code.
15406 if (Complement)
15407 Final = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, N1: Shifted,
15408 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i64));
15409
15410 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Final);
15411}
15412
15413SDValue PPCTargetLowering::ConvertSETCCToSubtract(SDNode *N,
15414 DAGCombinerInfo &DCI) const {
15415 assert(N->getOpcode() == ISD::SETCC && "ISD::SETCC Expected.");
15416
15417 SelectionDAG &DAG = DCI.DAG;
15418 SDLoc DL(N);
15419
15420 // Size of integers being compared has a critical role in the following
15421 // analysis, so we prefer to do this when all types are legal.
15422 if (!DCI.isAfterLegalizeDAG())
15423 return SDValue();
15424
15425 // If all users of SETCC extend its value to a legal integer type
15426 // then we replace SETCC with a subtraction
15427 for (const SDNode *U : N->users())
15428 if (U->getOpcode() != ISD::ZERO_EXTEND)
15429 return SDValue();
15430
15431 ISD::CondCode CC = N->getOperandCondCode(Num: 2);
15432 auto OpSize = N->getOperand(Num: 0).getValueSizeInBits();
15433
15434 unsigned Size = DAG.getDataLayout().getLargestLegalIntTypeSizeInBits();
15435
15436 if (OpSize < Size) {
15437 switch (CC) {
15438 default: break;
15439 case ISD::SETULT:
15440 return generateEquivalentSub(N, Size, Complement: false, Swap: false, DL, DAG);
15441 case ISD::SETULE:
15442 return generateEquivalentSub(N, Size, Complement: true, Swap: true, DL, DAG);
15443 case ISD::SETUGT:
15444 return generateEquivalentSub(N, Size, Complement: false, Swap: true, DL, DAG);
15445 case ISD::SETUGE:
15446 return generateEquivalentSub(N, Size, Complement: true, Swap: false, DL, DAG);
15447 }
15448 }
15449
15450 return SDValue();
15451}
15452
15453SDValue PPCTargetLowering::DAGCombineTruncBoolExt(SDNode *N,
15454 DAGCombinerInfo &DCI) const {
15455 SelectionDAG &DAG = DCI.DAG;
15456 SDLoc dl(N);
15457
15458 assert(Subtarget.useCRBits() && "Expecting to be tracking CR bits");
15459 // If we're tracking CR bits, we need to be careful that we don't have:
15460 // trunc(binary-ops(zext(x), zext(y)))
15461 // or
15462 // trunc(binary-ops(binary-ops(zext(x), zext(y)), ...)
15463 // such that we're unnecessarily moving things into GPRs when it would be
15464 // better to keep them in CR bits.
15465
15466 // Note that trunc here can be an actual i1 trunc, or can be the effective
15467 // truncation that comes from a setcc or select_cc.
15468 if (N->getOpcode() == ISD::TRUNCATE &&
15469 N->getValueType(ResNo: 0) != MVT::i1)
15470 return SDValue();
15471
15472 if (N->getOperand(Num: 0).getValueType() != MVT::i32 &&
15473 N->getOperand(Num: 0).getValueType() != MVT::i64)
15474 return SDValue();
15475
15476 if (N->getOpcode() == ISD::SETCC ||
15477 N->getOpcode() == ISD::SELECT_CC) {
15478 // If we're looking at a comparison, then we need to make sure that the
15479 // high bits (all except for the first) don't matter the result.
15480 ISD::CondCode CC =
15481 N->getOperandCondCode(Num: N->getOpcode() == ISD::SETCC ? 2 : 4);
15482 unsigned OpBits = N->getOperand(Num: 0).getValueSizeInBits();
15483
15484 if (ISD::isSignedIntSetCC(Code: CC)) {
15485 if (DAG.ComputeNumSignBits(Op: N->getOperand(Num: 0)) != OpBits ||
15486 DAG.ComputeNumSignBits(Op: N->getOperand(Num: 1)) != OpBits)
15487 return SDValue();
15488 } else if (ISD::isUnsignedIntSetCC(Code: CC)) {
15489 if (!DAG.MaskedValueIsZero(Op: N->getOperand(Num: 0),
15490 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)) ||
15491 !DAG.MaskedValueIsZero(Op: N->getOperand(Num: 1),
15492 Mask: APInt::getHighBitsSet(numBits: OpBits, hiBitsSet: OpBits-1)))
15493 return (N->getOpcode() == ISD::SETCC ? ConvertSETCCToSubtract(N, DCI)
15494 : SDValue());
15495 } else {
15496 // This is neither a signed nor an unsigned comparison, just make sure
15497 // that the high bits are equal.
15498 KnownBits Op1Known = DAG.computeKnownBits(Op: N->getOperand(Num: 0));
15499 KnownBits Op2Known = DAG.computeKnownBits(Op: N->getOperand(Num: 1));
15500
15501 // We don't really care about what is known about the first bit (if
15502 // anything), so pretend that it is known zero for both to ensure they can
15503 // be compared as constants.
15504 Op1Known.Zero.setBit(0); Op1Known.One.clearBit(BitPosition: 0);
15505 Op2Known.Zero.setBit(0); Op2Known.One.clearBit(BitPosition: 0);
15506
15507 if (!Op1Known.isConstant() || !Op2Known.isConstant() ||
15508 Op1Known.getConstant() != Op2Known.getConstant())
15509 return SDValue();
15510 }
15511 }
15512
15513 // We now know that the higher-order bits are irrelevant, we just need to
15514 // make sure that all of the intermediate operations are bit operations, and
15515 // all inputs are extensions.
15516 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15517 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15518 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15519 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15520 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC &&
15521 N->getOperand(Num: 0).getOpcode() != ISD::TRUNCATE &&
15522 N->getOperand(Num: 0).getOpcode() != ISD::SIGN_EXTEND &&
15523 N->getOperand(Num: 0).getOpcode() != ISD::ZERO_EXTEND &&
15524 N->getOperand(Num: 0).getOpcode() != ISD::ANY_EXTEND)
15525 return SDValue();
15526
15527 if ((N->getOpcode() == ISD::SETCC || N->getOpcode() == ISD::SELECT_CC) &&
15528 N->getOperand(Num: 1).getOpcode() != ISD::AND &&
15529 N->getOperand(Num: 1).getOpcode() != ISD::OR &&
15530 N->getOperand(Num: 1).getOpcode() != ISD::XOR &&
15531 N->getOperand(Num: 1).getOpcode() != ISD::SELECT &&
15532 N->getOperand(Num: 1).getOpcode() != ISD::SELECT_CC &&
15533 N->getOperand(Num: 1).getOpcode() != ISD::TRUNCATE &&
15534 N->getOperand(Num: 1).getOpcode() != ISD::SIGN_EXTEND &&
15535 N->getOperand(Num: 1).getOpcode() != ISD::ZERO_EXTEND &&
15536 N->getOperand(Num: 1).getOpcode() != ISD::ANY_EXTEND)
15537 return SDValue();
15538
15539 SmallVector<SDValue, 4> Inputs;
15540 SmallVector<SDValue, 8> BinOps, PromOps;
15541 SmallPtrSet<SDNode *, 16> Visited;
15542
15543 for (unsigned i = 0; i < 2; ++i) {
15544 if (((N->getOperand(Num: i).getOpcode() == ISD::SIGN_EXTEND ||
15545 N->getOperand(Num: i).getOpcode() == ISD::ZERO_EXTEND ||
15546 N->getOperand(Num: i).getOpcode() == ISD::ANY_EXTEND) &&
15547 N->getOperand(Num: i).getOperand(i: 0).getValueType() == MVT::i1) ||
15548 isa<ConstantSDNode>(Val: N->getOperand(Num: i)))
15549 Inputs.push_back(Elt: N->getOperand(Num: i));
15550 else
15551 BinOps.push_back(Elt: N->getOperand(Num: i));
15552
15553 if (N->getOpcode() == ISD::TRUNCATE)
15554 break;
15555 }
15556
15557 // Visit all inputs, collect all binary operations (and, or, xor and
15558 // select) that are all fed by extensions.
15559 while (!BinOps.empty()) {
15560 SDValue BinOp = BinOps.pop_back_val();
15561
15562 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15563 continue;
15564
15565 PromOps.push_back(Elt: BinOp);
15566
15567 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15568 // The condition of the select is not promoted.
15569 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15570 continue;
15571 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15572 continue;
15573
15574 if (((BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15575 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15576 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) &&
15577 BinOp.getOperand(i).getOperand(i: 0).getValueType() == MVT::i1) ||
15578 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15579 Inputs.push_back(Elt: BinOp.getOperand(i));
15580 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15581 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15582 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15583 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15584 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC ||
15585 BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15586 BinOp.getOperand(i).getOpcode() == ISD::SIGN_EXTEND ||
15587 BinOp.getOperand(i).getOpcode() == ISD::ZERO_EXTEND ||
15588 BinOp.getOperand(i).getOpcode() == ISD::ANY_EXTEND) {
15589 BinOps.push_back(Elt: BinOp.getOperand(i));
15590 } else {
15591 // We have an input that is not an extension or another binary
15592 // operation; we'll abort this transformation.
15593 return SDValue();
15594 }
15595 }
15596 }
15597
15598 // Make sure that this is a self-contained cluster of operations (which
15599 // is not quite the same thing as saying that everything has only one
15600 // use).
15601 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15602 if (isa<ConstantSDNode>(Val: Inputs[i]))
15603 continue;
15604
15605 for (const SDNode *User : Inputs[i].getNode()->users()) {
15606 if (User != N && !Visited.count(Ptr: User))
15607 return SDValue();
15608
15609 // Make sure that we're not going to promote the non-output-value
15610 // operand(s) or SELECT or SELECT_CC.
15611 // FIXME: Although we could sometimes handle this, and it does occur in
15612 // practice that one of the condition inputs to the select is also one of
15613 // the outputs, we currently can't deal with this.
15614 if (User->getOpcode() == ISD::SELECT) {
15615 if (User->getOperand(Num: 0) == Inputs[i])
15616 return SDValue();
15617 } else if (User->getOpcode() == ISD::SELECT_CC) {
15618 if (User->getOperand(Num: 0) == Inputs[i] ||
15619 User->getOperand(Num: 1) == Inputs[i])
15620 return SDValue();
15621 }
15622 }
15623 }
15624
15625 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15626 for (const SDNode *User : PromOps[i].getNode()->users()) {
15627 if (User != N && !Visited.count(Ptr: User))
15628 return SDValue();
15629
15630 // Make sure that we're not going to promote the non-output-value
15631 // operand(s) or SELECT or SELECT_CC.
15632 // FIXME: Although we could sometimes handle this, and it does occur in
15633 // practice that one of the condition inputs to the select is also one of
15634 // the outputs, we currently can't deal with this.
15635 if (User->getOpcode() == ISD::SELECT) {
15636 if (User->getOperand(Num: 0) == PromOps[i])
15637 return SDValue();
15638 } else if (User->getOpcode() == ISD::SELECT_CC) {
15639 if (User->getOperand(Num: 0) == PromOps[i] ||
15640 User->getOperand(Num: 1) == PromOps[i])
15641 return SDValue();
15642 }
15643 }
15644 }
15645
15646 // Replace all inputs with the extension operand.
15647 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15648 // Constants may have users outside the cluster of to-be-promoted nodes,
15649 // and so we need to replace those as we do the promotions.
15650 if (isa<ConstantSDNode>(Val: Inputs[i]))
15651 continue;
15652 else
15653 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: Inputs[i].getOperand(i: 0));
15654 }
15655
15656 std::list<HandleSDNode> PromOpHandles;
15657 for (auto &PromOp : PromOps)
15658 PromOpHandles.emplace_back(args&: PromOp);
15659
15660 // Replace all operations (these are all the same, but have a different
15661 // (i1) return type). DAG.getNode will validate that the types of
15662 // a binary operator match, so go through the list in reverse so that
15663 // we've likely promoted both operands first. Any intermediate truncations or
15664 // extensions disappear.
15665 while (!PromOpHandles.empty()) {
15666 SDValue PromOp = PromOpHandles.back().getValue();
15667 PromOpHandles.pop_back();
15668
15669 if (PromOp.getOpcode() == ISD::TRUNCATE ||
15670 PromOp.getOpcode() == ISD::SIGN_EXTEND ||
15671 PromOp.getOpcode() == ISD::ZERO_EXTEND ||
15672 PromOp.getOpcode() == ISD::ANY_EXTEND) {
15673 if (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: 0)) &&
15674 PromOp.getOperand(i: 0).getValueType() != MVT::i1) {
15675 // The operand is not yet ready (see comment below).
15676 PromOpHandles.emplace_front(args&: PromOp);
15677 continue;
15678 }
15679
15680 SDValue RepValue = PromOp.getOperand(i: 0);
15681 if (isa<ConstantSDNode>(Val: RepValue))
15682 RepValue = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: RepValue);
15683
15684 DAG.ReplaceAllUsesOfValueWith(From: PromOp, To: RepValue);
15685 continue;
15686 }
15687
15688 unsigned C;
15689 switch (PromOp.getOpcode()) {
15690 default: C = 0; break;
15691 case ISD::SELECT: C = 1; break;
15692 case ISD::SELECT_CC: C = 2; break;
15693 }
15694
15695 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15696 PromOp.getOperand(i: C).getValueType() != MVT::i1) ||
15697 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15698 PromOp.getOperand(i: C+1).getValueType() != MVT::i1)) {
15699 // The to-be-promoted operands of this node have not yet been
15700 // promoted (this should be rare because we're going through the
15701 // list backward, but if one of the operands has several users in
15702 // this cluster of to-be-promoted nodes, it is possible).
15703 PromOpHandles.emplace_front(args&: PromOp);
15704 continue;
15705 }
15706
15707 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15708
15709 // If there are any constant inputs, make sure they're replaced now.
15710 for (unsigned i = 0; i < 2; ++i)
15711 if (isa<ConstantSDNode>(Val: Ops[C+i]))
15712 Ops[C+i] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i1, Operand: Ops[C+i]);
15713
15714 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
15715 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: MVT::i1, Ops));
15716 }
15717
15718 // Now we're left with the initial truncation itself.
15719 if (N->getOpcode() == ISD::TRUNCATE)
15720 return N->getOperand(Num: 0);
15721
15722 // Otherwise, this is a comparison. The operands to be compared have just
15723 // changed type (to i1), but everything else is the same.
15724 return SDValue(N, 0);
15725}
15726
15727SDValue PPCTargetLowering::DAGCombineExtBoolTrunc(SDNode *N,
15728 DAGCombinerInfo &DCI) const {
15729 SelectionDAG &DAG = DCI.DAG;
15730 SDLoc dl(N);
15731
15732 // If we're tracking CR bits, we need to be careful that we don't have:
15733 // zext(binary-ops(trunc(x), trunc(y)))
15734 // or
15735 // zext(binary-ops(binary-ops(trunc(x), trunc(y)), ...)
15736 // such that we're unnecessarily moving things into CR bits that can more
15737 // efficiently stay in GPRs. Note that if we're not certain that the high
15738 // bits are set as required by the final extension, we still may need to do
15739 // some masking to get the proper behavior.
15740
15741 // This same functionality is important on PPC64 when dealing with
15742 // 32-to-64-bit extensions; these occur often when 32-bit values are used as
15743 // the return values of functions. Because it is so similar, it is handled
15744 // here as well.
15745
15746 if (N->getValueType(ResNo: 0) != MVT::i32 &&
15747 N->getValueType(ResNo: 0) != MVT::i64)
15748 return SDValue();
15749
15750 if (!((N->getOperand(Num: 0).getValueType() == MVT::i1 && Subtarget.useCRBits()) ||
15751 (N->getOperand(Num: 0).getValueType() == MVT::i32 && Subtarget.isPPC64())))
15752 return SDValue();
15753
15754 if (N->getOperand(Num: 0).getOpcode() != ISD::AND &&
15755 N->getOperand(Num: 0).getOpcode() != ISD::OR &&
15756 N->getOperand(Num: 0).getOpcode() != ISD::XOR &&
15757 N->getOperand(Num: 0).getOpcode() != ISD::SELECT &&
15758 N->getOperand(Num: 0).getOpcode() != ISD::SELECT_CC)
15759 return SDValue();
15760
15761 SmallVector<SDValue, 4> Inputs;
15762 SmallVector<SDValue, 8> BinOps(1, N->getOperand(Num: 0)), PromOps;
15763 SmallPtrSet<SDNode *, 16> Visited;
15764
15765 // Visit all inputs, collect all binary operations (and, or, xor and
15766 // select) that are all fed by truncations.
15767 while (!BinOps.empty()) {
15768 SDValue BinOp = BinOps.pop_back_val();
15769
15770 if (!Visited.insert(Ptr: BinOp.getNode()).second)
15771 continue;
15772
15773 PromOps.push_back(Elt: BinOp);
15774
15775 for (unsigned i = 0, ie = BinOp.getNumOperands(); i != ie; ++i) {
15776 // The condition of the select is not promoted.
15777 if (BinOp.getOpcode() == ISD::SELECT && i == 0)
15778 continue;
15779 if (BinOp.getOpcode() == ISD::SELECT_CC && i != 2 && i != 3)
15780 continue;
15781
15782 if (BinOp.getOperand(i).getOpcode() == ISD::TRUNCATE ||
15783 isa<ConstantSDNode>(Val: BinOp.getOperand(i))) {
15784 Inputs.push_back(Elt: BinOp.getOperand(i));
15785 } else if (BinOp.getOperand(i).getOpcode() == ISD::AND ||
15786 BinOp.getOperand(i).getOpcode() == ISD::OR ||
15787 BinOp.getOperand(i).getOpcode() == ISD::XOR ||
15788 BinOp.getOperand(i).getOpcode() == ISD::SELECT ||
15789 BinOp.getOperand(i).getOpcode() == ISD::SELECT_CC) {
15790 BinOps.push_back(Elt: BinOp.getOperand(i));
15791 } else {
15792 // We have an input that is not a truncation or another binary
15793 // operation; we'll abort this transformation.
15794 return SDValue();
15795 }
15796 }
15797 }
15798
15799 // The operands of a select that must be truncated when the select is
15800 // promoted because the operand is actually part of the to-be-promoted set.
15801 DenseMap<SDNode *, EVT> SelectTruncOp[2];
15802
15803 // Make sure that this is a self-contained cluster of operations (which
15804 // is not quite the same thing as saying that everything has only one
15805 // use).
15806 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15807 if (isa<ConstantSDNode>(Val: Inputs[i]))
15808 continue;
15809
15810 for (SDNode *User : Inputs[i].getNode()->users()) {
15811 if (User != N && !Visited.count(Ptr: User))
15812 return SDValue();
15813
15814 // If we're going to promote the non-output-value operand(s) or SELECT or
15815 // SELECT_CC, record them for truncation.
15816 if (User->getOpcode() == ISD::SELECT) {
15817 if (User->getOperand(Num: 0) == Inputs[i])
15818 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15819 y: User->getOperand(Num: 0).getValueType()));
15820 } else if (User->getOpcode() == ISD::SELECT_CC) {
15821 if (User->getOperand(Num: 0) == Inputs[i])
15822 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15823 y: User->getOperand(Num: 0).getValueType()));
15824 if (User->getOperand(Num: 1) == Inputs[i])
15825 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15826 y: User->getOperand(Num: 1).getValueType()));
15827 }
15828 }
15829 }
15830
15831 for (unsigned i = 0, ie = PromOps.size(); i != ie; ++i) {
15832 for (SDNode *User : PromOps[i].getNode()->users()) {
15833 if (User != N && !Visited.count(Ptr: User))
15834 return SDValue();
15835
15836 // If we're going to promote the non-output-value operand(s) or SELECT or
15837 // SELECT_CC, record them for truncation.
15838 if (User->getOpcode() == ISD::SELECT) {
15839 if (User->getOperand(Num: 0) == PromOps[i])
15840 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15841 y: User->getOperand(Num: 0).getValueType()));
15842 } else if (User->getOpcode() == ISD::SELECT_CC) {
15843 if (User->getOperand(Num: 0) == PromOps[i])
15844 SelectTruncOp[0].insert(KV: std::make_pair(x&: User,
15845 y: User->getOperand(Num: 0).getValueType()));
15846 if (User->getOperand(Num: 1) == PromOps[i])
15847 SelectTruncOp[1].insert(KV: std::make_pair(x&: User,
15848 y: User->getOperand(Num: 1).getValueType()));
15849 }
15850 }
15851 }
15852
15853 unsigned PromBits = N->getOperand(Num: 0).getValueSizeInBits();
15854 bool ReallyNeedsExt = false;
15855 if (N->getOpcode() != ISD::ANY_EXTEND) {
15856 // If all of the inputs are not already sign/zero extended, then
15857 // we'll still need to do that at the end.
15858 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15859 if (isa<ConstantSDNode>(Val: Inputs[i]))
15860 continue;
15861
15862 unsigned OpBits =
15863 Inputs[i].getOperand(i: 0).getValueSizeInBits();
15864 assert(PromBits < OpBits && "Truncation not to a smaller bit count?");
15865
15866 if ((N->getOpcode() == ISD::ZERO_EXTEND &&
15867 !DAG.MaskedValueIsZero(Op: Inputs[i].getOperand(i: 0),
15868 Mask: APInt::getHighBitsSet(numBits: OpBits,
15869 hiBitsSet: OpBits-PromBits))) ||
15870 (N->getOpcode() == ISD::SIGN_EXTEND &&
15871 DAG.ComputeNumSignBits(Op: Inputs[i].getOperand(i: 0)) <
15872 (OpBits-(PromBits-1)))) {
15873 ReallyNeedsExt = true;
15874 break;
15875 }
15876 }
15877 }
15878
15879 // Convert PromOps to handles before doing any RAUW operations, as these
15880 // may CSE with existing nodes, deleting the originals.
15881 std::list<HandleSDNode> PromOpHandles;
15882 for (auto &PromOp : PromOps)
15883 PromOpHandles.emplace_back(args&: PromOp);
15884
15885 // Replace all inputs, either with the truncation operand, or a
15886 // truncation or extension to the final output type.
15887 for (unsigned i = 0, ie = Inputs.size(); i != ie; ++i) {
15888 // Constant inputs need to be replaced with the to-be-promoted nodes that
15889 // use them because they might have users outside of the cluster of
15890 // promoted nodes.
15891 if (isa<ConstantSDNode>(Val: Inputs[i]))
15892 continue;
15893
15894 SDValue InSrc = Inputs[i].getOperand(i: 0);
15895 if (Inputs[i].getValueType() == N->getValueType(ResNo: 0))
15896 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i], To: InSrc);
15897 else if (N->getOpcode() == ISD::SIGN_EXTEND)
15898 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15899 To: DAG.getSExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15900 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15901 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15902 To: DAG.getZExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15903 else
15904 DAG.ReplaceAllUsesOfValueWith(From: Inputs[i],
15905 To: DAG.getAnyExtOrTrunc(Op: InSrc, DL: dl, VT: N->getValueType(ResNo: 0)));
15906 }
15907
15908 // Replace all operations (these are all the same, but have a different
15909 // (promoted) return type). DAG.getNode will validate that the types of
15910 // a binary operator match, so go through the list in reverse so that
15911 // we've likely promoted both operands first.
15912 while (!PromOpHandles.empty()) {
15913 SDValue PromOp = PromOpHandles.back().getValue();
15914 PromOpHandles.pop_back();
15915
15916 unsigned C;
15917 switch (PromOp.getOpcode()) {
15918 default: C = 0; break;
15919 case ISD::SELECT: C = 1; break;
15920 case ISD::SELECT_CC: C = 2; break;
15921 }
15922
15923 if ((!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C)) &&
15924 PromOp.getOperand(i: C).getValueType() != N->getValueType(ResNo: 0)) ||
15925 (!isa<ConstantSDNode>(Val: PromOp.getOperand(i: C+1)) &&
15926 PromOp.getOperand(i: C+1).getValueType() != N->getValueType(ResNo: 0))) {
15927 // The to-be-promoted operands of this node have not yet been
15928 // promoted (this should be rare because we're going through the
15929 // list backward, but if one of the operands has several users in
15930 // this cluster of to-be-promoted nodes, it is possible).
15931 PromOpHandles.emplace_front(args&: PromOp);
15932 continue;
15933 }
15934
15935 // For SELECT and SELECT_CC nodes, we do a similar check for any
15936 // to-be-promoted comparison inputs.
15937 if (PromOp.getOpcode() == ISD::SELECT ||
15938 PromOp.getOpcode() == ISD::SELECT_CC) {
15939 if ((SelectTruncOp[0].count(Val: PromOp.getNode()) &&
15940 PromOp.getOperand(i: 0).getValueType() != N->getValueType(ResNo: 0)) ||
15941 (SelectTruncOp[1].count(Val: PromOp.getNode()) &&
15942 PromOp.getOperand(i: 1).getValueType() != N->getValueType(ResNo: 0))) {
15943 PromOpHandles.emplace_front(args&: PromOp);
15944 continue;
15945 }
15946 }
15947
15948 SmallVector<SDValue, 3> Ops(PromOp.getNode()->ops());
15949
15950 // If this node has constant inputs, then they'll need to be promoted here.
15951 for (unsigned i = 0; i < 2; ++i) {
15952 if (!isa<ConstantSDNode>(Val: Ops[C+i]))
15953 continue;
15954 if (Ops[C+i].getValueType() == N->getValueType(ResNo: 0))
15955 continue;
15956
15957 if (N->getOpcode() == ISD::SIGN_EXTEND)
15958 Ops[C+i] = DAG.getSExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15959 else if (N->getOpcode() == ISD::ZERO_EXTEND)
15960 Ops[C+i] = DAG.getZExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15961 else
15962 Ops[C+i] = DAG.getAnyExtOrTrunc(Op: Ops[C+i], DL: dl, VT: N->getValueType(ResNo: 0));
15963 }
15964
15965 // If we've promoted the comparison inputs of a SELECT or SELECT_CC,
15966 // truncate them again to the original value type.
15967 if (PromOp.getOpcode() == ISD::SELECT ||
15968 PromOp.getOpcode() == ISD::SELECT_CC) {
15969 auto SI0 = SelectTruncOp[0].find(Val: PromOp.getNode());
15970 if (SI0 != SelectTruncOp[0].end())
15971 Ops[0] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI0->second, Operand: Ops[0]);
15972 auto SI1 = SelectTruncOp[1].find(Val: PromOp.getNode());
15973 if (SI1 != SelectTruncOp[1].end())
15974 Ops[1] = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: SI1->second, Operand: Ops[1]);
15975 }
15976
15977 DAG.ReplaceAllUsesOfValueWith(From: PromOp,
15978 To: DAG.getNode(Opcode: PromOp.getOpcode(), DL: dl, VT: N->getValueType(ResNo: 0), Ops));
15979 }
15980
15981 // Now we're left with the initial extension itself.
15982 if (!ReallyNeedsExt)
15983 return N->getOperand(Num: 0);
15984
15985 // To zero extend, just mask off everything except for the first bit (in the
15986 // i1 case).
15987 if (N->getOpcode() == ISD::ZERO_EXTEND)
15988 return DAG.getNode(Opcode: ISD::AND, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0),
15989 N2: DAG.getConstant(Val: APInt::getLowBitsSet(
15990 numBits: N->getValueSizeInBits(ResNo: 0), loBitsSet: PromBits),
15991 DL: dl, VT: N->getValueType(ResNo: 0)));
15992
15993 assert(N->getOpcode() == ISD::SIGN_EXTEND &&
15994 "Invalid extension type");
15995 EVT ShiftAmountTy = getShiftAmountTy(LHSTy: N->getValueType(ResNo: 0), DL: DAG.getDataLayout());
15996 SDValue ShiftCst =
15997 DAG.getConstant(Val: N->getValueSizeInBits(ResNo: 0) - PromBits, DL: dl, VT: ShiftAmountTy);
15998 return DAG.getNode(
15999 Opcode: ISD::SRA, DL: dl, VT: N->getValueType(ResNo: 0),
16000 N1: DAG.getNode(Opcode: ISD::SHL, DL: dl, VT: N->getValueType(ResNo: 0), N1: N->getOperand(Num: 0), N2: ShiftCst),
16001 N2: ShiftCst);
16002}
16003
16004// The function check a i128 load can convert to 16i8 load for Vcmpequb.
16005static bool canConvertToVcmpequb(SDValue &LHS, SDValue &RHS, bool IsPPC64) {
16006
16007 auto isValidForConvert = [IsPPC64](SDValue &Operand) {
16008 if (!Operand.hasOneUse())
16009 return false;
16010
16011 if (Operand.getValueType() != MVT::i128)
16012 return false;
16013
16014 if (Operand.getOpcode() == ISD::Constant) {
16015 auto *C = cast<ConstantSDNode>(Val&: Operand);
16016 const APInt &Val = C->getAPIntValue();
16017 // On PPC64, comparing an i128 value loaded from memory against a
16018 // constant smaller than 2^16 is usually better left to scalar lowering.
16019 // In that case, the compare can be lowered using xori (since xori has a
16020 // 16-bit immediate field), which is cheaper than materializing a vector
16021 // constant and using vcmpequb.
16022 if (IsPPC64 && Val.ult(RHS: 1ULL << 16))
16023 return false;
16024 return true;
16025 }
16026
16027 auto *LoadNode = dyn_cast<LoadSDNode>(Val&: Operand);
16028 if (!LoadNode)
16029 return false;
16030
16031 // If memory operation is volatile, do not perform any
16032 // optimization or transformation. Volatile operations must be preserved
16033 // as written to ensure correct program behavior, so we return an empty
16034 // SDValue to indicate no action.
16035
16036 if (LoadNode->isVolatile())
16037 return false;
16038
16039 // Only combine loads if both use the unindexed addressing mode.
16040 // PowerPC AltiVec/VMX does not support vector loads or stores with
16041 // pre/post-increment addressing. Indexed modes may imply implicit
16042 // pointer updates, which are not compatible with AltiVec vector
16043 // instructions.
16044 if (LoadNode->getAddressingMode() != ISD::UNINDEXED)
16045 return false;
16046
16047 // Only combine loads if both are non-extending loads
16048 // (ISD::NON_EXTLOAD). Extending loads (such as ISD::ZEXTLOAD or
16049 // ISD::SEXTLOAD) perform zero or sign extension, which may change the
16050 // loaded value's semantics and are not compatible with vector loads.
16051 if (LoadNode->getExtensionType() != ISD::NON_EXTLOAD)
16052 return false;
16053
16054 return true;
16055 };
16056
16057 return (isValidForConvert(LHS) && isValidForConvert(RHS));
16058}
16059
16060SDValue convertTwoLoadsAndCmpToVCMPEQUB(SelectionDAG &DAG, SDNode *N,
16061 const SDLoc &DL) {
16062
16063 assert(N->getOpcode() == ISD::SETCC && "Should be called with a SETCC node");
16064
16065 ISD::CondCode CC = N->getOperandCondCode(Num: 2);
16066 assert((CC == ISD::SETNE || CC == ISD::SETEQ) &&
16067 "CC mus be ISD::SETNE or ISD::SETEQ");
16068
16069 auto getV16i8Load = [&](const SDValue &Operand) {
16070 if (Operand.getOpcode() == ISD::Constant)
16071 return DAG.getBitcast(VT: MVT::v16i8, V: Operand);
16072
16073 assert(Operand.getOpcode() == ISD::LOAD && "Must be LoadSDNode here.");
16074
16075 auto *LoadNode = cast<LoadSDNode>(Val: Operand);
16076 // Create a new MachineMemOperand without range metadata.
16077 // Range metadata is only valid for integer scalar types, not vectors.
16078 // The original i128 load may have range metadata, but when we convert
16079 // to v16i8, that metadata is no longer semantically valid.
16080 MachineMemOperand *MMO = LoadNode->getMemOperand();
16081 MachineFunction &MF = DAG.getMachineFunction();
16082 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
16083 PtrInfo: MMO->getPointerInfo(), F: MMO->getFlags(), Size: MMO->getSize(), BaseAlignment: MMO->getAlign(),
16084 Metadata: MMO->getAAInfo(), SSID: MMO->getSyncScopeID(), Ordering: MMO->getSuccessOrdering(),
16085 FailureOrdering: MMO->getFailureOrdering());
16086 SDValue NewLoad = DAG.getLoad(VT: MVT::v16i8, dl: DL, Chain: LoadNode->getChain(),
16087 Ptr: LoadNode->getBasePtr(), MMO: NewMMO);
16088 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LoadNode, 1), To: NewLoad.getValue(R: 1));
16089 return NewLoad;
16090 };
16091
16092 // Following code transforms the DAG
16093 // t0: ch,glue = EntryToken
16094 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16095 // t3: i128,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16096 // undef:i64
16097 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16098 // t5: i128,ch =
16099 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64 t6: i1 =
16100 // setcc t3, t5, setne:ch
16101 //
16102 // ---->
16103 //
16104 // t0: ch,glue = EntryToken
16105 // t2: i64,ch = CopyFromReg t0, Register:i64 %0
16106 // t3: v16i8,ch = load<(load (s128) from %ir.a, align 1)> t0, t2,
16107 // undef:i64
16108 // t4: i64,ch = CopyFromReg t0, Register:i64 %1
16109 // t5: v16i8,ch =
16110 // load<(load (s128) from %ir.b, align 1)> t0, t4, undef:i64
16111 // t6: i32 =
16112 // llvm.ppc.altivec.vcmpequb.p TargetConstant:i32<10505>,
16113 // Constant:i32<2>, t3, t5
16114 // t7: i1 = setcc t6, Constant:i32<0>, seteq:ch
16115
16116 // Or transforms the DAG
16117 // t5: i128,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16118 // t8: i1 =
16119 // setcc Constant:i128<237684487579686500932345921536>, t5, setne:ch
16120 //
16121 // --->
16122 //
16123 // t5: v16i8,ch = load<(load (s128) from %ir.X, align 1)> t0, t2, undef:i64
16124 // t6: v16i8 = bitcast Constant:i128<237684487579686500932345921536>
16125 // t7: i32 =
16126 // llvm.ppc.altivec.vcmpequb.p Constant:i32<10962>, Constant:i32<2>, t5, t2
16127
16128 SDValue LHSVec = getV16i8Load(N->getOperand(Num: 0));
16129 SDValue RHSVec = getV16i8Load(N->getOperand(Num: 1));
16130
16131 SDValue IntrID =
16132 DAG.getConstant(Val: Intrinsic::ppc_altivec_vcmpequb_p, DL, VT: MVT::i32);
16133 SDValue CRSel = DAG.getConstant(Val: 2, DL, VT: MVT::i32); // which CR6 predicate field
16134 SDValue PredResult = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
16135 N1: IntrID, N2: CRSel, N3: LHSVec, N4: RHSVec);
16136 // ppc_altivec_vcmpequb_p returns 1 when two vectors are the same,
16137 // so we need to invert the CC opcode.
16138 return DAG.getSetCC(DL, VT: N->getValueType(ResNo: 0), LHS: PredResult,
16139 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32),
16140 Cond: CC == ISD::SETNE ? ISD::SETEQ : ISD::SETNE);
16141}
16142
16143// Detect whether there is a pattern like (setcc (and X, 1), 0, eq).
16144// If it is , return true; otherwise return false.
16145static bool canConvertSETCCToXori(SDNode *N) {
16146 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16147
16148 ISD::CondCode CC = N->getOperandCondCode(Num: 2);
16149 if (CC != ISD::SETEQ)
16150 return false;
16151
16152 SDValue LHS = N->getOperand(Num: 0);
16153 SDValue RHS = N->getOperand(Num: 1);
16154
16155 // Check the `SDValue &V` is from `and` with `1`.
16156 auto IsAndWithOne = [](SDValue &V) {
16157 if (V.getOpcode() == ISD::AND) {
16158 for (const SDValue &Op : V->ops())
16159 if (auto *C = dyn_cast<ConstantSDNode>(Val: Op))
16160 if (C->isOne())
16161 return true;
16162 }
16163 return false;
16164 };
16165
16166 // Check whether the SETCC compare with zero.
16167 auto IsCompareWithZero = [](SDValue &V) {
16168 if (auto *C = dyn_cast<ConstantSDNode>(Val&: V))
16169 if (C->isZero())
16170 return true;
16171 return false;
16172 };
16173
16174 return (IsAndWithOne(LHS) && IsCompareWithZero(RHS)) ||
16175 (IsAndWithOne(RHS) && IsCompareWithZero(LHS));
16176}
16177
16178// You must check whether the `SDNode* N` can be converted to Xori using
16179// the function `static bool canConvertSETCCToXori(SDNode *N)`
16180// before calling the function; otherwise, it may produce incorrect results.
16181static SDValue ConvertSETCCToXori(SDNode *N, SelectionDAG &DAG) {
16182
16183 assert(N->getOpcode() == ISD::SETCC && "Should be SETCC SDNode here.");
16184 SDValue LHS = N->getOperand(Num: 0);
16185 SDValue RHS = N->getOperand(Num: 1);
16186 SDLoc DL(N);
16187
16188 [[maybe_unused]] ISD::CondCode CC = N->getOperandCondCode(Num: 2);
16189 assert((CC == ISD::SETEQ) && "CC must be ISD::SETEQ.");
16190 // Rewrite it as XORI (and X, 1), 1.
16191 auto MakeXor1 = [&](SDValue V) {
16192 EVT VT = V.getValueType();
16193 SDValue One = DAG.getConstant(Val: 1, DL, VT);
16194 SDValue Xor = DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: V, N2: One);
16195 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i1, Operand: Xor);
16196 };
16197
16198 if (LHS.getOpcode() == ISD::AND && RHS.getOpcode() != ISD::AND)
16199 return MakeXor1(LHS);
16200
16201 if (RHS.getOpcode() == ISD::AND && LHS.getOpcode() != ISD::AND)
16202 return MakeXor1(RHS);
16203
16204 llvm_unreachable("Should not reach here.");
16205}
16206
16207// Match `sext(setcc X, 0, eq)` and turn it into an ADDIC/SUBFE sequence.
16208//
16209// This generates code for:
16210// X == 0 ? -1 : 0
16211//
16212// On pre-ISA 3.1 targets, this is better than the longer CNTLZW/SRWI/NEG
16213// sequence. This is useful for cases like:
16214// uint8_t f(uint8_t x) { return (x == 0) ? -1 : 0; }
16215//
16216// ISA 3.1+ is skipped because those targets can use SETBC.
16217
16218SDValue PPCTargetLowering::combineSignExtendSetCC(SDNode *N,
16219 DAGCombinerInfo &DCI) const {
16220 if (Subtarget.isISA3_1())
16221 return SDValue();
16222
16223 EVT VT = N->getValueType(ResNo: 0);
16224 if (VT != MVT::i32 && (VT != MVT::i64 || !Subtarget.isPPC64()))
16225 return SDValue();
16226
16227 SDValue N0 = N->getOperand(Num: 0);
16228 if (N0.getOpcode() != ISD::SETCC)
16229 return SDValue();
16230
16231 ISD::CondCode CC = N0.getOperandCondCode(i: 2);
16232 SDValue LHS = N0.getOperand(i: 0);
16233 SDValue RHS = N0.getOperand(i: 1);
16234
16235 // Not match: sext (setcc x, 0, eq) or sext (setcc 0, x, eq)
16236 if (CC != ISD::SETEQ || (!isNullConstant(V: LHS) && !isNullConstant(V: RHS)))
16237 return SDValue();
16238
16239 SDLoc dl(N);
16240 SelectionDAG &DAG = DCI.DAG;
16241 SDValue X = isNullConstant(V: LHS) ? RHS : LHS;
16242 EVT XVT = X.getValueType(); // The type of x in the setcc x, 0, eq.
16243
16244 // The type that ADDC/SUBE operate on. Reject larger types and zero-extend
16245 // smaller ones.
16246 MVT OpVT = Subtarget.isPPC64() ? MVT::i64 : MVT::i32;
16247 if (XVT.bitsGT(VT: OpVT))
16248 return SDValue();
16249
16250 if (XVT.bitsLT(VT: OpVT))
16251 X = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: dl, VT: OpVT, Operand: X);
16252
16253 // Generate: SUBFE(ADDC(X, -1)).
16254 SDValue MinusOne = DAG.getAllOnesConstant(DL: dl, VT: OpVT);
16255 SDValue Addc =
16256 DAG.getNode(Opcode: PPCISD::ADDC, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32), N1: X, N2: MinusOne);
16257 SDValue Carry = Addc.getValue(R: 1);
16258 SDValue Sube = DAG.getNode(Opcode: PPCISD::SUBE, DL: dl, VTList: DAG.getVTList(VT1: OpVT, VT2: MVT::i32),
16259 N1: Addc, N2: Addc, N3: Carry);
16260
16261 // Truncate back to i32 if we used i64 operations.
16262 if (OpVT == MVT::i64 && VT == MVT::i32)
16263 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT, Operand: Sube);
16264
16265 return Sube;
16266}
16267
16268SDValue PPCTargetLowering::combineSetCC(SDNode *N,
16269 DAGCombinerInfo &DCI) const {
16270 assert(N->getOpcode() == ISD::SETCC &&
16271 "Should be called with a SETCC node");
16272
16273 // Check if the pattern (setcc (and X, 1), 0, eq) is present.
16274 // If it is, rewrite it as XORI (and X, 1), 1.
16275 if (canConvertSETCCToXori(N))
16276 return ConvertSETCCToXori(N, DAG&: DCI.DAG);
16277
16278 ISD::CondCode CC = N->getOperandCondCode(Num: 2);
16279 if (CC == ISD::SETNE || CC == ISD::SETEQ) {
16280 SDValue LHS = N->getOperand(Num: 0);
16281 SDValue RHS = N->getOperand(Num: 1);
16282
16283 // If there is a '0 - y' pattern, canonicalize the pattern to the RHS.
16284 if (LHS.getOpcode() == ISD::SUB && isNullConstant(V: LHS.getOperand(i: 0)) &&
16285 LHS.hasOneUse())
16286 std::swap(a&: LHS, b&: RHS);
16287
16288 // x == 0-y --> x+y == 0
16289 // x != 0-y --> x+y != 0
16290 if (RHS.getOpcode() == ISD::SUB && isNullConstant(V: RHS.getOperand(i: 0)) &&
16291 RHS.hasOneUse()) {
16292 SDLoc DL(N);
16293 SelectionDAG &DAG = DCI.DAG;
16294 EVT VT = N->getValueType(ResNo: 0);
16295 EVT OpVT = LHS.getValueType();
16296 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: OpVT, N1: LHS, N2: RHS.getOperand(i: 1));
16297 return DAG.getSetCC(DL, VT, LHS: Add, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT), Cond: CC);
16298 }
16299
16300 // Optimization: Fold i128 equality/inequality compares of two loads into a
16301 // vectorized compare using vcmpequb.p when Altivec is available.
16302 //
16303 // Rationale:
16304 // A scalar i128 SETCC (eq/ne) normally lowers to multiple scalar ops.
16305 // On VSX-capable subtargets, we can instead reinterpret the i128 loads
16306 // as v16i8 vectors and use the Altive vcmpequb.p instruction to
16307 // perform a full 128-bit equality check in a single vector compare.
16308 //
16309 // Example Result:
16310 // This transformation replaces memcmp(a, b, 16) with two vector loads
16311 // and one vector compare instruction.
16312
16313 if (Subtarget.hasAltivec() &&
16314 canConvertToVcmpequb(LHS, RHS, IsPPC64: Subtarget.isPPC64()))
16315 return convertTwoLoadsAndCmpToVCMPEQUB(DAG&: DCI.DAG, N, DL: SDLoc(N));
16316 }
16317
16318 return DAGCombineTruncBoolExt(N, DCI);
16319}
16320
16321// Is this an extending load from an f32 to an f64?
16322static bool isFPExtLoad(SDValue Op) {
16323 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: Op.getNode()))
16324 return LD->getExtensionType() == ISD::EXTLOAD &&
16325 Op.getValueType() == MVT::f64;
16326 return false;
16327}
16328
16329/// Reduces the number of fp-to-int conversion when building a vector.
16330///
16331/// If this vector is built out of floating to integer conversions,
16332/// transform it to a vector built out of floating point values followed by a
16333/// single floating to integer conversion of the vector.
16334/// Namely (build_vector (fptosi $A), (fptosi $B), ...)
16335/// becomes (fptosi (build_vector ($A, $B, ...)))
16336SDValue PPCTargetLowering::
16337combineElementTruncationToVectorTruncation(SDNode *N,
16338 DAGCombinerInfo &DCI) const {
16339 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16340 "Should be called with a BUILD_VECTOR node");
16341
16342 SelectionDAG &DAG = DCI.DAG;
16343 SDLoc dl(N);
16344
16345 SDValue FirstInput = N->getOperand(Num: 0);
16346 assert(FirstInput.getOpcode() == PPCISD::MFVSR &&
16347 "The input operand must be an fp-to-int conversion.");
16348
16349 // This combine happens after legalization so the fp_to_[su]i nodes are
16350 // already converted to PPCSISD nodes.
16351 unsigned FirstConversion = FirstInput.getOperand(i: 0).getOpcode();
16352 if (FirstConversion == PPCISD::FCTIDZ ||
16353 FirstConversion == PPCISD::FCTIDUZ ||
16354 FirstConversion == PPCISD::FCTIWZ ||
16355 FirstConversion == PPCISD::FCTIWUZ) {
16356 bool IsSplat = true;
16357 bool Is32Bit = FirstConversion == PPCISD::FCTIWZ ||
16358 FirstConversion == PPCISD::FCTIWUZ;
16359 EVT SrcVT = FirstInput.getOperand(i: 0).getValueType();
16360 SmallVector<SDValue, 4> Ops;
16361 EVT TargetVT = N->getValueType(ResNo: 0);
16362 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16363 SDValue NextOp = N->getOperand(Num: i);
16364 if (NextOp.getOpcode() != PPCISD::MFVSR)
16365 return SDValue();
16366 unsigned NextConversion = NextOp.getOperand(i: 0).getOpcode();
16367 if (NextConversion != FirstConversion)
16368 return SDValue();
16369 // If we are converting to 32-bit integers, we need to add an FP_ROUND.
16370 // This is not valid if the input was originally double precision. It is
16371 // also not profitable to do unless this is an extending load in which
16372 // case doing this combine will allow us to combine consecutive loads.
16373 if (Is32Bit && !isFPExtLoad(Op: NextOp.getOperand(i: 0).getOperand(i: 0)))
16374 return SDValue();
16375 if (N->getOperand(Num: i) != FirstInput)
16376 IsSplat = false;
16377 }
16378
16379 // If this is a splat, we leave it as-is since there will be only a single
16380 // fp-to-int conversion followed by a splat of the integer. This is better
16381 // for 32-bit and smaller ints and neutral for 64-bit ints.
16382 if (IsSplat)
16383 return SDValue();
16384
16385 // Now that we know we have the right type of node, get its operands
16386 for (int i = 0, e = N->getNumOperands(); i < e; ++i) {
16387 SDValue In = N->getOperand(Num: i).getOperand(i: 0);
16388 if (Is32Bit) {
16389 // For 32-bit values, we need to add an FP_ROUND node (if we made it
16390 // here, we know that all inputs are extending loads so this is safe).
16391 if (In.isUndef())
16392 Ops.push_back(Elt: DAG.getUNDEF(VT: SrcVT));
16393 else {
16394 SDValue Trunc =
16395 DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: In.getOperand(i: 0),
16396 N2: DAG.getIntPtrConstant(Val: 1, DL: dl, /*isTarget=*/true));
16397 Ops.push_back(Elt: Trunc);
16398 }
16399 } else
16400 Ops.push_back(Elt: In.isUndef() ? DAG.getUNDEF(VT: SrcVT) : In.getOperand(i: 0));
16401 }
16402
16403 unsigned Opcode;
16404 if (FirstConversion == PPCISD::FCTIDZ ||
16405 FirstConversion == PPCISD::FCTIWZ)
16406 Opcode = ISD::FP_TO_SINT;
16407 else
16408 Opcode = ISD::FP_TO_UINT;
16409
16410 EVT NewVT = TargetVT == MVT::v2i64 ? MVT::v2f64 : MVT::v4f32;
16411 SDValue BV = DAG.getBuildVector(VT: NewVT, DL: dl, Ops);
16412 return DAG.getNode(Opcode, DL: dl, VT: TargetVT, Operand: BV);
16413 }
16414 return SDValue();
16415}
16416
16417// LXVKQ instruction load VSX vector with a special quadword value
16418// based on an immediate value. This helper method returns the details of the
16419// match as a tuple of {LXVKQ unsigned IMM Value, right_shift_amount}
16420// to help generate the LXVKQ instruction and the subsequent shift instruction
16421// required to match the original build vector pattern.
16422
16423// LXVKQPattern: {LXVKQ unsigned IMM Value, right_shift_amount}
16424using LXVKQPattern = std::tuple<uint32_t, uint8_t>;
16425
16426static std::optional<LXVKQPattern> getPatternInfo(const APInt &FullVal) {
16427
16428 // LXVKQ instruction loads the Quadword value:
16429 // 0x8000_0000_0000_0000_0000_0000_0000_0000 when imm = 0b10000
16430 static const APInt BasePattern = APInt(128, 0x8000000000000000ULL) << 64;
16431 static const uint32_t Uim = 16;
16432
16433 // Check for direct LXVKQ match (no shift needed)
16434 if (FullVal == BasePattern)
16435 return std::make_tuple(args: Uim, args: uint8_t{0});
16436
16437 // Check if FullValue is 1 (the result of the base pattern >> 127)
16438 if (FullVal == APInt(128, 1))
16439 return std::make_tuple(args: Uim, args: uint8_t{127});
16440
16441 return std::nullopt;
16442}
16443
16444/// Combine vector loads to a single load (using lxvkq) or splat with shift of a
16445/// constant (xxspltib + vsrq) by recognising patterns in the Build Vector.
16446/// LXVKQ instruction load VSX vector with a special quadword value based on an
16447/// immediate value. if UIM=0b10000 then LXVKQ loads VSR[32×TX+T] with value
16448/// 0x8000_0000_0000_0000_0000_0000_0000_0000.
16449/// This can be used to inline the build vector constants that have the
16450/// following patterns:
16451///
16452/// 0x8000_0000_0000_0000_0000_0000_0000_0000 (MSB set pattern)
16453/// 0x0000_0000_0000_0000_0000_0000_0000_0001 (LSB set pattern)
16454/// MSB pattern can directly loaded using LXVKQ while LSB is loaded using a
16455/// combination of splatting and right shift instructions.
16456
16457SDValue PPCTargetLowering::combineBVLoadsSpecialValue(SDValue Op,
16458 SelectionDAG &DAG) const {
16459
16460 assert((Op.getNode() && Op.getOpcode() == ISD::BUILD_VECTOR) &&
16461 "Expected a BuildVectorSDNode in combineBVLoadsSpecialValue");
16462
16463 // This transformation is only supported if we are loading either a byte,
16464 // halfword, word, or doubleword.
16465 EVT VT = Op.getValueType();
16466 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
16467 VT == MVT::v2i64))
16468 return SDValue();
16469
16470 LLVM_DEBUG(llvm::dbgs() << "\ncombineBVLoadsSpecialValue: Build vector ("
16471 << VT << "): ";
16472 Op->dump());
16473
16474 unsigned NumElems = VT.getVectorNumElements();
16475 unsigned ElemBits = VT.getScalarSizeInBits();
16476
16477 bool IsLittleEndian = DAG.getDataLayout().isLittleEndian();
16478
16479 // Check for Non-constant operand in the build vector.
16480 for (const SDValue &Operand : Op.getNode()->op_values()) {
16481 if (!isa<ConstantSDNode>(Val: Operand))
16482 return SDValue();
16483 }
16484
16485 // Assemble build vector operands as a 128-bit register value
16486 // We need to reconstruct what the 128-bit register pattern would be
16487 // that produces this vector when interpreted with the current endianness
16488 APInt FullVal = APInt::getZero(numBits: 128);
16489
16490 for (unsigned Index = 0; Index < NumElems; ++Index) {
16491 auto *C = cast<ConstantSDNode>(Val: Op.getOperand(i: Index));
16492
16493 // Get element value as raw bits (zero-extended)
16494 uint64_t ElemValue = C->getZExtValue();
16495
16496 // Mask to element size to ensure we only get the relevant bits
16497 if (ElemBits < 64)
16498 ElemValue &= ((1ULL << ElemBits) - 1);
16499
16500 // Calculate bit position for this element in the 128-bit register
16501 unsigned BitPos =
16502 (IsLittleEndian) ? (Index * ElemBits) : (128 - (Index + 1) * ElemBits);
16503
16504 // Create APInt for the element value and shift it to correct position
16505 APInt ElemAPInt(128, ElemValue);
16506 ElemAPInt <<= BitPos;
16507
16508 // Place the element value at the correct bit position
16509 FullVal |= ElemAPInt;
16510 }
16511
16512 if (FullVal.isZero() || FullVal.isAllOnes())
16513 return SDValue();
16514
16515 if (auto UIMOpt = getPatternInfo(FullVal)) {
16516 const auto &[Uim, ShiftAmount] = *UIMOpt;
16517 SDLoc Dl(Op);
16518
16519 // Generate LXVKQ instruction if the shift amount is zero.
16520 if (ShiftAmount == 0) {
16521 SDValue UimVal = DAG.getTargetConstant(Val: Uim, DL: Dl, VT: MVT::i32);
16522 SDValue LxvkqInstr =
16523 SDValue(DAG.getMachineNode(Opcode: PPC::LXVKQ, dl: Dl, VT, Op1: UimVal), 0);
16524 LLVM_DEBUG(llvm::dbgs()
16525 << "combineBVLoadsSpecialValue: Instruction Emitted ";
16526 LxvkqInstr.dump());
16527 return LxvkqInstr;
16528 }
16529
16530 assert(ShiftAmount == 127 && "Unexpected lxvkq shift amount value");
16531
16532 // The right shifted pattern can be constructed using a combination of
16533 // XXSPLTIB and VSRQ instruction. VSRQ uses the shift amount from the lower
16534 // 7 bits of byte 15. This can be specified using XXSPLTIB with immediate
16535 // value 255.
16536 SDValue ShiftAmountVec =
16537 SDValue(DAG.getMachineNode(Opcode: PPC::XXSPLTIB, dl: Dl, VT: MVT::v4i32,
16538 Op1: DAG.getTargetConstant(Val: 255, DL: Dl, VT: MVT::i32)),
16539 0);
16540 // Generate appropriate right shift instruction
16541 SDValue ShiftVec = SDValue(
16542 DAG.getMachineNode(Opcode: PPC::VSRQ, dl: Dl, VT, Op1: ShiftAmountVec, Op2: ShiftAmountVec),
16543 0);
16544 LLVM_DEBUG(llvm::dbgs()
16545 << "\n combineBVLoadsSpecialValue: Instruction Emitted ";
16546 ShiftVec.dump());
16547 return ShiftVec;
16548 }
16549 // No patterns matched for build vectors.
16550 return SDValue();
16551}
16552
16553/// Reduce the number of loads when building a vector.
16554///
16555/// Building a vector out of multiple loads can be converted to a load
16556/// of the vector type if the loads are consecutive. If the loads are
16557/// consecutive but in descending order, a shuffle is added at the end
16558/// to reorder the vector.
16559static SDValue combineBVOfConsecutiveLoads(SDNode *N, SelectionDAG &DAG) {
16560 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16561 "Should be called with a BUILD_VECTOR node");
16562
16563 SDLoc dl(N);
16564
16565 // Return early for non byte-sized type, as they can't be consecutive.
16566 if (!N->getValueType(ResNo: 0).getVectorElementType().isByteSized())
16567 return SDValue();
16568
16569 bool InputsAreConsecutiveLoads = true;
16570 bool InputsAreReverseConsecutive = true;
16571 unsigned ElemSize = N->getValueType(ResNo: 0).getScalarType().getStoreSize();
16572 SDValue FirstInput = N->getOperand(Num: 0);
16573 bool IsRoundOfExtLoad = false;
16574 LoadSDNode *FirstLoad = nullptr;
16575
16576 if (FirstInput.getOpcode() == ISD::FP_ROUND &&
16577 FirstInput.getOperand(i: 0).getOpcode() == ISD::LOAD) {
16578 FirstLoad = cast<LoadSDNode>(Val: FirstInput.getOperand(i: 0));
16579 IsRoundOfExtLoad = FirstLoad->getExtensionType() == ISD::EXTLOAD;
16580 }
16581 // Not a build vector of (possibly fp_rounded) loads.
16582 if ((!IsRoundOfExtLoad && FirstInput.getOpcode() != ISD::LOAD) ||
16583 N->getNumOperands() == 1)
16584 return SDValue();
16585
16586 if (!IsRoundOfExtLoad)
16587 FirstLoad = cast<LoadSDNode>(Val&: FirstInput);
16588
16589 SmallVector<LoadSDNode *, 4> InputLoads;
16590 InputLoads.push_back(Elt: FirstLoad);
16591 for (int i = 1, e = N->getNumOperands(); i < e; ++i) {
16592 // If any inputs are fp_round(extload), they all must be.
16593 if (IsRoundOfExtLoad && N->getOperand(Num: i).getOpcode() != ISD::FP_ROUND)
16594 return SDValue();
16595
16596 SDValue NextInput = IsRoundOfExtLoad ? N->getOperand(Num: i).getOperand(i: 0) :
16597 N->getOperand(Num: i);
16598 if (NextInput.getOpcode() != ISD::LOAD)
16599 return SDValue();
16600
16601 SDValue PreviousInput =
16602 IsRoundOfExtLoad ? N->getOperand(Num: i-1).getOperand(i: 0) : N->getOperand(Num: i-1);
16603 LoadSDNode *LD1 = cast<LoadSDNode>(Val&: PreviousInput);
16604 LoadSDNode *LD2 = cast<LoadSDNode>(Val&: NextInput);
16605
16606 // If any inputs are fp_round(extload), they all must be.
16607 if (IsRoundOfExtLoad && LD2->getExtensionType() != ISD::EXTLOAD)
16608 return SDValue();
16609
16610 // We only care about regular loads. The PPC-specific load intrinsics
16611 // will not lead to a merge opportunity.
16612 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD2, Base: LD1, Bytes: ElemSize, Dist: 1))
16613 InputsAreConsecutiveLoads = false;
16614 if (!DAG.areNonVolatileConsecutiveLoads(LD: LD1, Base: LD2, Bytes: ElemSize, Dist: 1))
16615 InputsAreReverseConsecutive = false;
16616
16617 // Exit early if the loads are neither consecutive nor reverse consecutive.
16618 if (!InputsAreConsecutiveLoads && !InputsAreReverseConsecutive)
16619 return SDValue();
16620 InputLoads.push_back(Elt: LD2);
16621 }
16622
16623 assert(!(InputsAreConsecutiveLoads && InputsAreReverseConsecutive) &&
16624 "The loads cannot be both consecutive and reverse consecutive.");
16625
16626 SDValue WideLoad;
16627 SDValue ReturnSDVal;
16628 if (InputsAreConsecutiveLoads) {
16629 assert(FirstLoad && "Input needs to be a LoadSDNode.");
16630 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: FirstLoad->getChain(),
16631 Ptr: FirstLoad->getBasePtr(), PtrInfo: FirstLoad->getPointerInfo(),
16632 Alignment: FirstLoad->getAlign());
16633 ReturnSDVal = WideLoad;
16634 } else if (InputsAreReverseConsecutive) {
16635 LoadSDNode *LastLoad = InputLoads.back();
16636 assert(LastLoad && "Input needs to be a LoadSDNode.");
16637 WideLoad = DAG.getLoad(VT: N->getValueType(ResNo: 0), dl, Chain: LastLoad->getChain(),
16638 Ptr: LastLoad->getBasePtr(), PtrInfo: LastLoad->getPointerInfo(),
16639 Alignment: LastLoad->getAlign());
16640 SmallVector<int, 16> Ops;
16641 for (int i = N->getNumOperands() - 1; i >= 0; i--)
16642 Ops.push_back(Elt: i);
16643
16644 ReturnSDVal = DAG.getVectorShuffle(VT: N->getValueType(ResNo: 0), dl, N1: WideLoad,
16645 N2: DAG.getUNDEF(VT: N->getValueType(ResNo: 0)), Mask: Ops);
16646 } else
16647 return SDValue();
16648
16649 for (auto *LD : InputLoads)
16650 DAG.makeEquivalentMemoryOrdering(OldLoad: LD, NewMemOp: WideLoad);
16651 return ReturnSDVal;
16652}
16653
16654// This function adds the required vector_shuffle needed to get
16655// the elements of the vector extract in the correct position
16656// as specified by the CorrectElems encoding.
16657static SDValue addShuffleForVecExtend(SDNode *N, SelectionDAG &DAG,
16658 SDValue Input, uint64_t Elems,
16659 uint64_t CorrectElems) {
16660 SDLoc dl(N);
16661
16662 unsigned NumElems = Input.getValueType().getVectorNumElements();
16663 SmallVector<int, 16> ShuffleMask(NumElems, -1);
16664
16665 // Knowing the element indices being extracted from the original
16666 // vector and the order in which they're being inserted, just put
16667 // them at element indices required for the instruction.
16668 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16669 if (DAG.getDataLayout().isLittleEndian())
16670 ShuffleMask[CorrectElems & 0xF] = Elems & 0xF;
16671 else
16672 ShuffleMask[(CorrectElems & 0xF0) >> 4] = (Elems & 0xF0) >> 4;
16673 CorrectElems = CorrectElems >> 8;
16674 Elems = Elems >> 8;
16675 }
16676
16677 SDValue Shuffle =
16678 DAG.getVectorShuffle(VT: Input.getValueType(), dl, N1: Input,
16679 N2: DAG.getUNDEF(VT: Input.getValueType()), Mask: ShuffleMask);
16680
16681 EVT VT = N->getValueType(ResNo: 0);
16682 SDValue Conv = DAG.getBitcast(VT, V: Shuffle);
16683
16684 EVT ExtVT = EVT::getVectorVT(Context&: *DAG.getContext(),
16685 VT: Input.getValueType().getVectorElementType(),
16686 NumElements: VT.getVectorNumElements());
16687 return DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT, N1: Conv,
16688 N2: DAG.getValueType(ExtVT));
16689}
16690
16691// Look for build vector patterns where input operands come from sign
16692// extended vector_extract elements of specific indices. If the correct indices
16693// aren't used, add a vector shuffle to fix up the indices and create
16694// SIGN_EXTEND_INREG node which selects the vector sign extend instructions
16695// during instruction selection.
16696static SDValue combineBVOfVecSExt(SDNode *N, SelectionDAG &DAG) {
16697 // This array encodes the indices that the vector sign extend instructions
16698 // extract from when extending from one type to another for both BE and LE.
16699 // The right nibble of each byte corresponds to the LE incides.
16700 // and the left nibble of each byte corresponds to the BE incides.
16701 // For example: 0x3074B8FC byte->word
16702 // For LE: the allowed indices are: 0x0,0x4,0x8,0xC
16703 // For BE: the allowed indices are: 0x3,0x7,0xB,0xF
16704 // For example: 0x000070F8 byte->double word
16705 // For LE: the allowed indices are: 0x0,0x8
16706 // For BE: the allowed indices are: 0x7,0xF
16707 uint64_t TargetElems[] = {
16708 0x3074B8FC, // b->w
16709 0x000070F8, // b->d
16710 0x10325476, // h->w
16711 0x00003074, // h->d
16712 0x00001032, // w->d
16713 };
16714
16715 uint64_t Elems = 0;
16716 int Index;
16717 SDValue Input;
16718
16719 auto isSExtOfVecExtract = [&](SDValue Op) -> bool {
16720 if (!Op)
16721 return false;
16722 if (Op.getOpcode() != ISD::SIGN_EXTEND &&
16723 Op.getOpcode() != ISD::SIGN_EXTEND_INREG)
16724 return false;
16725
16726 // A SIGN_EXTEND_INREG might be fed by an ANY_EXTEND to produce a value
16727 // of the right width.
16728 SDValue Extract = Op.getOperand(i: 0);
16729 if (Extract.getOpcode() == ISD::ANY_EXTEND)
16730 Extract = Extract.getOperand(i: 0);
16731 if (Extract.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16732 return false;
16733
16734 ConstantSDNode *ExtOp = dyn_cast<ConstantSDNode>(Val: Extract.getOperand(i: 1));
16735 if (!ExtOp)
16736 return false;
16737
16738 Index = ExtOp->getZExtValue();
16739 if (Input && Input != Extract.getOperand(i: 0))
16740 return false;
16741
16742 if (!Input)
16743 Input = Extract.getOperand(i: 0);
16744
16745 Elems = Elems << 8;
16746 Index = DAG.getDataLayout().isLittleEndian() ? Index : Index << 4;
16747 Elems |= Index;
16748
16749 return true;
16750 };
16751
16752 // If the build vector operands aren't sign extended vector extracts,
16753 // of the same input vector, then return.
16754 for (unsigned i = 0; i < N->getNumOperands(); i++) {
16755 if (!isSExtOfVecExtract(N->getOperand(Num: i))) {
16756 return SDValue();
16757 }
16758 }
16759
16760 // If the vector extract indices are not correct, add the appropriate
16761 // vector_shuffle.
16762 int TgtElemArrayIdx;
16763 int InputSize = Input.getValueType().getScalarSizeInBits();
16764 int OutputSize = N->getValueType(ResNo: 0).getScalarSizeInBits();
16765 if (InputSize + OutputSize == 40)
16766 TgtElemArrayIdx = 0;
16767 else if (InputSize + OutputSize == 72)
16768 TgtElemArrayIdx = 1;
16769 else if (InputSize + OutputSize == 48)
16770 TgtElemArrayIdx = 2;
16771 else if (InputSize + OutputSize == 80)
16772 TgtElemArrayIdx = 3;
16773 else if (InputSize + OutputSize == 96)
16774 TgtElemArrayIdx = 4;
16775 else
16776 return SDValue();
16777
16778 uint64_t CorrectElems = TargetElems[TgtElemArrayIdx];
16779 CorrectElems = DAG.getDataLayout().isLittleEndian()
16780 ? CorrectElems & 0x0F0F0F0F0F0F0F0F
16781 : CorrectElems & 0xF0F0F0F0F0F0F0F0;
16782 if (Elems != CorrectElems) {
16783 return addShuffleForVecExtend(N, DAG, Input, Elems, CorrectElems);
16784 }
16785
16786 // Regular lowering will catch cases where a shuffle is not needed.
16787 return SDValue();
16788}
16789
16790// Look for the pattern of a load from a narrow width to i128, feeding
16791// into a BUILD_VECTOR of v1i128. Replace this sequence with a PPCISD node
16792// (LXVRZX). This node represents a zero extending load that will be matched
16793// to the Load VSX Vector Rightmost instructions.
16794static SDValue combineBVZEXTLOAD(SDNode *N, SelectionDAG &DAG) {
16795 SDLoc DL(N);
16796
16797 // This combine is only eligible for a BUILD_VECTOR of v1i128.
16798 if (N->getValueType(ResNo: 0) != MVT::v1i128)
16799 return SDValue();
16800
16801 SDValue Operand = N->getOperand(Num: 0);
16802 // Proceed with the transformation if the operand to the BUILD_VECTOR
16803 // is a load instruction.
16804 if (Operand.getOpcode() != ISD::LOAD)
16805 return SDValue();
16806
16807 auto *LD = cast<LoadSDNode>(Val&: Operand);
16808 EVT MemoryType = LD->getMemoryVT();
16809
16810 // This transformation is only valid if the we are loading either a byte,
16811 // halfword, word, or doubleword.
16812 bool ValidLDType = MemoryType == MVT::i8 || MemoryType == MVT::i16 ||
16813 MemoryType == MVT::i32 || MemoryType == MVT::i64;
16814
16815 // Ensure that the load from the narrow width is being zero extended to i128.
16816 if (!ValidLDType ||
16817 (LD->getExtensionType() != ISD::ZEXTLOAD &&
16818 LD->getExtensionType() != ISD::EXTLOAD))
16819 return SDValue();
16820
16821 SDValue LoadOps[] = {
16822 LD->getChain(), LD->getBasePtr(),
16823 DAG.getIntPtrConstant(Val: MemoryType.getScalarSizeInBits(), DL)};
16824
16825 return DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVRZX, dl: DL,
16826 VTList: DAG.getVTList(VT1: MVT::v1i128, VT2: MVT::Other),
16827 Ops: LoadOps, MemVT: MemoryType, MMO: LD->getMemOperand());
16828}
16829
16830SDValue PPCTargetLowering::DAGCombineBuildVector(SDNode *N,
16831 DAGCombinerInfo &DCI) const {
16832 assert(N->getOpcode() == ISD::BUILD_VECTOR &&
16833 "Should be called with a BUILD_VECTOR node");
16834
16835 SelectionDAG &DAG = DCI.DAG;
16836 SDLoc dl(N);
16837
16838 if (!Subtarget.hasVSX())
16839 return SDValue();
16840
16841 // The target independent DAG combiner will leave a build_vector of
16842 // float-to-int conversions intact. We can generate MUCH better code for
16843 // a float-to-int conversion of a vector of floats.
16844 SDValue FirstInput = N->getOperand(Num: 0);
16845 if (FirstInput.getOpcode() == PPCISD::MFVSR) {
16846 SDValue Reduced = combineElementTruncationToVectorTruncation(N, DCI);
16847 if (Reduced)
16848 return Reduced;
16849 }
16850
16851 // If we're building a vector out of consecutive loads, just load that
16852 // vector type.
16853 SDValue Reduced = combineBVOfConsecutiveLoads(N, DAG);
16854 if (Reduced)
16855 return Reduced;
16856
16857 // If we're building a vector out of extended elements from another vector
16858 // we have P9 vector integer extend instructions. The code assumes legal
16859 // input types (i.e. it can't handle things like v4i16) so do not run before
16860 // legalization.
16861 if (Subtarget.hasP9Altivec() && !DCI.isBeforeLegalize()) {
16862 Reduced = combineBVOfVecSExt(N, DAG);
16863 if (Reduced)
16864 return Reduced;
16865 }
16866
16867 // On Power10, the Load VSX Vector Rightmost instructions can be utilized
16868 // if this is a BUILD_VECTOR of v1i128, and if the operand to the BUILD_VECTOR
16869 // is a load from <valid narrow width> to i128.
16870 if (Subtarget.isISA3_1()) {
16871 SDValue BVOfZLoad = combineBVZEXTLOAD(N, DAG);
16872 if (BVOfZLoad)
16873 return BVOfZLoad;
16874 }
16875
16876 if (N->getValueType(ResNo: 0) != MVT::v2f64)
16877 return SDValue();
16878
16879 // Looking for:
16880 // (build_vector ([su]int_to_fp (extractelt 0)), [su]int_to_fp (extractelt 1))
16881 if (FirstInput.getOpcode() != ISD::SINT_TO_FP &&
16882 FirstInput.getOpcode() != ISD::UINT_TO_FP)
16883 return SDValue();
16884 if (N->getOperand(Num: 1).getOpcode() != ISD::SINT_TO_FP &&
16885 N->getOperand(Num: 1).getOpcode() != ISD::UINT_TO_FP)
16886 return SDValue();
16887 if (FirstInput.getOpcode() != N->getOperand(Num: 1).getOpcode())
16888 return SDValue();
16889
16890 SDValue Ext1 = FirstInput.getOperand(i: 0);
16891 SDValue Ext2 = N->getOperand(Num: 1).getOperand(i: 0);
16892 if(Ext1.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
16893 Ext2.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16894 return SDValue();
16895
16896 ConstantSDNode *Ext1Op = dyn_cast<ConstantSDNode>(Val: Ext1.getOperand(i: 1));
16897 ConstantSDNode *Ext2Op = dyn_cast<ConstantSDNode>(Val: Ext2.getOperand(i: 1));
16898 if (!Ext1Op || !Ext2Op)
16899 return SDValue();
16900 if (Ext1.getOperand(i: 0).getValueType() != MVT::v4i32 ||
16901 Ext1.getOperand(i: 0) != Ext2.getOperand(i: 0))
16902 return SDValue();
16903
16904 int FirstElem = Ext1Op->getZExtValue();
16905 int SecondElem = Ext2Op->getZExtValue();
16906 int SubvecIdx;
16907 if (FirstElem == 0 && SecondElem == 1)
16908 SubvecIdx = Subtarget.isLittleEndian() ? 1 : 0;
16909 else if (FirstElem == 2 && SecondElem == 3)
16910 SubvecIdx = Subtarget.isLittleEndian() ? 0 : 1;
16911 else
16912 return SDValue();
16913
16914 SDValue SrcVec = Ext1.getOperand(i: 0);
16915 auto NodeType = (N->getOperand(Num: 1).getOpcode() == ISD::SINT_TO_FP) ?
16916 PPCISD::SINT_VEC_TO_FP : PPCISD::UINT_VEC_TO_FP;
16917 return DAG.getNode(Opcode: NodeType, DL: dl, VT: MVT::v2f64,
16918 N1: SrcVec, N2: DAG.getIntPtrConstant(Val: SubvecIdx, DL: dl));
16919}
16920
16921SDValue PPCTargetLowering::combineFPToIntToFP(SDNode *N,
16922 DAGCombinerInfo &DCI) const {
16923 assert((N->getOpcode() == ISD::SINT_TO_FP ||
16924 N->getOpcode() == ISD::UINT_TO_FP) &&
16925 "Need an int -> FP conversion node here");
16926
16927 if (useSoftFloat() || !Subtarget.has64BitSupport())
16928 return SDValue();
16929
16930 SelectionDAG &DAG = DCI.DAG;
16931 SDLoc dl(N);
16932 SDValue Op(N, 0);
16933
16934 // Don't handle ppc_fp128 here or conversions that are out-of-range capable
16935 // from the hardware.
16936 if (Op.getValueType() != MVT::f32 && Op.getValueType() != MVT::f64)
16937 return SDValue();
16938 if (!Op.getOperand(i: 0).getValueType().isSimple())
16939 return SDValue();
16940 if (Op.getOperand(i: 0).getValueType().getSimpleVT() <= MVT(MVT::i1) ||
16941 Op.getOperand(i: 0).getValueType().getSimpleVT() > MVT(MVT::i64))
16942 return SDValue();
16943
16944 SDValue FirstOperand(Op.getOperand(i: 0));
16945 bool SubWordLoad = FirstOperand.getOpcode() == ISD::LOAD &&
16946 (FirstOperand.getValueType() == MVT::i8 ||
16947 FirstOperand.getValueType() == MVT::i16);
16948 if (Subtarget.hasP9Vector() && Subtarget.hasP9Altivec() && SubWordLoad) {
16949 bool Signed = N->getOpcode() == ISD::SINT_TO_FP;
16950 bool DstDouble = Op.getValueType() == MVT::f64;
16951 unsigned ConvOp = Signed ?
16952 (DstDouble ? PPCISD::FCFID : PPCISD::FCFIDS) :
16953 (DstDouble ? PPCISD::FCFIDU : PPCISD::FCFIDUS);
16954 SDValue WidthConst =
16955 DAG.getIntPtrConstant(Val: FirstOperand.getValueType() == MVT::i8 ? 1 : 2,
16956 DL: dl, isTarget: false);
16957 LoadSDNode *LDN = cast<LoadSDNode>(Val: FirstOperand.getNode());
16958 SDValue Ops[] = { LDN->getChain(), LDN->getBasePtr(), WidthConst };
16959 SDValue Ld = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXSIZX, dl,
16960 VTList: DAG.getVTList(VT1: MVT::f64, VT2: MVT::Other),
16961 Ops, MemVT: MVT::i8, MMO: LDN->getMemOperand());
16962 DAG.makeEquivalentMemoryOrdering(OldLoad: LDN, NewMemOp: Ld);
16963
16964 // For signed conversion, we need to sign-extend the value in the VSR
16965 if (Signed) {
16966 SDValue ExtOps[] = { Ld, WidthConst };
16967 SDValue Ext = DAG.getNode(Opcode: PPCISD::VEXTS, DL: dl, VT: MVT::f64, Ops: ExtOps);
16968 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ext);
16969 } else
16970 return DAG.getNode(Opcode: ConvOp, DL: dl, VT: DstDouble ? MVT::f64 : MVT::f32, Operand: Ld);
16971 }
16972
16973
16974 // For i32 intermediate values, unfortunately, the conversion functions
16975 // leave the upper 32 bits of the value are undefined. Within the set of
16976 // scalar instructions, we have no method for zero- or sign-extending the
16977 // value. Thus, we cannot handle i32 intermediate values here.
16978 if (Op.getOperand(i: 0).getValueType() == MVT::i32)
16979 return SDValue();
16980
16981 assert((Op.getOpcode() == ISD::SINT_TO_FP || Subtarget.hasFPCVT()) &&
16982 "UINT_TO_FP is supported only with FPCVT");
16983
16984 // If we have FCFIDS, then use it when converting to single-precision.
16985 // Otherwise, convert to double-precision and then round.
16986 unsigned FCFOp = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
16987 ? (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDUS
16988 : PPCISD::FCFIDS)
16989 : (Op.getOpcode() == ISD::UINT_TO_FP ? PPCISD::FCFIDU
16990 : PPCISD::FCFID);
16991 MVT FCFTy = (Subtarget.hasFPCVT() && Op.getValueType() == MVT::f32)
16992 ? MVT::f32
16993 : MVT::f64;
16994
16995 // If we're converting from a float, to an int, and back to a float again,
16996 // then we don't need the store/load pair at all.
16997 if ((Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_UINT &&
16998 Subtarget.hasFPCVT()) ||
16999 (Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT)) {
17000 SDValue Src = Op.getOperand(i: 0).getOperand(i: 0);
17001 if (Src.getValueType() == MVT::f32) {
17002 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: dl, VT: MVT::f64, Operand: Src);
17003 DCI.AddToWorklist(N: Src.getNode());
17004 } else if (Src.getValueType() != MVT::f64) {
17005 // Make sure that we don't pick up a ppc_fp128 source value.
17006 return SDValue();
17007 }
17008
17009 unsigned FCTOp =
17010 Op.getOperand(i: 0).getOpcode() == ISD::FP_TO_SINT ? PPCISD::FCTIDZ :
17011 PPCISD::FCTIDUZ;
17012
17013 SDValue Tmp = DAG.getNode(Opcode: FCTOp, DL: dl, VT: MVT::f64, Operand: Src);
17014 SDValue FP = DAG.getNode(Opcode: FCFOp, DL: dl, VT: FCFTy, Operand: Tmp);
17015
17016 if (Op.getValueType() == MVT::f32 && !Subtarget.hasFPCVT()) {
17017 FP = DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT: MVT::f32, N1: FP,
17018 N2: DAG.getIntPtrConstant(Val: 0, DL: dl, /*isTarget=*/true));
17019 DCI.AddToWorklist(N: FP.getNode());
17020 }
17021
17022 return FP;
17023 }
17024
17025 return SDValue();
17026}
17027
17028// expandVSXLoadForLE - Convert VSX loads (which may be intrinsics for
17029// builtins) into loads with swaps.
17030SDValue PPCTargetLowering::expandVSXLoadForLE(SDNode *N,
17031 DAGCombinerInfo &DCI) const {
17032 // Delay VSX load for LE combine until after LegalizeOps to prioritize other
17033 // load combines.
17034 if (DCI.isBeforeLegalizeOps())
17035 return SDValue();
17036
17037 SelectionDAG &DAG = DCI.DAG;
17038 SDLoc dl(N);
17039 SDValue Chain;
17040 SDValue Base;
17041 MachineMemOperand *MMO;
17042
17043 switch (N->getOpcode()) {
17044 default:
17045 llvm_unreachable("Unexpected opcode for little endian VSX load");
17046 case ISD::LOAD: {
17047 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
17048 Chain = LD->getChain();
17049 Base = LD->getBasePtr();
17050 MMO = LD->getMemOperand();
17051 // If the MMO suggests this isn't a load of a full vector, leave
17052 // things alone. For a built-in, we have to make the change for
17053 // correctness, so if there is a size problem that will be a bug.
17054 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17055 return SDValue();
17056 break;
17057 }
17058 case ISD::INTRINSIC_W_CHAIN: {
17059 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17060 Chain = Intrin->getChain();
17061 // Similarly to the store case below, Intrin->getBasePtr() doesn't get
17062 // us what we want. Get operand 2 instead.
17063 Base = Intrin->getOperand(Num: 2);
17064 MMO = Intrin->getMemOperand();
17065 break;
17066 }
17067 }
17068
17069 MVT VecTy = N->getValueType(ResNo: 0).getSimpleVT();
17070
17071 SDValue LoadOps[] = { Chain, Base };
17072 SDValue Load = DAG.getMemIntrinsicNode(Opcode: PPCISD::LXVD2X, dl,
17073 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other),
17074 Ops: LoadOps, MemVT: MVT::v2f64, MMO);
17075
17076 DCI.AddToWorklist(N: Load.getNode());
17077 Chain = Load.getValue(R: 1);
17078 SDValue Swap = DAG.getNode(
17079 Opcode: PPCISD::XXSWAPD, DL: dl, VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Load);
17080 DCI.AddToWorklist(N: Swap.getNode());
17081
17082 // Add a bitcast if the resulting load type doesn't match v2f64.
17083 if (VecTy != MVT::v2f64) {
17084 SDValue N = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: VecTy, Operand: Swap);
17085 DCI.AddToWorklist(N: N.getNode());
17086 // Package {bitcast value, swap's chain} to match Load's shape.
17087 return DAG.getNode(Opcode: ISD::MERGE_VALUES, DL: dl, VTList: DAG.getVTList(VT1: VecTy, VT2: MVT::Other),
17088 N1: N, N2: Swap.getValue(R: 1));
17089 }
17090
17091 return Swap;
17092}
17093
17094// expandVSXStoreForLE - Convert VSX stores (which may be intrinsics for
17095// builtins) into stores with swaps.
17096SDValue PPCTargetLowering::expandVSXStoreForLE(SDNode *N,
17097 DAGCombinerInfo &DCI) const {
17098 // Delay VSX store for LE combine until after LegalizeOps to prioritize other
17099 // store combines.
17100 if (DCI.isBeforeLegalizeOps())
17101 return SDValue();
17102
17103 SelectionDAG &DAG = DCI.DAG;
17104 SDLoc dl(N);
17105 SDValue Chain;
17106 SDValue Base;
17107 unsigned SrcOpnd;
17108 MachineMemOperand *MMO;
17109
17110 switch (N->getOpcode()) {
17111 default:
17112 llvm_unreachable("Unexpected opcode for little endian VSX store");
17113 case ISD::STORE: {
17114 StoreSDNode *ST = cast<StoreSDNode>(Val: N);
17115 Chain = ST->getChain();
17116 Base = ST->getBasePtr();
17117 MMO = ST->getMemOperand();
17118 SrcOpnd = 1;
17119 // If the MMO suggests this isn't a store of a full vector, leave
17120 // things alone. For a built-in, we have to make the change for
17121 // correctness, so if there is a size problem that will be a bug.
17122 if (!MMO->getSize().hasValue() || MMO->getSize().getValue() < 16)
17123 return SDValue();
17124 break;
17125 }
17126 case ISD::INTRINSIC_VOID: {
17127 MemIntrinsicSDNode *Intrin = cast<MemIntrinsicSDNode>(Val: N);
17128 Chain = Intrin->getChain();
17129 // Intrin->getBasePtr() oddly does not get what we want.
17130 Base = Intrin->getOperand(Num: 3);
17131 MMO = Intrin->getMemOperand();
17132 SrcOpnd = 2;
17133 break;
17134 }
17135 }
17136
17137 SDValue Src = N->getOperand(Num: SrcOpnd);
17138 MVT VecTy = Src.getValueType().getSimpleVT();
17139
17140 // All stores are done as v2f64 and possible bit cast.
17141 if (VecTy != MVT::v2f64) {
17142 Src = DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT: MVT::v2f64, Operand: Src);
17143 DCI.AddToWorklist(N: Src.getNode());
17144 }
17145
17146 SDValue Swap = DAG.getNode(Opcode: PPCISD::XXSWAPD, DL: dl,
17147 VTList: DAG.getVTList(VT1: MVT::v2f64, VT2: MVT::Other), N1: Chain, N2: Src);
17148 DCI.AddToWorklist(N: Swap.getNode());
17149 Chain = Swap.getValue(R: 1);
17150 SDValue StoreOps[] = { Chain, Swap, Base };
17151 SDValue Store = DAG.getMemIntrinsicNode(Opcode: PPCISD::STXVD2X, dl,
17152 VTList: DAG.getVTList(VT: MVT::Other),
17153 Ops: StoreOps, MemVT: VecTy, MMO);
17154 DCI.AddToWorklist(N: Store.getNode());
17155 return Store;
17156}
17157
17158// Handle DAG combine for STORE (FP_TO_INT F).
17159SDValue PPCTargetLowering::combineStoreFPToInt(SDNode *N,
17160 DAGCombinerInfo &DCI) const {
17161 SelectionDAG &DAG = DCI.DAG;
17162 SDLoc dl(N);
17163 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
17164 (void)Opcode;
17165 bool Strict = N->getOperand(Num: 1)->isStrictFPOpcode();
17166
17167 assert((Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
17168 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT)
17169 && "Not a FP_TO_INT Instruction!");
17170
17171 SDValue Val = N->getOperand(Num: 1).getOperand(i: Strict ? 1 : 0);
17172 EVT Op1VT = N->getOperand(Num: 1).getValueType();
17173 EVT ResVT = Val.getValueType();
17174
17175 if (!Subtarget.hasVSX() || !Subtarget.hasFPCVT() || !isTypeLegal(VT: ResVT))
17176 return SDValue();
17177
17178 // Only perform combine for conversion to i64/i32 or power9 i16/i8.
17179 bool ValidTypeForStoreFltAsInt =
17180 (Op1VT == MVT::i32 || (Op1VT == MVT::i64 && Subtarget.isPPC64()) ||
17181 (Subtarget.hasP9Vector() && (Op1VT == MVT::i16 || Op1VT == MVT::i8)));
17182
17183 // TODO: Lower conversion from f128 on all VSX targets
17184 if (ResVT == MVT::ppcf128 || (ResVT == MVT::f128 && !Subtarget.hasP9Vector()))
17185 return SDValue();
17186
17187 if ((Op1VT != MVT::i64 && !Subtarget.hasP8Vector()) ||
17188 cast<StoreSDNode>(Val: N)->isTruncatingStore() || !ValidTypeForStoreFltAsInt)
17189 return SDValue();
17190
17191 Val = convertFPToInt(Op: N->getOperand(Num: 1), DAG, Subtarget);
17192
17193 // Set number of bytes being converted.
17194 unsigned ByteSize = Op1VT.getScalarSizeInBits() / 8;
17195 SDValue Ops[] = {N->getOperand(Num: 0), Val, N->getOperand(Num: 2),
17196 DAG.getIntPtrConstant(Val: ByteSize, DL: dl, isTarget: false),
17197 DAG.getValueType(Op1VT)};
17198
17199 Val = DAG.getMemIntrinsicNode(Opcode: PPCISD::ST_VSR_SCAL_INT, dl,
17200 VTList: DAG.getVTList(VT: MVT::Other), Ops,
17201 MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
17202 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
17203
17204 return Val;
17205}
17206
17207static bool isAlternatingShuffMask(const ArrayRef<int> &Mask, int NumElts) {
17208 // Check that the source of the element keeps flipping
17209 // (i.e. Mask[i] < NumElts -> Mask[i+i] >= NumElts).
17210 bool PrevElemFromFirstVec = Mask[0] < NumElts;
17211 for (int i = 1, e = Mask.size(); i < e; i++) {
17212 if (PrevElemFromFirstVec && Mask[i] < NumElts)
17213 return false;
17214 if (!PrevElemFromFirstVec && Mask[i] >= NumElts)
17215 return false;
17216 PrevElemFromFirstVec = !PrevElemFromFirstVec;
17217 }
17218 return true;
17219}
17220
17221static bool isSplatBV(SDValue Op) {
17222 if (Op.getOpcode() != ISD::BUILD_VECTOR)
17223 return false;
17224 SDValue FirstOp;
17225
17226 // Find first non-undef input.
17227 for (int i = 0, e = Op.getNumOperands(); i < e; i++) {
17228 FirstOp = Op.getOperand(i);
17229 if (!FirstOp.isUndef())
17230 break;
17231 }
17232
17233 // All inputs are undef or the same as the first non-undef input.
17234 for (int i = 1, e = Op.getNumOperands(); i < e; i++)
17235 if (Op.getOperand(i) != FirstOp && !Op.getOperand(i).isUndef())
17236 return false;
17237 return true;
17238}
17239
17240static SDValue isScalarToVec(SDValue Op) {
17241 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17242 return Op;
17243 if (Op.getOpcode() != ISD::BITCAST)
17244 return SDValue();
17245 Op = Op.getOperand(i: 0);
17246 if (Op.getOpcode() == ISD::SCALAR_TO_VECTOR)
17247 return Op;
17248 return SDValue();
17249}
17250
17251// Fix up the shuffle mask to account for the fact that the result of
17252// scalar_to_vector is not in lane zero. This just takes all values in
17253// the ranges specified by the min/max indices and adds the number of
17254// elements required to ensure each element comes from the respective
17255// position in the valid lane.
17256// On little endian, that's just the corresponding element in the other
17257// half of the vector. On big endian, it is in the same half but right
17258// justified rather than left justified in that half.
17259static void fixupShuffleMaskForPermutedSToV(
17260 SmallVectorImpl<int> &ShuffV, int LHSFirstElt, int LHSLastElt,
17261 int RHSFirstElt, int RHSLastElt, int HalfVec, unsigned LHSNumValidElts,
17262 unsigned RHSNumValidElts, const PPCSubtarget &Subtarget) {
17263 int LHSEltFixup =
17264 Subtarget.isLittleEndian() ? HalfVec : HalfVec - LHSNumValidElts;
17265 int RHSEltFixup =
17266 Subtarget.isLittleEndian() ? HalfVec : HalfVec - RHSNumValidElts;
17267 for (int I = 0, E = ShuffV.size(); I < E; ++I) {
17268 int Idx = ShuffV[I];
17269 if (Idx >= LHSFirstElt && Idx <= LHSLastElt)
17270 ShuffV[I] += LHSEltFixup;
17271 else if (Idx >= RHSFirstElt && Idx <= RHSLastElt)
17272 ShuffV[I] += RHSEltFixup;
17273 }
17274}
17275
17276// Replace a SCALAR_TO_VECTOR with a SCALAR_TO_VECTOR_PERMUTED except if
17277// the original is:
17278// (<n x Ty> (scalar_to_vector (Ty (extract_elt <n x Ty> %a, C))))
17279// In such a case, just change the shuffle mask to extract the element
17280// from the permuted index.
17281static SDValue getSToVPermuted(SDValue OrigSToV, SelectionDAG &DAG,
17282 const PPCSubtarget &Subtarget) {
17283 SDLoc dl(OrigSToV);
17284 EVT VT = OrigSToV.getValueType();
17285 assert(OrigSToV.getOpcode() == ISD::SCALAR_TO_VECTOR &&
17286 "Expecting a SCALAR_TO_VECTOR here");
17287 SDValue Input = OrigSToV.getOperand(i: 0);
17288
17289 if (Input.getOpcode() == ISD::EXTRACT_VECTOR_ELT) {
17290 ConstantSDNode *Idx = dyn_cast<ConstantSDNode>(Val: Input.getOperand(i: 1));
17291 SDValue OrigVector = Input.getOperand(i: 0);
17292
17293 // Can't handle non-const element indices or different vector types
17294 // for the input to the extract and the output of the scalar_to_vector.
17295 if (Idx && VT == OrigVector.getValueType()) {
17296 unsigned NumElts = VT.getVectorNumElements();
17297 assert(
17298 NumElts > 1 &&
17299 "Cannot produce a permuted scalar_to_vector for one element vector");
17300 SmallVector<int, 16> NewMask(NumElts, -1);
17301 unsigned ResultInElt = NumElts / 2;
17302 ResultInElt -= Subtarget.isLittleEndian() ? 0 : 1;
17303 NewMask[ResultInElt] = Idx->getZExtValue();
17304 return DAG.getVectorShuffle(VT, dl, N1: OrigVector, N2: OrigVector, Mask: NewMask);
17305 }
17306 }
17307 return DAG.getNode(Opcode: PPCISD::SCALAR_TO_VECTOR_PERMUTED, DL: dl, VT,
17308 Operand: OrigSToV.getOperand(i: 0));
17309}
17310
17311static bool isShuffleMaskInRange(const SmallVectorImpl<int> &ShuffV,
17312 int HalfVec, int LHSLastElementDefined,
17313 int RHSLastElementDefined) {
17314 for (int Index : ShuffV) {
17315 if (Index < 0) // Skip explicitly undefined mask indices.
17316 continue;
17317 // Handle first input vector of the vector_shuffle.
17318 if ((LHSLastElementDefined >= 0) && (Index < HalfVec) &&
17319 (Index > LHSLastElementDefined))
17320 return false;
17321 // Handle second input vector of the vector_shuffle.
17322 if ((RHSLastElementDefined >= 0) &&
17323 (Index > HalfVec + RHSLastElementDefined))
17324 return false;
17325 }
17326 return true;
17327}
17328
17329static SDValue generateSToVPermutedForVecShuffle(
17330 int ScalarSize, uint64_t ShuffleEltWidth, unsigned &NumValidElts,
17331 int FirstElt, int &LastElt, SDValue VecShuffOperand, SDValue SToVNode,
17332 SelectionDAG &DAG, const PPCSubtarget &Subtarget) {
17333 EVT VecShuffOperandType = VecShuffOperand.getValueType();
17334 // Set up the values for the shuffle vector fixup.
17335 NumValidElts = ScalarSize / VecShuffOperandType.getScalarSizeInBits();
17336 // The last element depends on if the input comes from the LHS or RHS.
17337 //
17338 // For example:
17339 // (shuff (s_to_v i32), (bitcast (s_to_v i64), v4i32), ...)
17340 //
17341 // For the LHS: The last element that comes from the LHS is actually 0, not 3
17342 // because elements 1 and higher of a scalar_to_vector are undefined.
17343 // For the RHS: The last element that comes from the RHS is actually 5, not 7
17344 // because elements 1 and higher of a scalar_to_vector are undefined.
17345 // It is also not 4 because the original scalar_to_vector is wider and
17346 // actually contains two i32 elements.
17347 LastElt = (uint64_t)ScalarSize > ShuffleEltWidth
17348 ? ScalarSize / ShuffleEltWidth - 1 + FirstElt
17349 : FirstElt;
17350 SDValue SToVPermuted = getSToVPermuted(OrigSToV: SToVNode, DAG, Subtarget);
17351 if (SToVPermuted.getValueType() != VecShuffOperandType)
17352 SToVPermuted = DAG.getBitcast(VT: VecShuffOperandType, V: SToVPermuted);
17353 return SToVPermuted;
17354}
17355
17356// On little endian subtargets, combine shuffles such as:
17357// vector_shuffle<16,1,17,3,18,5,19,7,20,9,21,11,22,13,23,15>, <zero>, %b
17358// into:
17359// vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7>, <zero>, %b
17360// because the latter can be matched to a single instruction merge.
17361// Furthermore, SCALAR_TO_VECTOR on little endian always involves a permute
17362// to put the value into element zero. Adjust the shuffle mask so that the
17363// vector can remain in permuted form (to prevent a swap prior to a shuffle).
17364// On big endian targets, this is still useful for SCALAR_TO_VECTOR
17365// nodes with elements smaller than doubleword because all the ways
17366// of getting scalar data into a vector register put the value in the
17367// rightmost element of the left half of the vector.
17368SDValue PPCTargetLowering::combineVectorShuffle(ShuffleVectorSDNode *SVN,
17369 SelectionDAG &DAG) const {
17370 SDValue LHS = SVN->getOperand(Num: 0);
17371 SDValue RHS = SVN->getOperand(Num: 1);
17372 auto Mask = SVN->getMask();
17373 int NumElts = LHS.getValueType().getVectorNumElements();
17374 SDValue Res(SVN, 0);
17375 SDLoc dl(SVN);
17376 bool IsLittleEndian = Subtarget.isLittleEndian();
17377
17378 // On big endian targets this is only useful for subtargets with direct moves.
17379 // On little endian targets it would be useful for all subtargets with VSX.
17380 // However adding special handling for LE subtargets without direct moves
17381 // would be wasted effort since the minimum arch for LE is ISA 2.07 (Power8)
17382 // which includes direct moves.
17383 if (!Subtarget.hasDirectMove())
17384 return Res;
17385
17386 // If this is not a shuffle of a shuffle and the first element comes from
17387 // the second vector, canonicalize to the commuted form. This will make it
17388 // more likely to match one of the single instruction patterns.
17389 if (Mask[0] >= NumElts && LHS.getOpcode() != ISD::VECTOR_SHUFFLE &&
17390 RHS.getOpcode() != ISD::VECTOR_SHUFFLE) {
17391 std::swap(a&: LHS, b&: RHS);
17392 Res = DAG.getCommutedVectorShuffle(SV: *SVN);
17393
17394 if (!isa<ShuffleVectorSDNode>(Val: Res))
17395 return Res;
17396
17397 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17398 }
17399
17400 // Adjust the shuffle mask if either input vector comes from a
17401 // SCALAR_TO_VECTOR and keep the respective input vector in permuted
17402 // form (to prevent the need for a swap).
17403 SmallVector<int, 16> ShuffV(Mask);
17404 SDValue SToVLHS = isScalarToVec(Op: LHS);
17405 SDValue SToVRHS = isScalarToVec(Op: RHS);
17406 if (SToVLHS || SToVRHS) {
17407 EVT VT = SVN->getValueType(ResNo: 0);
17408 uint64_t ShuffleEltWidth = VT.getVectorElementType().getSizeInBits();
17409 int ShuffleNumElts = ShuffV.size();
17410 int HalfVec = ShuffleNumElts / 2;
17411 // The width of the "valid lane" (i.e. the lane that contains the value that
17412 // is vectorized) needs to be expressed in terms of the number of elements
17413 // of the shuffle. It is thereby the ratio of the values before and after
17414 // any bitcast, which will be set later on if the LHS or RHS are
17415 // SCALAR_TO_VECTOR nodes.
17416 unsigned LHSNumValidElts = HalfVec;
17417 unsigned RHSNumValidElts = HalfVec;
17418
17419 // Initially assume that neither input is permuted. These will be adjusted
17420 // accordingly if either input is. Note, that -1 means that all elements
17421 // are undefined.
17422 int LHSFirstElt = 0;
17423 int RHSFirstElt = ShuffleNumElts;
17424 int LHSLastElt = -1;
17425 int RHSLastElt = -1;
17426
17427 // Get the permuted scalar to vector nodes for the source(s) that come from
17428 // ISD::SCALAR_TO_VECTOR.
17429 // On big endian systems, this only makes sense for element sizes smaller
17430 // than 64 bits since for 64-bit elements, all instructions already put
17431 // the value into element zero. Since scalar size of LHS and RHS may differ
17432 // after isScalarToVec, this should be checked using their own sizes.
17433 int LHSScalarSize = 0;
17434 int RHSScalarSize = 0;
17435 if (SToVLHS) {
17436 LHSScalarSize = SToVLHS.getValueType().getScalarSizeInBits();
17437 if (!IsLittleEndian && LHSScalarSize >= 64)
17438 return Res;
17439 }
17440 if (SToVRHS) {
17441 RHSScalarSize = SToVRHS.getValueType().getScalarSizeInBits();
17442 if (!IsLittleEndian && RHSScalarSize >= 64)
17443 return Res;
17444 }
17445 if (LHSScalarSize != 0)
17446 LHS = generateSToVPermutedForVecShuffle(
17447 ScalarSize: LHSScalarSize, ShuffleEltWidth, NumValidElts&: LHSNumValidElts, FirstElt: LHSFirstElt,
17448 LastElt&: LHSLastElt, VecShuffOperand: LHS, SToVNode: SToVLHS, DAG, Subtarget);
17449 if (RHSScalarSize != 0)
17450 RHS = generateSToVPermutedForVecShuffle(
17451 ScalarSize: RHSScalarSize, ShuffleEltWidth, NumValidElts&: RHSNumValidElts, FirstElt: RHSFirstElt,
17452 LastElt&: RHSLastElt, VecShuffOperand: RHS, SToVNode: SToVRHS, DAG, Subtarget);
17453
17454 if (!isShuffleMaskInRange(ShuffV, HalfVec, LHSLastElementDefined: LHSLastElt, RHSLastElementDefined: RHSLastElt))
17455 return Res;
17456
17457 // Fix up the shuffle mask to reflect where the desired element actually is.
17458 // The minimum and maximum indices that correspond to element zero for both
17459 // the LHS and RHS are computed and will control which shuffle mask entries
17460 // are to be changed. For example, if the RHS is permuted, any shuffle mask
17461 // entries in the range [RHSFirstElt,RHSLastElt] will be adjusted.
17462 fixupShuffleMaskForPermutedSToV(
17463 ShuffV, LHSFirstElt, LHSLastElt, RHSFirstElt, RHSLastElt, HalfVec,
17464 LHSNumValidElts, RHSNumValidElts, Subtarget);
17465 Res = DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17466
17467 // We may have simplified away the shuffle. We won't be able to do anything
17468 // further with it here.
17469 if (!isa<ShuffleVectorSDNode>(Val: Res))
17470 return Res;
17471 Mask = cast<ShuffleVectorSDNode>(Val&: Res)->getMask();
17472 }
17473
17474 SDValue TheSplat = IsLittleEndian ? RHS : LHS;
17475 // The common case after we commuted the shuffle is that the RHS is a splat
17476 // and we have elements coming in from the splat at indices that are not
17477 // conducive to using a merge.
17478 // Example:
17479 // vector_shuffle<0,17,1,19,2,21,3,23,4,25,5,27,6,29,7,31> t1, <zero>
17480 if (!isSplatBV(Op: TheSplat))
17481 return Res;
17482
17483 // We are looking for a mask such that all even elements are from
17484 // one vector and all odd elements from the other.
17485 if (!isAlternatingShuffMask(Mask, NumElts))
17486 return Res;
17487
17488 // Adjust the mask so we are pulling in the same index from the splat
17489 // as the index from the interesting vector in consecutive elements.
17490 if (IsLittleEndian) {
17491 // Example (even elements from first vector):
17492 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> t1, <zero>
17493 if (Mask[0] < NumElts)
17494 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17495 if (ShuffV[i] < 0)
17496 continue;
17497 // If element from non-splat is undef, pick first element from splat.
17498 ShuffV[i] = (ShuffV[i - 1] >= 0 ? ShuffV[i - 1] : 0) + NumElts;
17499 }
17500 // Example (odd elements from first vector):
17501 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> t1, <zero>
17502 else
17503 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17504 if (ShuffV[i] < 0)
17505 continue;
17506 // If element from non-splat is undef, pick first element from splat.
17507 ShuffV[i] = (ShuffV[i + 1] >= 0 ? ShuffV[i + 1] : 0) + NumElts;
17508 }
17509 } else {
17510 // Example (even elements from first vector):
17511 // vector_shuffle<0,16,1,17,2,18,3,19,4,20,5,21,6,22,7,23> <zero>, t1
17512 if (Mask[0] < NumElts)
17513 for (int i = 0, e = Mask.size(); i < e; i += 2) {
17514 if (ShuffV[i] < 0)
17515 continue;
17516 // If element from non-splat is undef, pick first element from splat.
17517 ShuffV[i] = ShuffV[i + 1] >= 0 ? ShuffV[i + 1] - NumElts : 0;
17518 }
17519 // Example (odd elements from first vector):
17520 // vector_shuffle<16,0,17,1,18,2,19,3,20,4,21,5,22,6,23,7> <zero>, t1
17521 else
17522 for (int i = 1, e = Mask.size(); i < e; i += 2) {
17523 if (ShuffV[i] < 0)
17524 continue;
17525 // If element from non-splat is undef, pick first element from splat.
17526 ShuffV[i] = ShuffV[i - 1] >= 0 ? ShuffV[i - 1] - NumElts : 0;
17527 }
17528 }
17529
17530 // If the RHS has undefs, we need to remove them since we may have created
17531 // a shuffle that adds those instead of the splat value.
17532 SDValue SplatVal =
17533 cast<BuildVectorSDNode>(Val: TheSplat.getNode())->getSplatValue();
17534 TheSplat = DAG.getSplatBuildVector(VT: TheSplat.getValueType(), DL: dl, Op: SplatVal);
17535
17536 if (IsLittleEndian)
17537 RHS = TheSplat;
17538 else
17539 LHS = TheSplat;
17540 return DAG.getVectorShuffle(VT: SVN->getValueType(ResNo: 0), dl, N1: LHS, N2: RHS, Mask: ShuffV);
17541}
17542
17543SDValue PPCTargetLowering::combineVReverseMemOP(ShuffleVectorSDNode *SVN,
17544 LSBaseSDNode *LSBase,
17545 DAGCombinerInfo &DCI) const {
17546 assert((ISD::isNormalLoad(LSBase) || ISD::isNormalStore(LSBase)) &&
17547 "Not a reverse memop pattern!");
17548
17549 auto IsElementReverse = [](const ShuffleVectorSDNode *SVN) -> bool {
17550 auto Mask = SVN->getMask();
17551 int i = 0;
17552 auto I = Mask.rbegin();
17553 auto E = Mask.rend();
17554
17555 for (; I != E; ++I) {
17556 if (*I != i)
17557 return false;
17558 i++;
17559 }
17560 return true;
17561 };
17562
17563 SelectionDAG &DAG = DCI.DAG;
17564 EVT VT = SVN->getValueType(ResNo: 0);
17565
17566 if (!isTypeLegal(VT) || !Subtarget.isLittleEndian() || !Subtarget.hasVSX())
17567 return SDValue();
17568
17569 // Before P9, we have PPCVSXSwapRemoval pass to hack the element order.
17570 // See comment in PPCVSXSwapRemoval.cpp.
17571 // It is conflict with PPCVSXSwapRemoval opt. So we don't do it.
17572 if (!Subtarget.hasP9Vector())
17573 return SDValue();
17574
17575 if(!IsElementReverse(SVN))
17576 return SDValue();
17577
17578 if (LSBase->getOpcode() == ISD::LOAD) {
17579 // If the load return value 0 has more than one user except the
17580 // shufflevector instruction, it is not profitable to replace the
17581 // shufflevector with a reverse load.
17582 for (SDUse &Use : LSBase->uses())
17583 if (Use.getResNo() == 0 &&
17584 Use.getUser()->getOpcode() != ISD::VECTOR_SHUFFLE)
17585 return SDValue();
17586
17587 SDLoc dl(LSBase);
17588 SDValue LoadOps[] = {LSBase->getChain(), LSBase->getBasePtr()};
17589 return DAG.getMemIntrinsicNode(
17590 Opcode: PPCISD::LOAD_VEC_BE, dl, VTList: DAG.getVTList(VT1: VT, VT2: MVT::Other), Ops: LoadOps,
17591 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17592 }
17593
17594 if (LSBase->getOpcode() == ISD::STORE) {
17595 // If there are other uses of the shuffle, the swap cannot be avoided.
17596 // Forcing the use of an X-Form (since swapped stores only have
17597 // X-Forms) without removing the swap is unprofitable.
17598 if (!SVN->hasOneUse())
17599 return SDValue();
17600
17601 SDLoc dl(LSBase);
17602 SDValue StoreOps[] = {LSBase->getChain(), SVN->getOperand(Num: 0),
17603 LSBase->getBasePtr()};
17604 return DAG.getMemIntrinsicNode(
17605 Opcode: PPCISD::STORE_VEC_BE, dl, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps,
17606 MemVT: LSBase->getMemoryVT(), MMO: LSBase->getMemOperand());
17607 }
17608
17609 llvm_unreachable("Expected a load or store node here");
17610}
17611
17612static bool isStoreConditional(SDValue Intrin, unsigned &StoreWidth) {
17613 unsigned IntrinsicID = Intrin.getConstantOperandVal(i: 1);
17614 if (IntrinsicID == Intrinsic::ppc_stdcx)
17615 StoreWidth = 8;
17616 else if (IntrinsicID == Intrinsic::ppc_stwcx)
17617 StoreWidth = 4;
17618 else if (IntrinsicID == Intrinsic::ppc_sthcx)
17619 StoreWidth = 2;
17620 else if (IntrinsicID == Intrinsic::ppc_stbcx)
17621 StoreWidth = 1;
17622 else
17623 return false;
17624 return true;
17625}
17626
17627static SDValue DAGCombineAddc(SDNode *N,
17628 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17629 if (N->getOpcode() == PPCISD::ADDC && N->hasAnyUseOfValue(Value: 1)) {
17630 // (ADDC (ADDE 0, 0, C), -1) -> C
17631 SDValue LHS = N->getOperand(Num: 0);
17632 SDValue RHS = N->getOperand(Num: 1);
17633 if (LHS->getOpcode() == PPCISD::ADDE &&
17634 isNullConstant(V: LHS->getOperand(Num: 0)) &&
17635 isNullConstant(V: LHS->getOperand(Num: 1)) && isAllOnesConstant(V: RHS)) {
17636 return DCI.CombineTo(N, Res0: SDValue(N, 0), Res1: LHS->getOperand(Num: 2));
17637 }
17638 }
17639 return SDValue();
17640}
17641
17642static SDValue DAGCombineSube(SDNode *N,
17643 llvm::PPCTargetLowering::DAGCombinerInfo &DCI) {
17644 if (N->getOpcode() == PPCISD::SUBE) {
17645 SDValue LHS = N->getOperand(Num: 0);
17646 SDValue RHS = N->getOperand(Num: 1);
17647 SDValue Carry = N->getOperand(Num: 2);
17648
17649 // SUBE(ADDC(ADDE(0, 0, C), -1), same, C) -> ADDC(ADDE(0, 0, C), -1)
17650 if (LHS == RHS && LHS.getOpcode() == PPCISD::ADDC) {
17651 SDValue AddcLHS = LHS.getOperand(i: 0);
17652 SDValue AddcRHS = LHS.getOperand(i: 1);
17653 if (AddcLHS.getOpcode() == PPCISD::ADDE &&
17654 isNullConstant(V: AddcLHS.getOperand(i: 0)) &&
17655 isNullConstant(V: AddcLHS.getOperand(i: 1)) && isAllOnesConstant(V: AddcRHS) &&
17656 Carry == AddcLHS.getOperand(i: 2)) {
17657 return LHS;
17658 }
17659 }
17660 }
17661 return SDValue();
17662}
17663
17664/// Optimize the bitfloor(X) pattern for PowerPC.
17665/// Transforms: select_cc X, 0, 0, (srl MinSignedValue, (ctlz X)), seteq
17666/// Into: srl MinSignedValue, (ctlz X)
17667///
17668/// This is safe on PowerPC because the srw instruction returns 0 when the
17669/// shift amount is == bitwidth, which matches the behavior we need for X=0.
17670static SDValue combineSELECT_CCBitFloor(SDNode *N, SelectionDAG &DAG) {
17671 if (N->getOpcode() != ISD::SELECT_CC)
17672 return SDValue();
17673
17674 // SELECT_CC operands: LHS, RHS, TrueVal, FalseVal, CC
17675 SDValue CmpLHS = N->getOperand(Num: 0);
17676 SDValue CmpRHS = N->getOperand(Num: 1);
17677 SDValue TrueVal = N->getOperand(Num: 2);
17678 SDValue FalseVal = N->getOperand(Num: 3);
17679 ISD::CondCode CC = N->getOperandCondCode(Num: 4);
17680
17681 // Check if condition is (X == 0)
17682 if (CC != ISD::SETEQ || !isNullConstant(V: CmpRHS))
17683 return SDValue();
17684
17685 // Check if TrueVal is constant 0
17686 if (!isNullConstant(V: TrueVal))
17687 return SDValue();
17688
17689 // This combine is replacing a select_cc with a PPC srl, not an srl with a
17690 // PPC srl. If the original srl had multiple uses it would just remain in the
17691 // code. This is at most a performance consideration.
17692 if (FalseVal.getOpcode() != ISD::SRL || !FalseVal.hasOneUse())
17693 return SDValue();
17694
17695 SDValue ShiftVal = FalseVal.getOperand(i: 0);
17696 SDValue ShiftAmt = FalseVal.getOperand(i: 1);
17697
17698 // Check if ShiftVal is MinSignedValue
17699 auto *ShiftConst = dyn_cast<ConstantSDNode>(Val&: ShiftVal);
17700 if (!ShiftConst || !ShiftConst->getAPIntValue().isMinSignedValue())
17701 return SDValue();
17702
17703 SDValue CtlzArg;
17704 // Check if ShiftAmt is (ctlz CmpLHS) or (truncate (ctlz ...))
17705 if (ShiftAmt.getOpcode() != ISD::CTLZ) {
17706 // Look through truncate if present (for i64 ctlz truncated to i32 shift
17707 // amount)
17708 if (ShiftAmt.getOpcode() != ISD::TRUNCATE)
17709 return SDValue();
17710
17711 // Verify the truncate target type is appropriate for shift amount (i32, not
17712 // i1 or other)
17713 if (ShiftAmt.getValueType() != MVT::i32)
17714 return SDValue();
17715
17716 SDValue CtlzNode = ShiftAmt.getOperand(i: 0);
17717
17718 if (CtlzNode.getOpcode() != ISD::CTLZ)
17719 return SDValue();
17720
17721 CtlzArg = CtlzNode.getOperand(i: 0);
17722 } else {
17723 CtlzArg = ShiftAmt.getOperand(i: 0);
17724 }
17725
17726 // Check if ctlz operates on the same value as the comparison
17727 if (CtlzArg != CmpLHS)
17728 return SDValue();
17729
17730 // Using PPCISD::SRL to ensure well-defined behavior.
17731 // On PowerPC, PPCISD::SRL guarantees that shift by bitwidth returns 0,
17732 // which is exactly what we need for the bitfloor(0) case.
17733 SDLoc DL(N);
17734 SDValue PPCSrl =
17735 DAG.getNode(Opcode: PPCISD::SRL, DL, VT: FalseVal.getValueType(), N1: ShiftVal, N2: ShiftAmt);
17736 return PPCSrl;
17737}
17738
17739// Optimize zero-extension of setcc when the compared value is known to be 0
17740// or 1.
17741//
17742// Pattern: zext(setcc(Value, 0, seteq/setne)) where Value is 0 or 1
17743// -> zext(xor(Value, 1)) for seteq
17744// -> zext(Value) for setne
17745//
17746// This optimization avoids the i32 -> i1 -> i32/i64 conversion sequence
17747// by keeping the value in its original i32 type throughout.
17748//
17749// Example:
17750// Before: zext(setcc(test_data_class(...), 0, seteq))
17751// // test_data_class returns 0 or 1 in i32
17752// // setcc converts i32 -> i1
17753// // zext converts i1 -> i64
17754// After: zext(xor(test_data_class(...), 1))
17755// // Stays in i32, then extends to i64
17756//
17757// This is beneficial because:
17758// 1. Eliminates the setcc instruction
17759// 2. Avoids i32 -> i1 truncation
17760// 3. Keeps computation in native integer width
17761
17762static SDValue combineZextSetccWithZero(SDNode *N, SelectionDAG &DAG) {
17763 // Check if this is a zero_extend
17764 if (N->getOpcode() != ISD::ZERO_EXTEND)
17765 return SDValue();
17766
17767 SDValue Src = N->getOperand(Num: 0);
17768
17769 // Check if the source is a setcc
17770 if (Src.getOpcode() != ISD::SETCC)
17771 return SDValue();
17772
17773 SDValue LHS = Src.getOperand(i: 0);
17774 SDValue RHS = Src.getOperand(i: 1);
17775 ISD::CondCode CC = Src.getOperandCondCode(i: 2);
17776
17777 if (!isNullConstant(V: RHS) && !isNullConstant(V: LHS))
17778 return SDValue();
17779
17780 SDValue NonNullConstant = isNullConstant(V: RHS) ? LHS : RHS;
17781
17782 auto isZeroOrOne = [=](SDValue &V) {
17783 if (V.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
17784 V.getConstantOperandVal(i: 0) == Intrinsic::ppc_test_data_class)
17785 return true;
17786 return false;
17787 };
17788
17789 if (!isZeroOrOne(NonNullConstant))
17790 return SDValue();
17791
17792 // Check for pattern: zext(setcc (Value), 0, seteq)) or
17793 // zext(setcc (Value), 0, setne))
17794 if (CC == ISD::SETEQ || CC == ISD::SETNE) {
17795 // Replace with: zext(xor(Value, 1)) for seteq
17796 // or: zext(Value) for setne
17797 // This keeps the value in i32 instead of converting to i1
17798 SDLoc DL(N);
17799 EVT VType = N->getValueType(ResNo: 0);
17800 SDValue NewNonNullConstant = DAG.getZExtOrTrunc(Op: NonNullConstant, DL, VT: VType);
17801
17802 if (CC == ISD::SETNE)
17803 return NewNonNullConstant;
17804
17805 SDValue One = DAG.getConstant(Val: 1, DL, VT: VType);
17806 return DAG.getNode(Opcode: ISD::XOR, DL, VT: VType, N1: NewNonNullConstant, N2: One);
17807 }
17808
17809 return SDValue();
17810}
17811
17812// Combine XOR patterns with SELECT_CC_I4/I8, for Example:
17813// 1. XOR(SELECT_CC_I4(cond, 1, 0, cc), 1) -> SELECT_CC_I4(cond, 0, 1, cc)
17814// 2. XOR(ZEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond, 0,
17815// 1, cc))
17816// 3. XOR(ANYEXT(SELECT_CC_I4(cond, 1, 0, cc)), 1) -> SELECT_CC_I4/I8(cond,
17817// 0, 1, cc))
17818// 4. etc
17819static SDValue combineXorSelectCC(SDNode *N, SelectionDAG &DAG) {
17820 assert(N->getOpcode() == ISD::XOR && "Expected XOR node");
17821
17822 EVT XorVT = N->getValueType(ResNo: 0);
17823 if ((XorVT != MVT::i32 && XorVT != MVT::i64))
17824 return SDValue();
17825
17826 SDValue LHS = N->getOperand(Num: 0);
17827 SDValue RHS = N->getOperand(Num: 1);
17828
17829 // Check for XOR with constant 1
17830 ConstantSDNode *XorConst = dyn_cast<ConstantSDNode>(Val&: RHS);
17831 if (!XorConst || !XorConst->isOne()) {
17832 XorConst = dyn_cast<ConstantSDNode>(Val&: LHS);
17833 if (!XorConst || !XorConst->isOne())
17834 return SDValue();
17835 // Swap so LHS is the SELECT_CC_I4 (or extension) and RHS is the constant
17836 std::swap(a&: LHS, b&: RHS);
17837 }
17838
17839 // Check if LHS has only one use
17840 if (!LHS.hasOneUse())
17841 return SDValue();
17842
17843 // Handle extensions: ZEXT, ANYEXT
17844 SDValue SelectNode = LHS;
17845
17846 if (LHS.getOpcode() == ISD::ZERO_EXTEND ||
17847 LHS.getOpcode() == ISD::ANY_EXTEND) {
17848 SelectNode = LHS.getOperand(i: 0);
17849
17850 // Check if the extension input has only one use
17851 if (!SelectNode.hasOneUse())
17852 return SDValue();
17853 }
17854
17855 // Check if SelectNode is a MachineSDNode with SELECT_CC_I4/I8 opcode
17856 if (!SelectNode.isMachineOpcode())
17857 return SDValue();
17858
17859 unsigned MachineOpc = SelectNode.getMachineOpcode();
17860
17861 // Handle both SELECT_CC_I4 and SELECT_CC_I8
17862 if (MachineOpc != PPC::SELECT_CC_I4 && MachineOpc != PPC::SELECT_CC_I8)
17863 return SDValue();
17864
17865 // SELECT_CC_I4 operands: (cond, true_val, false_val, bropc)
17866 if (SelectNode.getNumOperands() != 4)
17867 return SDValue();
17868
17869 ConstantSDNode *ConstOp1 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 1));
17870 ConstantSDNode *ConstOp2 = dyn_cast<ConstantSDNode>(Val: SelectNode.getOperand(i: 2));
17871
17872 if (!ConstOp1 || !ConstOp2)
17873 return SDValue();
17874
17875 // Only optimize if operands are {0, 1} or {1, 0}
17876 if (!((ConstOp1->isOne() && ConstOp2->isZero()) ||
17877 (ConstOp1->isZero() && ConstOp2->isOne())))
17878 return SDValue();
17879
17880 // Pattern matched! Create new SELECT_CC with swapped 0/1 operands to
17881 // eliminate XOR. If original was SELECT_CC(cond, 1, 0, pred), create
17882 // SELECT_CC(cond, 0, 1, pred). If original was SELECT_CC(cond, 0, 1, pred),
17883 // create SELECT_CC(cond, 1, 0, pred).
17884 SDLoc DL(N);
17885 MachineOpc = (XorVT == MVT::i32) ? PPC::SELECT_CC_I4 : PPC::SELECT_CC_I8;
17886
17887 bool ConstOp1IsOne = ConstOp1->isOne();
17888 return SDValue(
17889 DAG.getMachineNode(Opcode: MachineOpc, dl: DL, VT: XorVT,
17890 Ops: {SelectNode.getOperand(i: 0),
17891 DAG.getConstant(Val: ConstOp1IsOne ? 0 : 1, DL, VT: XorVT),
17892 DAG.getConstant(Val: ConstOp1IsOne ? 1 : 0, DL, VT: XorVT),
17893 SelectNode.getOperand(i: 3)}),
17894 0);
17895}
17896
17897SDValue PPCTargetLowering::PerformDAGCombine(SDNode *N,
17898 DAGCombinerInfo &DCI) const {
17899 SelectionDAG &DAG = DCI.DAG;
17900 SDLoc dl(N);
17901 switch (N->getOpcode()) {
17902 default: break;
17903 case ISD::ADD:
17904 return combineADD(N, DCI);
17905 case ISD::AND: {
17906 // We don't want (and (zext (shift...)), C) if C fits in the width of the
17907 // original input as that will prevent us from selecting optimal rotates.
17908 // This only matters if the input to the extend is i32 widened to i64.
17909 SDValue Op1 = N->getOperand(Num: 0);
17910 SDValue Op2 = N->getOperand(Num: 1);
17911 if ((Op1.getOpcode() != ISD::ZERO_EXTEND &&
17912 Op1.getOpcode() != ISD::ANY_EXTEND) ||
17913 !isa<ConstantSDNode>(Val: Op2) || N->getValueType(ResNo: 0) != MVT::i64 ||
17914 Op1.getOperand(i: 0).getValueType() != MVT::i32)
17915 break;
17916 SDValue NarrowOp = Op1.getOperand(i: 0);
17917 if (NarrowOp.getOpcode() != ISD::SHL && NarrowOp.getOpcode() != ISD::SRL &&
17918 NarrowOp.getOpcode() != ISD::ROTL && NarrowOp.getOpcode() != ISD::ROTR)
17919 break;
17920
17921 uint64_t Imm = Op2->getAsZExtVal();
17922 // Make sure that the constant is narrow enough to fit in the narrow type.
17923 if (!isUInt<32>(x: Imm))
17924 break;
17925 SDValue ConstOp = DAG.getConstant(Val: Imm, DL: dl, VT: MVT::i32);
17926 SDValue NarrowAnd = DAG.getNode(Opcode: ISD::AND, DL: dl, VT: MVT::i32, N1: NarrowOp, N2: ConstOp);
17927 return DAG.getZExtOrTrunc(Op: NarrowAnd, DL: dl, VT: N->getValueType(ResNo: 0));
17928 }
17929 case ISD::XOR: {
17930 // Optimize XOR(ISEL(1,0,CR), 1) -> ISEL(0,1,CR)
17931 if (SDValue V = combineXorSelectCC(N, DAG))
17932 return V;
17933 break;
17934 }
17935 case ISD::SHL:
17936 return combineSHL(N, DCI);
17937 case ISD::SRA:
17938 return combineSRA(N, DCI);
17939 case ISD::SRL:
17940 return combineSRL(N, DCI);
17941 case ISD::MUL:
17942 return combineMUL(N, DCI);
17943 case ISD::FMA:
17944 case PPCISD::FNMSUB:
17945 return combineFMALike(N, DCI);
17946 case PPCISD::SHL:
17947 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 << V -> 0.
17948 return N->getOperand(Num: 0);
17949 break;
17950 case PPCISD::SRL:
17951 if (isNullConstant(V: N->getOperand(Num: 0))) // 0 >>u V -> 0.
17952 return N->getOperand(Num: 0);
17953 break;
17954 case PPCISD::SRA:
17955 if (ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 0))) {
17956 if (C->isZero() || // 0 >>s V -> 0.
17957 C->isAllOnes()) // -1 >>s V -> -1.
17958 return N->getOperand(Num: 0);
17959 }
17960 break;
17961 case ISD::SIGN_EXTEND: {
17962 // Optimize: sign_extend(scmp) -> scmp with i64 result (on PPC64)
17963 // This eliminates the sign_extend when scmp result is being extended to i64
17964 // Pattern: t6: i64 = sign_extend t5:i32
17965 // t5: i32 = scmp t2, t4
17966 // Result: t5: i64 = scmp t2, t4
17967 SDValue N0 = N->getOperand(Num: 0);
17968 if (Subtarget.isPPC64() && N0.getOpcode() == ISD::SCMP &&
17969 N->getValueType(ResNo: 0) == MVT::i64 && N0.getValueType() == MVT::i32 &&
17970 N0.hasOneUse()) {
17971 // Recreate the SCMP with i64 result type directly
17972 SDValue LHS = N0.getOperand(i: 0);
17973 SDValue RHS = N0.getOperand(i: 1);
17974 return DAG.getNode(Opcode: ISD::SCMP, DL: dl, VT: MVT::i64, N1: LHS, N2: RHS);
17975 }
17976
17977 if (SDValue SECC = combineSignExtendSetCC(N, DCI))
17978 return SECC;
17979 [[fallthrough]];
17980 }
17981 case ISD::ZERO_EXTEND:
17982 if (SDValue RetV = combineZextSetccWithZero(N, DAG&: DCI.DAG))
17983 return RetV;
17984 [[fallthrough]];
17985 case ISD::ANY_EXTEND:
17986 return DAGCombineExtBoolTrunc(N, DCI);
17987 case ISD::TRUNCATE:
17988 return combineTRUNCATE(N, DCI);
17989 case ISD::SETCC:
17990 if (SDValue CSCC = combineSetCC(N, DCI))
17991 return CSCC;
17992 [[fallthrough]];
17993 case ISD::SELECT_CC:
17994 if (SDValue V = combineSELECT_CCBitFloor(N, DAG))
17995 return V;
17996 return DAGCombineTruncBoolExt(N, DCI);
17997 case ISD::SINT_TO_FP:
17998 case ISD::UINT_TO_FP:
17999 return combineFPToIntToFP(N, DCI);
18000 case ISD::VECTOR_SHUFFLE:
18001 if (ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode())) {
18002 LSBaseSDNode* LSBase = cast<LSBaseSDNode>(Val: N->getOperand(Num: 0));
18003 return combineVReverseMemOP(SVN: cast<ShuffleVectorSDNode>(Val: N), LSBase, DCI);
18004 }
18005 return combineVectorShuffle(SVN: cast<ShuffleVectorSDNode>(Val: N), DAG&: DCI.DAG);
18006 case ISD::STORE: {
18007
18008 EVT Op1VT = N->getOperand(Num: 1).getValueType();
18009 unsigned Opcode = N->getOperand(Num: 1).getOpcode();
18010
18011 if (Opcode == ISD::FP_TO_SINT || Opcode == ISD::FP_TO_UINT ||
18012 Opcode == ISD::STRICT_FP_TO_SINT || Opcode == ISD::STRICT_FP_TO_UINT) {
18013 SDValue Val = combineStoreFPToInt(N, DCI);
18014 if (Val)
18015 return Val;
18016 }
18017
18018 if (Opcode == ISD::VECTOR_SHUFFLE && ISD::isNormalStore(N)) {
18019 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N->getOperand(Num: 1));
18020 SDValue Val= combineVReverseMemOP(SVN, LSBase: cast<LSBaseSDNode>(Val: N), DCI);
18021 if (Val)
18022 return Val;
18023 }
18024
18025 // Turn STORE (BSWAP) -> sthbrx/stwbrx.
18026 if (cast<StoreSDNode>(Val: N)->isUnindexed() && Opcode == ISD::BSWAP &&
18027 N->getOperand(Num: 1).getNode()->hasOneUse() &&
18028 (Op1VT == MVT::i32 || Op1VT == MVT::i16 ||
18029 (Subtarget.hasLDBRX() && Subtarget.isPPC64() && Op1VT == MVT::i64))) {
18030
18031 // STBRX can only handle simple types and it makes no sense to store less
18032 // two bytes in byte-reversed order.
18033 EVT mVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
18034 if (mVT.isExtended() || mVT.getSizeInBits() < 16)
18035 break;
18036
18037 SDValue BSwapOp = N->getOperand(Num: 1).getOperand(i: 0);
18038 // Do an any-extend to 32-bits if this is a half-word input.
18039 if (BSwapOp.getValueType() == MVT::i16)
18040 BSwapOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL: dl, VT: MVT::i32, Operand: BSwapOp);
18041
18042 // If the type of BSWAP operand is wider than stored memory width
18043 // it need to be shifted to the right side before STBRX.
18044 if (Op1VT.bitsGT(VT: mVT)) {
18045 int Shift = Op1VT.getSizeInBits() - mVT.getSizeInBits();
18046 BSwapOp = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: Op1VT, N1: BSwapOp,
18047 N2: DAG.getConstant(Val: Shift, DL: dl, VT: MVT::i32));
18048 // Need to truncate if this is a bswap of i64 stored as i32/i16.
18049 if (Op1VT == MVT::i64)
18050 BSwapOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: BSwapOp);
18051 }
18052
18053 SDValue Ops[] = {
18054 N->getOperand(Num: 0), BSwapOp, N->getOperand(Num: 2), DAG.getValueType(mVT)
18055 };
18056 return
18057 DAG.getMemIntrinsicNode(Opcode: PPCISD::STBRX, dl, VTList: DAG.getVTList(VT: MVT::Other),
18058 Ops, MemVT: cast<StoreSDNode>(Val: N)->getMemoryVT(),
18059 MMO: cast<StoreSDNode>(Val: N)->getMemOperand());
18060 }
18061
18062 // STORE Constant:i32<0> -> STORE<trunc to i32> Constant:i64<0>
18063 // So it can increase the chance of CSE constant construction.
18064 if (Subtarget.isPPC64() && !DCI.isBeforeLegalize() &&
18065 isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) && Op1VT == MVT::i32) {
18066 // Need to sign-extended to 64-bits to handle negative values.
18067 EVT MemVT = cast<StoreSDNode>(Val: N)->getMemoryVT();
18068 uint64_t Val64 = SignExtend64(X: N->getConstantOperandVal(Num: 1),
18069 B: MemVT.getSizeInBits());
18070 SDValue Const64 = DAG.getConstant(Val: Val64, DL: dl, VT: MVT::i64);
18071
18072 auto *ST = cast<StoreSDNode>(Val: N);
18073 SDValue NewST = DAG.getStore(Chain: ST->getChain(), dl, Val: Const64,
18074 Ptr: ST->getBasePtr(), Offset: ST->getOffset(), SVT: MemVT,
18075 MMO: ST->getMemOperand(), AM: ST->getAddressingMode(),
18076 /*IsTruncating=*/true);
18077 // Note we use CombineTo here to prevent DAGCombiner from visiting the
18078 // new store which will change the constant by removing non-demanded bits.
18079 return ST->isUnindexed()
18080 ? DCI.CombineTo(N, Res: NewST, /*AddTo=*/false)
18081 : DCI.CombineTo(N, Res0: NewST, Res1: NewST.getValue(R: 1), /*AddTo=*/false);
18082 }
18083
18084 // For little endian, VSX stores require generating xxswapd/lxvd2x.
18085 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18086 if (Op1VT.isSimple()) {
18087 MVT StoreVT = Op1VT.getSimpleVT();
18088 if (Subtarget.needsSwapsForVSXMemOps() &&
18089 (StoreVT == MVT::v2f64 || StoreVT == MVT::v2i64 ||
18090 StoreVT == MVT::v4f32 || StoreVT == MVT::v4i32))
18091 return expandVSXStoreForLE(N, DCI);
18092 }
18093 break;
18094 }
18095 case ISD::LOAD: {
18096 LoadSDNode *LD = cast<LoadSDNode>(Val: N);
18097 EVT VT = LD->getValueType(ResNo: 0);
18098
18099 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18100 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18101 if (VT.isSimple()) {
18102 MVT LoadVT = VT.getSimpleVT();
18103 if (Subtarget.needsSwapsForVSXMemOps() &&
18104 (LoadVT == MVT::v2f64 || LoadVT == MVT::v2i64 ||
18105 LoadVT == MVT::v4f32 || LoadVT == MVT::v4i32))
18106 return expandVSXLoadForLE(N, DCI);
18107 }
18108
18109 // We sometimes end up with a 64-bit integer load, from which we extract
18110 // two single-precision floating-point numbers. This happens with
18111 // std::complex<float>, and other similar structures, because of the way we
18112 // canonicalize structure copies. However, if we lack direct moves,
18113 // then the final bitcasts from the extracted integer values to the
18114 // floating-point numbers turn into store/load pairs. Even with direct moves,
18115 // just loading the two floating-point numbers is likely better.
18116 auto ReplaceTwoFloatLoad = [&]() {
18117 if (VT != MVT::i64)
18118 return false;
18119
18120 if (LD->getExtensionType() != ISD::NON_EXTLOAD ||
18121 LD->isVolatile())
18122 return false;
18123
18124 // We're looking for a sequence like this:
18125 // t13: i64,ch = load<LD8[%ref.tmp]> t0, t6, undef:i64
18126 // t16: i64 = srl t13, Constant:i32<32>
18127 // t17: i32 = truncate t16
18128 // t18: f32 = bitcast t17
18129 // t19: i32 = truncate t13
18130 // t20: f32 = bitcast t19
18131
18132 if (!LD->hasNUsesOfValue(NUses: 2, Value: 0))
18133 return false;
18134
18135 auto UI = LD->user_begin();
18136 while (UI.getUse().getResNo() != 0) ++UI;
18137 SDNode *Trunc = *UI++;
18138 while (UI.getUse().getResNo() != 0) ++UI;
18139 SDNode *RightShift = *UI;
18140 if (Trunc->getOpcode() != ISD::TRUNCATE)
18141 std::swap(a&: Trunc, b&: RightShift);
18142
18143 if (Trunc->getOpcode() != ISD::TRUNCATE ||
18144 Trunc->getValueType(ResNo: 0) != MVT::i32 ||
18145 !Trunc->hasOneUse())
18146 return false;
18147 if (RightShift->getOpcode() != ISD::SRL ||
18148 !isa<ConstantSDNode>(Val: RightShift->getOperand(Num: 1)) ||
18149 RightShift->getConstantOperandVal(Num: 1) != 32 ||
18150 !RightShift->hasOneUse())
18151 return false;
18152
18153 SDNode *Trunc2 = *RightShift->user_begin();
18154 if (Trunc2->getOpcode() != ISD::TRUNCATE ||
18155 Trunc2->getValueType(ResNo: 0) != MVT::i32 ||
18156 !Trunc2->hasOneUse())
18157 return false;
18158
18159 SDNode *Bitcast = *Trunc->user_begin();
18160 SDNode *Bitcast2 = *Trunc2->user_begin();
18161
18162 if (Bitcast->getOpcode() != ISD::BITCAST ||
18163 Bitcast->getValueType(ResNo: 0) != MVT::f32)
18164 return false;
18165 if (Bitcast2->getOpcode() != ISD::BITCAST ||
18166 Bitcast2->getValueType(ResNo: 0) != MVT::f32)
18167 return false;
18168
18169 if (Subtarget.isLittleEndian())
18170 std::swap(a&: Bitcast, b&: Bitcast2);
18171
18172 // Bitcast has the second float (in memory-layout order) and Bitcast2
18173 // has the first one.
18174
18175 SDValue BasePtr = LD->getBasePtr();
18176 if (LD->isIndexed()) {
18177 assert(LD->getAddressingMode() == ISD::PRE_INC &&
18178 "Non-pre-inc AM on PPC?");
18179 BasePtr =
18180 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18181 N2: LD->getOffset());
18182 }
18183
18184 auto MMOFlags =
18185 LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOVolatile;
18186 SDValue FloatLoad = DAG.getLoad(VT: MVT::f32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18187 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign(),
18188 MMOFlags, Metadata: LD->getAAInfo());
18189 SDValue AddPtr =
18190 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(),
18191 N1: BasePtr, N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18192 SDValue FloatLoad2 = DAG.getLoad(
18193 VT: MVT::f32, dl, Chain: SDValue(FloatLoad.getNode(), 1), Ptr: AddPtr,
18194 PtrInfo: LD->getPointerInfo().getWithOffset(O: 4),
18195 Alignment: commonAlignment(A: LD->getAlign(), Offset: 4), MMOFlags, Metadata: LD->getAAInfo());
18196
18197 if (LD->isIndexed()) {
18198 // Note that DAGCombine should re-form any pre-increment load(s) from
18199 // what is produced here if that makes sense.
18200 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: BasePtr);
18201 }
18202
18203 DCI.CombineTo(N: Bitcast2, Res: FloatLoad);
18204 DCI.CombineTo(N: Bitcast, Res: FloatLoad2);
18205
18206 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, LD->isIndexed() ? 2 : 1),
18207 To: SDValue(FloatLoad2.getNode(), 1));
18208 return true;
18209 };
18210
18211 if (ReplaceTwoFloatLoad())
18212 return SDValue(N, 0);
18213
18214 EVT MemVT = LD->getMemoryVT();
18215 Type *Ty = MemVT.getTypeForEVT(Context&: *DAG.getContext());
18216 Align ABIAlignment = DAG.getDataLayout().getABITypeAlign(Ty);
18217 if (LD->isUnindexed() && VT.isVector() &&
18218 ((Subtarget.hasAltivec() && ISD::isNON_EXTLoad(N) &&
18219 // P8 and later hardware should just use LOAD.
18220 !Subtarget.hasP8Vector() &&
18221 (VT == MVT::v16i8 || VT == MVT::v8i16 || VT == MVT::v4i32 ||
18222 VT == MVT::v4f32))) &&
18223 LD->getAlign() < ABIAlignment) {
18224 // This is a type-legal unaligned Altivec load.
18225 SDValue Chain = LD->getChain();
18226 SDValue Ptr = LD->getBasePtr();
18227 bool isLittleEndian = Subtarget.isLittleEndian();
18228
18229 // This implements the loading of unaligned vectors as described in
18230 // the venerable Apple Velocity Engine overview. Specifically:
18231 // https://developer.apple.com/hardwaredrivers/ve/alignment.html
18232 // https://developer.apple.com/hardwaredrivers/ve/code_optimization.html
18233 //
18234 // The general idea is to expand a sequence of one or more unaligned
18235 // loads into an alignment-based permutation-control instruction (lvsl
18236 // or lvsr), a series of regular vector loads (which always truncate
18237 // their input address to an aligned address), and a series of
18238 // permutations. The results of these permutations are the requested
18239 // loaded values. The trick is that the last "extra" load is not taken
18240 // from the address you might suspect (sizeof(vector) bytes after the
18241 // last requested load), but rather sizeof(vector) - 1 bytes after the
18242 // last requested vector. The point of this is to avoid a page fault if
18243 // the base address happened to be aligned. This works because if the
18244 // base address is aligned, then adding less than a full vector length
18245 // will cause the last vector in the sequence to be (re)loaded.
18246 // Otherwise, the next vector will be fetched as you might suspect was
18247 // necessary.
18248
18249 // We might be able to reuse the permutation generation from
18250 // a different base address offset from this one by an aligned amount.
18251 // The INTRINSIC_WO_CHAIN DAG combine will attempt to perform this
18252 // optimization later.
18253 Intrinsic::ID Intr, IntrLD, IntrPerm;
18254 MVT PermCntlTy, PermTy, LDTy;
18255 Intr = isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18256 : Intrinsic::ppc_altivec_lvsl;
18257 IntrLD = Intrinsic::ppc_altivec_lvx;
18258 IntrPerm = Intrinsic::ppc_altivec_vperm;
18259 PermCntlTy = MVT::v16i8;
18260 PermTy = MVT::v4i32;
18261 LDTy = MVT::v4i32;
18262
18263 SDValue PermCntl = BuildIntrinsicOp(IID: Intr, Op: Ptr, DAG, dl, DestVT: PermCntlTy);
18264
18265 // Create the new MMO for the new base load. It is like the original MMO,
18266 // but represents an area in memory almost twice the vector size centered
18267 // on the original address. If the address is unaligned, we might start
18268 // reading up to (sizeof(vector)-1) bytes below the address of the
18269 // original unaligned load.
18270 MachineFunction &MF = DAG.getMachineFunction();
18271 MachineMemOperand *BaseMMO =
18272 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18273 Offset: -(int64_t)MemVT.getStoreSize()+1,
18274 Size: 2*MemVT.getStoreSize()-1);
18275
18276 // Create the new base load.
18277 SDValue LDXIntID =
18278 DAG.getTargetConstant(Val: IntrLD, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18279 SDValue BaseLoadOps[] = { Chain, LDXIntID, Ptr };
18280 SDValue BaseLoad =
18281 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18282 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18283 Ops: BaseLoadOps, MemVT: LDTy, MMO: BaseMMO);
18284
18285 // Note that the value of IncOffset (which is provided to the next
18286 // load's pointer info offset value, and thus used to calculate the
18287 // alignment), and the value of IncValue (which is actually used to
18288 // increment the pointer value) are different! This is because we
18289 // require the next load to appear to be aligned, even though it
18290 // is actually offset from the base pointer by a lesser amount.
18291 int IncOffset = VT.getSizeInBits() / 8;
18292 int IncValue = IncOffset;
18293
18294 // Walk (both up and down) the chain looking for another load at the real
18295 // (aligned) offset (the alignment of the other load does not matter in
18296 // this case). If found, then do not use the offset reduction trick, as
18297 // that will prevent the loads from being later combined (as they would
18298 // otherwise be duplicates).
18299 if (!findConsecutiveLoad(LD, DAG))
18300 --IncValue;
18301
18302 SDValue Increment =
18303 DAG.getConstant(Val: IncValue, DL: dl, VT: getPointerTy(DL: MF.getDataLayout()));
18304 Ptr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: Ptr.getValueType(), N1: Ptr, N2: Increment);
18305
18306 MachineMemOperand *ExtraMMO =
18307 MF.getMachineMemOperand(MMO: LD->getMemOperand(),
18308 Offset: 1, Size: 2*MemVT.getStoreSize()-1);
18309 SDValue ExtraLoadOps[] = { Chain, LDXIntID, Ptr };
18310 SDValue ExtraLoad =
18311 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl,
18312 VTList: DAG.getVTList(VT1: PermTy, VT2: MVT::Other),
18313 Ops: ExtraLoadOps, MemVT: LDTy, MMO: ExtraMMO);
18314
18315 SDValue TF = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18316 N1: BaseLoad.getValue(R: 1), N2: ExtraLoad.getValue(R: 1));
18317
18318 // Because vperm has a big-endian bias, we must reverse the order
18319 // of the input vectors and complement the permute control vector
18320 // when generating little endian code. We have already handled the
18321 // latter by using lvsr instead of lvsl, so just reverse BaseLoad
18322 // and ExtraLoad here.
18323 SDValue Perm;
18324 if (isLittleEndian)
18325 Perm = BuildIntrinsicOp(IID: IntrPerm,
18326 Op0: ExtraLoad, Op1: BaseLoad, Op2: PermCntl, DAG, dl);
18327 else
18328 Perm = BuildIntrinsicOp(IID: IntrPerm,
18329 Op0: BaseLoad, Op1: ExtraLoad, Op2: PermCntl, DAG, dl);
18330
18331 if (VT != PermTy)
18332 Perm = Subtarget.hasAltivec()
18333 ? DAG.getNode(Opcode: ISD::BITCAST, DL: dl, VT, Operand: Perm)
18334 : DAG.getNode(Opcode: ISD::FP_ROUND, DL: dl, VT, N1: Perm,
18335 N2: DAG.getTargetConstant(Val: 1, DL: dl, VT: MVT::i64));
18336 // second argument is 1 because this rounding
18337 // is always exact.
18338
18339 // The output of the permutation is our loaded result, the TokenFactor is
18340 // our new chain.
18341 DCI.CombineTo(N, Res0: Perm, Res1: TF);
18342 return SDValue(N, 0);
18343 }
18344 }
18345 break;
18346 case ISD::INTRINSIC_WO_CHAIN: {
18347 bool isLittleEndian = Subtarget.isLittleEndian();
18348 unsigned IID = N->getConstantOperandVal(Num: 0);
18349 Intrinsic::ID Intr = (isLittleEndian ? Intrinsic::ppc_altivec_lvsr
18350 : Intrinsic::ppc_altivec_lvsl);
18351 if (IID == Intr && N->getOperand(Num: 1)->getOpcode() == ISD::ADD) {
18352 SDValue Add = N->getOperand(Num: 1);
18353
18354 int Bits = 4 /* 16 byte alignment */;
18355
18356 if (DAG.MaskedValueIsZero(Op: Add->getOperand(Num: 1),
18357 Mask: APInt::getAllOnes(numBits: Bits /* alignment */)
18358 .zext(width: Add.getScalarValueSizeInBits()))) {
18359 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18360 for (SDNode *U : BasePtr->users()) {
18361 if (U->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18362 U->getConstantOperandVal(Num: 0) == IID) {
18363 // We've found another LVSL/LVSR, and this address is an aligned
18364 // multiple of that one. The results will be the same, so use the
18365 // one we've just found instead.
18366
18367 return SDValue(U, 0);
18368 }
18369 }
18370 }
18371
18372 if (isa<ConstantSDNode>(Val: Add->getOperand(Num: 1))) {
18373 SDNode *BasePtr = Add->getOperand(Num: 0).getNode();
18374 for (SDNode *U : BasePtr->users()) {
18375 if (U->getOpcode() == ISD::ADD &&
18376 isa<ConstantSDNode>(Val: U->getOperand(Num: 1)) &&
18377 (Add->getConstantOperandVal(Num: 1) - U->getConstantOperandVal(Num: 1)) %
18378 (1ULL << Bits) ==
18379 0) {
18380 SDNode *OtherAdd = U;
18381 for (SDNode *V : OtherAdd->users()) {
18382 if (V->getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18383 V->getConstantOperandVal(Num: 0) == IID) {
18384 return SDValue(V, 0);
18385 }
18386 }
18387 }
18388 }
18389 }
18390 }
18391
18392 // Combine vmaxsw/h/b(a, a's negation) to abs(a)
18393 // Expose the vabsduw/h/b opportunity for down stream
18394 if (!DCI.isAfterLegalizeDAG() && Subtarget.hasP9Altivec() &&
18395 (IID == Intrinsic::ppc_altivec_vmaxsw ||
18396 IID == Intrinsic::ppc_altivec_vmaxsh ||
18397 IID == Intrinsic::ppc_altivec_vmaxsb)) {
18398 SDValue V1 = N->getOperand(Num: 1);
18399 SDValue V2 = N->getOperand(Num: 2);
18400 if ((V1.getSimpleValueType() == MVT::v4i32 ||
18401 V1.getSimpleValueType() == MVT::v8i16 ||
18402 V1.getSimpleValueType() == MVT::v16i8) &&
18403 V1.getSimpleValueType() == V2.getSimpleValueType()) {
18404 // (0-a, a)
18405 if (V1.getOpcode() == ISD::SUB &&
18406 ISD::isBuildVectorAllZeros(V: V1.getOperand(i: 0)) &&
18407 V1.getOperand(i: 1) == V2) {
18408 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V2.getValueType(), Operand: V2);
18409 }
18410 // (a, 0-a)
18411 if (V2.getOpcode() == ISD::SUB &&
18412 ISD::isBuildVectorAllZeros(V: V2.getOperand(i: 0)) &&
18413 V2.getOperand(i: 1) == V1) {
18414 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18415 }
18416 // (x-y, y-x)
18417 if (V1.getOpcode() == ISD::SUB && V2.getOpcode() == ISD::SUB &&
18418 V1.getOperand(i: 0) == V2.getOperand(i: 1) &&
18419 V1.getOperand(i: 1) == V2.getOperand(i: 0)) {
18420 return DAG.getNode(Opcode: ISD::ABS, DL: dl, VT: V1.getValueType(), Operand: V1);
18421 }
18422 }
18423 }
18424 }
18425
18426 break;
18427 case ISD::INTRINSIC_W_CHAIN:
18428 switch (N->getConstantOperandVal(Num: 1)) {
18429 default:
18430 break;
18431 case Intrinsic::ppc_altivec_vsum4sbs:
18432 case Intrinsic::ppc_altivec_vsum4shs:
18433 case Intrinsic::ppc_altivec_vsum4ubs: {
18434 // These sum-across intrinsics only have a chain due to the side effect
18435 // that they may set the SAT bit. If we know the SAT bit will not be set
18436 // for some inputs, we can replace any uses of their chain with the
18437 // input chain.
18438 if (BuildVectorSDNode *BVN =
18439 dyn_cast<BuildVectorSDNode>(Val: N->getOperand(Num: 3))) {
18440 APInt APSplatBits, APSplatUndef;
18441 unsigned SplatBitSize;
18442 bool HasAnyUndefs;
18443 bool BVNIsConstantSplat = BVN->isConstantSplat(
18444 SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize, HasAnyUndefs, MinSplatBits: 0,
18445 isBigEndian: !Subtarget.isLittleEndian());
18446 // If the constant splat vector is 0, the SAT bit will not be set.
18447 if (BVNIsConstantSplat && APSplatBits == 0)
18448 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N, 1), To: N->getOperand(Num: 0));
18449 }
18450 return SDValue();
18451 }
18452 case Intrinsic::ppc_vsx_lxvw4x:
18453 case Intrinsic::ppc_vsx_lxvd2x:
18454 // For little endian, VSX loads require generating lxvd2x/xxswapd.
18455 // Not needed on ISA 3.0 based CPUs since we have a non-permuting load.
18456 if (Subtarget.needsSwapsForVSXMemOps())
18457 return expandVSXLoadForLE(N, DCI);
18458 break;
18459 }
18460 break;
18461 case ISD::INTRINSIC_VOID:
18462 // For little endian, VSX stores require generating xxswapd/stxvd2x.
18463 // Not needed on ISA 3.0 based CPUs since we have a non-permuting store.
18464 if (Subtarget.needsSwapsForVSXMemOps()) {
18465 switch (N->getConstantOperandVal(Num: 1)) {
18466 default:
18467 break;
18468 case Intrinsic::ppc_vsx_stxvw4x:
18469 case Intrinsic::ppc_vsx_stxvd2x:
18470 return expandVSXStoreForLE(N, DCI);
18471 }
18472 }
18473 break;
18474 case ISD::BSWAP: {
18475 // Turn BSWAP (LOAD) -> lhbrx/lwbrx.
18476 // For subtargets without LDBRX, we can still do better than the default
18477 // expansion even for 64-bit BSWAP (LOAD).
18478 bool Is64BitBswapOn64BitTgt =
18479 Subtarget.isPPC64() && N->getValueType(ResNo: 0) == MVT::i64;
18480 bool IsSingleUseNormalLd = ISD::isNormalLoad(N: N->getOperand(Num: 0).getNode()) &&
18481 N->getOperand(Num: 0).hasOneUse();
18482 if (IsSingleUseNormalLd &&
18483 (N->getValueType(ResNo: 0) == MVT::i32 || N->getValueType(ResNo: 0) == MVT::i16 ||
18484 (Subtarget.hasLDBRX() && Is64BitBswapOn64BitTgt))) {
18485 SDValue Load = N->getOperand(Num: 0);
18486 LoadSDNode *LD = cast<LoadSDNode>(Val&: Load);
18487 // Create the byte-swapping load.
18488 SDValue Ops[] = {
18489 LD->getChain(), // Chain
18490 LD->getBasePtr(), // Ptr
18491 DAG.getValueType(N->getValueType(ResNo: 0)) // VT
18492 };
18493 SDValue BSLoad =
18494 DAG.getMemIntrinsicNode(Opcode: PPCISD::LBRX, dl,
18495 VTList: DAG.getVTList(VT1: N->getValueType(ResNo: 0) == MVT::i64 ?
18496 MVT::i64 : MVT::i32, VT2: MVT::Other),
18497 Ops, MemVT: LD->getMemoryVT(), MMO: LD->getMemOperand());
18498
18499 // If this is an i16 load, insert the truncate.
18500 SDValue ResVal = BSLoad;
18501 if (N->getValueType(ResNo: 0) == MVT::i16)
18502 ResVal = DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i16, Operand: BSLoad);
18503
18504 // First, combine the bswap away. This makes the value produced by the
18505 // load dead.
18506 DCI.CombineTo(N, Res: ResVal);
18507
18508 // Next, combine the load away, we give it a bogus result value but a real
18509 // chain result. The result value is dead because the bswap is dead.
18510 DCI.CombineTo(N: Load.getNode(), Res0: ResVal, Res1: BSLoad.getValue(R: 1));
18511
18512 // Return N so it doesn't get rechecked!
18513 return SDValue(N, 0);
18514 }
18515 // Convert this to two 32-bit bswap loads and a BUILD_PAIR. Do this only
18516 // before legalization so that the BUILD_PAIR is handled correctly.
18517 if (!DCI.isBeforeLegalize() || !Is64BitBswapOn64BitTgt ||
18518 !IsSingleUseNormalLd)
18519 return SDValue();
18520 LoadSDNode *LD = cast<LoadSDNode>(Val: N->getOperand(Num: 0));
18521
18522 // Can't split volatile or atomic loads.
18523 if (!LD->isSimple())
18524 return SDValue();
18525 SDValue BasePtr = LD->getBasePtr();
18526 SDValue Lo = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr,
18527 PtrInfo: LD->getPointerInfo(), Alignment: LD->getAlign());
18528 Lo = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Lo);
18529 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: BasePtr.getValueType(), N1: BasePtr,
18530 N2: DAG.getIntPtrConstant(Val: 4, DL: dl));
18531 MachineMemOperand *NewMMO = DAG.getMachineFunction().getMachineMemOperand(
18532 MMO: LD->getMemOperand(), Offset: 4, Size: 4);
18533 SDValue Hi = DAG.getLoad(VT: MVT::i32, dl, Chain: LD->getChain(), Ptr: BasePtr, MMO: NewMMO);
18534 Hi = DAG.getNode(Opcode: ISD::BSWAP, DL: dl, VT: MVT::i32, Operand: Hi);
18535 SDValue Res;
18536 if (Subtarget.isLittleEndian())
18537 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Hi, N2: Lo);
18538 else
18539 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL: dl, VT: MVT::i64, N1: Lo, N2: Hi);
18540 SDValue TF =
18541 DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other,
18542 N1: Hi.getOperand(i: 0).getValue(R: 1), N2: Lo.getOperand(i: 0).getValue(R: 1));
18543 DAG.ReplaceAllUsesOfValueWith(From: SDValue(LD, 1), To: TF);
18544 return Res;
18545 }
18546 case PPCISD::VCMP:
18547 // If a VCMP_rec node already exists with exactly the same operands as this
18548 // node, use its result instead of this node (VCMP_rec computes both a CR6
18549 // and a normal output).
18550 //
18551 if (!N->getOperand(Num: 0).hasOneUse() &&
18552 !N->getOperand(Num: 1).hasOneUse() &&
18553 !N->getOperand(Num: 2).hasOneUse()) {
18554
18555 // Scan all of the users of the LHS, looking for VCMP_rec's that match.
18556 SDNode *VCMPrecNode = nullptr;
18557
18558 SDNode *LHSN = N->getOperand(Num: 0).getNode();
18559 for (SDNode *User : LHSN->users())
18560 if (User->getOpcode() == PPCISD::VCMP_rec &&
18561 User->getOperand(Num: 1) == N->getOperand(Num: 1) &&
18562 User->getOperand(Num: 2) == N->getOperand(Num: 2) &&
18563 User->getOperand(Num: 0) == N->getOperand(Num: 0)) {
18564 VCMPrecNode = User;
18565 break;
18566 }
18567
18568 // If there is no VCMP_rec node, or if the flag value has a single use,
18569 // don't transform this.
18570 if (!VCMPrecNode || VCMPrecNode->hasNUsesOfValue(NUses: 0, Value: 1))
18571 break;
18572
18573 // Look at the (necessarily single) use of the flag value. If it has a
18574 // chain, this transformation is more complex. Note that multiple things
18575 // could use the value result, which we should ignore.
18576 SDNode *FlagUser = nullptr;
18577 for (SDNode::use_iterator UI = VCMPrecNode->use_begin();
18578 FlagUser == nullptr; ++UI) {
18579 assert(UI != VCMPrecNode->use_end() && "Didn't find user!");
18580 SDNode *User = UI->getUser();
18581 for (unsigned i = 0, e = User->getNumOperands(); i != e; ++i) {
18582 if (User->getOperand(Num: i) == SDValue(VCMPrecNode, 1)) {
18583 FlagUser = User;
18584 break;
18585 }
18586 }
18587 }
18588
18589 // If the user is a MFOCRF instruction, we know this is safe.
18590 // Otherwise we give up for right now.
18591 if (FlagUser->getOpcode() == PPCISD::MFOCRF)
18592 return SDValue(VCMPrecNode, 0);
18593 }
18594 break;
18595 case ISD::BR_CC: {
18596 // If this is a branch on an altivec predicate comparison, lower this so
18597 // that we don't have to do a MFOCRF: instead, branch directly on CR6. This
18598 // lowering is done pre-legalize, because the legalizer lowers the predicate
18599 // compare down to code that is difficult to reassemble.
18600 // This code also handles branches that depend on the result of a store
18601 // conditional.
18602 ISD::CondCode CC = N->getOperandCondCode(Num: 1);
18603 SDValue LHS = N->getOperand(Num: 2), RHS = N->getOperand(Num: 3);
18604
18605 int CompareOpc;
18606 bool isDot;
18607
18608 if (!isa<ConstantSDNode>(Val: RHS) || (CC != ISD::SETEQ && CC != ISD::SETNE))
18609 break;
18610
18611 // Since we are doing this pre-legalize, the RHS can be a constant of
18612 // arbitrary bitwidth which may cause issues when trying to get the value
18613 // from the underlying APInt.
18614 auto RHSAPInt = RHS->getAsAPIntVal();
18615 if (!RHSAPInt.isIntN(N: 64))
18616 break;
18617
18618 unsigned Val = RHSAPInt.getZExtValue();
18619 auto isImpossibleCompare = [&]() {
18620 // If this is a comparison against something other than 0/1, then we know
18621 // that the condition is never/always true.
18622 if (Val != 0 && Val != 1) {
18623 if (CC == ISD::SETEQ) // Cond never true, remove branch.
18624 return N->getOperand(Num: 0);
18625 // Always !=, turn it into an unconditional branch.
18626 return DAG.getNode(Opcode: ISD::BR, DL: dl, VT: MVT::Other,
18627 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 4));
18628 }
18629 return SDValue();
18630 };
18631 // Combine branches fed by store conditional instructions (st[bhwd]cx).
18632 unsigned StoreWidth = 0;
18633 if (LHS.getOpcode() == ISD::INTRINSIC_W_CHAIN &&
18634 isStoreConditional(Intrin: LHS, StoreWidth)) {
18635 if (SDValue Impossible = isImpossibleCompare())
18636 return Impossible;
18637 PPC::Predicate CompOpc;
18638 // eq 0 => ne
18639 // ne 0 => eq
18640 // eq 1 => eq
18641 // ne 1 => ne
18642 if (Val == 0)
18643 CompOpc = CC == ISD::SETEQ ? PPC::PRED_NE : PPC::PRED_EQ;
18644 else
18645 CompOpc = CC == ISD::SETEQ ? PPC::PRED_EQ : PPC::PRED_NE;
18646
18647 SDValue Ops[] = {LHS.getOperand(i: 0), LHS.getOperand(i: 2), LHS.getOperand(i: 3),
18648 DAG.getConstant(Val: StoreWidth, DL: dl, VT: MVT::i32)};
18649 auto *MemNode = cast<MemSDNode>(Val&: LHS);
18650 SDValue ConstSt = DAG.getMemIntrinsicNode(
18651 Opcode: PPCISD::STORE_COND, dl,
18652 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::Other, VT3: MVT::Glue), Ops,
18653 MemVT: MemNode->getMemoryVT(), MMO: MemNode->getMemOperand());
18654
18655 SDValue InChain;
18656 // Unchain the branch from the original store conditional.
18657 if (N->getOperand(Num: 0) == LHS.getValue(R: 1))
18658 InChain = LHS.getOperand(i: 0);
18659 else if (N->getOperand(Num: 0).getOpcode() == ISD::TokenFactor) {
18660 SmallVector<SDValue, 4> InChains;
18661 SDValue InTF = N->getOperand(Num: 0);
18662 for (int i = 0, e = InTF.getNumOperands(); i < e; i++)
18663 if (InTF.getOperand(i) != LHS.getValue(R: 1))
18664 InChains.push_back(Elt: InTF.getOperand(i));
18665 InChain = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: InChains);
18666 }
18667
18668 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: InChain,
18669 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18670 N3: DAG.getRegister(Reg: PPC::CR0, VT: MVT::i32), N4: N->getOperand(Num: 4),
18671 N5: ConstSt.getValue(R: 2));
18672 }
18673
18674 if (LHS.getOpcode() == ISD::INTRINSIC_WO_CHAIN &&
18675 getVectorCompareInfo(Intrin: LHS, CompareOpc, isDot, Subtarget)) {
18676 assert(isDot && "Can't compare against a vector result!");
18677
18678 if (SDValue Impossible = isImpossibleCompare())
18679 return Impossible;
18680
18681 bool BranchOnWhenPredTrue = (CC == ISD::SETEQ) ^ (Val == 0);
18682 // Create the PPCISD altivec 'dot' comparison node.
18683 SDValue Ops[] = {
18684 LHS.getOperand(i: 2), // LHS of compare
18685 LHS.getOperand(i: 3), // RHS of compare
18686 DAG.getConstant(Val: CompareOpc, DL: dl, VT: MVT::i32)
18687 };
18688 EVT VTs[] = { LHS.getOperand(i: 2).getValueType(), MVT::Glue };
18689 SDValue CompNode = DAG.getNode(Opcode: PPCISD::VCMP_rec, DL: dl, ResultTys: VTs, Ops);
18690
18691 // Unpack the result based on how the target uses it.
18692 PPC::Predicate CompOpc;
18693 switch (LHS.getConstantOperandVal(i: 1)) {
18694 default: // Can't happen, don't crash on invalid number though.
18695 case 0: // Branch on the value of the EQ bit of CR6.
18696 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_EQ : PPC::PRED_NE;
18697 break;
18698 case 1: // Branch on the inverted value of the EQ bit of CR6.
18699 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_NE : PPC::PRED_EQ;
18700 break;
18701 case 2: // Branch on the value of the LT bit of CR6.
18702 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_LT : PPC::PRED_GE;
18703 break;
18704 case 3: // Branch on the inverted value of the LT bit of CR6.
18705 CompOpc = BranchOnWhenPredTrue ? PPC::PRED_GE : PPC::PRED_LT;
18706 break;
18707 }
18708
18709 return DAG.getNode(Opcode: PPCISD::COND_BRANCH, DL: dl, VT: MVT::Other, N1: N->getOperand(Num: 0),
18710 N2: DAG.getConstant(Val: CompOpc, DL: dl, VT: MVT::i32),
18711 N3: DAG.getRegister(Reg: PPC::CR6, VT: MVT::i32),
18712 N4: N->getOperand(Num: 4), N5: CompNode.getValue(R: 1));
18713 }
18714 break;
18715 }
18716 case ISD::BUILD_VECTOR:
18717 return DAGCombineBuildVector(N, DCI);
18718 case PPCISD::ADDC:
18719 return DAGCombineAddc(N, DCI);
18720 case PPCISD::SUBE:
18721 return DAGCombineSube(N, DCI);
18722
18723 case ISD::BITCAST:
18724 return DAGCombineBitcast(N, DCI);
18725 }
18726
18727 return SDValue();
18728}
18729
18730SDValue
18731PPCTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
18732 SelectionDAG &DAG,
18733 SmallVectorImpl<SDNode *> &Created) const {
18734 // fold (sdiv X, pow2)
18735 EVT VT = N->getValueType(ResNo: 0);
18736 if (VT == MVT::i64 && !Subtarget.isPPC64())
18737 return SDValue();
18738 if ((VT != MVT::i32 && VT != MVT::i64) ||
18739 !(Divisor.isPowerOf2() || Divisor.isNegatedPowerOf2()))
18740 return SDValue();
18741
18742 SDLoc DL(N);
18743 SDValue N0 = N->getOperand(Num: 0);
18744
18745 bool IsNegPow2 = Divisor.isNegatedPowerOf2();
18746 unsigned Lg2 = (IsNegPow2 ? -Divisor : Divisor).countr_zero();
18747 SDValue ShiftAmt = DAG.getConstant(Val: Lg2, DL, VT);
18748
18749 SDValue Op = DAG.getNode(Opcode: PPCISD::SRA_ADDZE, DL, VT, N1: N0, N2: ShiftAmt);
18750 Created.push_back(Elt: Op.getNode());
18751
18752 if (IsNegPow2) {
18753 Op = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Op);
18754 Created.push_back(Elt: Op.getNode());
18755 }
18756
18757 return Op;
18758}
18759
18760//===----------------------------------------------------------------------===//
18761// Inline Assembly Support
18762//===----------------------------------------------------------------------===//
18763
18764void PPCTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
18765 KnownBits &Known,
18766 const APInt &DemandedElts,
18767 const SelectionDAG &DAG,
18768 unsigned Depth) const {
18769 Known.resetAll();
18770 switch (Op.getOpcode()) {
18771 default: break;
18772 case PPCISD::LBRX: {
18773 // lhbrx is known to have the top bits cleared out.
18774 if (cast<VTSDNode>(Val: Op.getOperand(i: 2))->getVT() == MVT::i16)
18775 Known.Zero = 0xFFFF0000;
18776 break;
18777 }
18778 case PPCISD::ADDE: {
18779 if (Op.getResNo() == 0) {
18780 // (0|1), _ = ADDE 0, 0, CARRY
18781 SDValue LHS = Op.getOperand(i: 0);
18782 SDValue RHS = Op.getOperand(i: 1);
18783 if (isNullConstant(V: LHS) && isNullConstant(V: RHS))
18784 Known.Zero = ~1ULL;
18785 }
18786 break;
18787 }
18788 case ISD::INTRINSIC_WO_CHAIN: {
18789 switch (Op.getConstantOperandVal(i: 0)) {
18790 default: break;
18791 case Intrinsic::ppc_altivec_vcmpbfp_p:
18792 case Intrinsic::ppc_altivec_vcmpeqfp_p:
18793 case Intrinsic::ppc_altivec_vcmpequb_p:
18794 case Intrinsic::ppc_altivec_vcmpequh_p:
18795 case Intrinsic::ppc_altivec_vcmpequw_p:
18796 case Intrinsic::ppc_altivec_vcmpequd_p:
18797 case Intrinsic::ppc_altivec_vcmpequq_p:
18798 case Intrinsic::ppc_altivec_vcmpgefp_p:
18799 case Intrinsic::ppc_altivec_vcmpgtfp_p:
18800 case Intrinsic::ppc_altivec_vcmpgtsb_p:
18801 case Intrinsic::ppc_altivec_vcmpgtsh_p:
18802 case Intrinsic::ppc_altivec_vcmpgtsw_p:
18803 case Intrinsic::ppc_altivec_vcmpgtsd_p:
18804 case Intrinsic::ppc_altivec_vcmpgtsq_p:
18805 case Intrinsic::ppc_altivec_vcmpgtub_p:
18806 case Intrinsic::ppc_altivec_vcmpgtuh_p:
18807 case Intrinsic::ppc_altivec_vcmpgtuw_p:
18808 case Intrinsic::ppc_altivec_vcmpgtud_p:
18809 case Intrinsic::ppc_altivec_vcmpgtuq_p:
18810 Known.Zero = ~1U; // All bits but the low one are known to be zero.
18811 break;
18812 }
18813 break;
18814 }
18815 case ISD::INTRINSIC_W_CHAIN: {
18816 switch (Op.getConstantOperandVal(i: 1)) {
18817 default:
18818 break;
18819 case Intrinsic::ppc_load2r:
18820 // Top bits are cleared for load2r (which is the same as lhbrx).
18821 Known.Zero = 0xFFFF0000;
18822 break;
18823 }
18824 break;
18825 }
18826 }
18827}
18828
18829Align PPCTargetLowering::getPrefLoopAlignment(
18830 MachineLoop *ML, const MachineBasicBlock *BlockToAlign) const {
18831 switch (Subtarget.getCPUDirective()) {
18832 default: break;
18833 case PPC::DIR_970:
18834 case PPC::DIR_PWR4:
18835 case PPC::DIR_PWR5:
18836 case PPC::DIR_PWR5X:
18837 case PPC::DIR_PWR6:
18838 case PPC::DIR_PWR6X:
18839 case PPC::DIR_PWR7:
18840 case PPC::DIR_PWR8:
18841 case PPC::DIR_PWR9:
18842 case PPC::DIR_PWR10:
18843 case PPC::DIR_PWR11:
18844 case PPC::DIR_PWR_FUTURE: {
18845 if (!ML)
18846 break;
18847
18848 if (EnableInnermostLoopAlign32) {
18849 // If the nested loop is an innermost loop, prefer to a 32-byte alignment,
18850 // so that we can decrease cache misses and branch-prediction misses.
18851 // Actual alignment of the loop will depend on the hotness check and other
18852 // logic in alignBlocks.
18853 if (ML->getLoopDepth() > 1 && ML->getSubLoops().empty())
18854 return Align(32);
18855 }
18856
18857 const PPCInstrInfo *TII = Subtarget.getInstrInfo();
18858
18859 // For small loops (between 5 and 8 instructions), align to a 32-byte
18860 // boundary so that the entire loop fits in one instruction-cache line.
18861 uint64_t LoopSize = 0;
18862 for (auto I = ML->block_begin(), IE = ML->block_end(); I != IE; ++I)
18863 for (const MachineInstr &J : **I) {
18864 LoopSize += TII->getInstSizeInBytes(MI: J);
18865 if (LoopSize > 32)
18866 break;
18867 }
18868
18869 if (LoopSize > 16 && LoopSize <= 32)
18870 return Align(32);
18871
18872 break;
18873 }
18874 }
18875
18876 return TargetLowering::getPrefLoopAlignment(ML);
18877}
18878
18879/// getConstraintType - Given a constraint, return the type of
18880/// constraint it is for this target.
18881PPCTargetLowering::ConstraintType
18882PPCTargetLowering::getConstraintType(StringRef Constraint) const {
18883 if (Constraint.size() == 1) {
18884 switch (Constraint[0]) {
18885 default: break;
18886 case 'b':
18887 case 'r':
18888 case 'f':
18889 case 'd':
18890 case 'v':
18891 case 'y':
18892 return C_RegisterClass;
18893 case 'Z':
18894 // FIXME: While Z does indicate a memory constraint, it specifically
18895 // indicates an r+r address (used in conjunction with the 'y' modifier
18896 // in the replacement string). Currently, we're forcing the base
18897 // register to be r0 in the asm printer (which is interpreted as zero)
18898 // and forming the complete address in the second register. This is
18899 // suboptimal.
18900 return C_Memory;
18901 }
18902 } else if (Constraint == "wc") { // individual CR bits.
18903 return C_RegisterClass;
18904 } else if (Constraint == "wa" || Constraint == "wd" ||
18905 Constraint == "wf" || Constraint == "ws" ||
18906 Constraint == "wi" || Constraint == "ww") {
18907 return C_RegisterClass; // VSX registers.
18908 }
18909 return TargetLowering::getConstraintType(Constraint);
18910}
18911
18912/// Examine constraint type and operand type and determine a weight value.
18913/// This object must already have been set up with the operand type
18914/// and the current alternative constraint selected.
18915TargetLowering::ConstraintWeight
18916PPCTargetLowering::getSingleConstraintMatchWeight(
18917 AsmOperandInfo &info, const char *constraint) const {
18918 ConstraintWeight weight = CW_Invalid;
18919 Value *CallOperandVal = info.CallOperandVal;
18920 // If we don't have a value, we can't do a match,
18921 // but allow it at the lowest weight.
18922 if (!CallOperandVal)
18923 return CW_Default;
18924 Type *type = CallOperandVal->getType();
18925
18926 // Look at the constraint type.
18927 if (StringRef(constraint) == "wc" && type->isIntegerTy(BitWidth: 1))
18928 return CW_Register; // an individual CR bit.
18929 else if ((StringRef(constraint) == "wa" ||
18930 StringRef(constraint) == "wd" ||
18931 StringRef(constraint) == "wf") &&
18932 type->isVectorTy())
18933 return CW_Register;
18934 else if (StringRef(constraint) == "wi" && type->isIntegerTy(BitWidth: 64))
18935 return CW_Register; // just hold 64-bit integers data.
18936 else if (StringRef(constraint) == "ws" && type->isDoubleTy())
18937 return CW_Register;
18938 else if (StringRef(constraint) == "ww" && type->isFloatTy())
18939 return CW_Register;
18940
18941 switch (*constraint) {
18942 default:
18943 weight = TargetLowering::getSingleConstraintMatchWeight(info, constraint);
18944 break;
18945 case 'b':
18946 if (type->isIntegerTy())
18947 weight = CW_Register;
18948 break;
18949 case 'f':
18950 if (type->isFloatTy())
18951 weight = CW_Register;
18952 break;
18953 case 'd':
18954 if (type->isDoubleTy())
18955 weight = CW_Register;
18956 break;
18957 case 'v':
18958 if (type->isVectorTy())
18959 weight = CW_Register;
18960 break;
18961 case 'y':
18962 weight = CW_Register;
18963 break;
18964 case 'Z':
18965 weight = CW_Memory;
18966 break;
18967 }
18968 return weight;
18969}
18970
18971std::pair<unsigned, const TargetRegisterClass *>
18972PPCTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
18973 StringRef Constraint,
18974 MVT VT) const {
18975 if (Constraint.size() == 1) {
18976 // GCC RS6000 Constraint Letters
18977 switch (Constraint[0]) {
18978 case 'b': // R1-R31
18979 if (VT == MVT::i64 && Subtarget.isPPC64())
18980 return std::make_pair(x: 0U, y: &PPC::G8RC_NOX0RegClass);
18981 return std::make_pair(x: 0U, y: &PPC::GPRC_NOR0RegClass);
18982 case 'r': // R0-R31
18983 if (VT == MVT::i64 && Subtarget.isPPC64())
18984 return std::make_pair(x: 0U, y: &PPC::G8RCRegClass);
18985 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
18986 // 'd' and 'f' constraints are both defined to be "the floating point
18987 // registers", where one is for 32-bit and the other for 64-bit. We don't
18988 // really care overly much here so just give them all the same reg classes.
18989 case 'd':
18990 case 'f':
18991 if (Subtarget.hasSPE()) {
18992 if (VT == MVT::f32 || VT == MVT::i32)
18993 return std::make_pair(x: 0U, y: &PPC::GPRCRegClass);
18994 if (VT == MVT::f64 || VT == MVT::i64)
18995 return std::make_pair(x: 0U, y: &PPC::SPERCRegClass);
18996 } else {
18997 if (VT == MVT::f32 || VT == MVT::i32)
18998 return std::make_pair(x: 0U, y: &PPC::F4RCRegClass);
18999 if (VT == MVT::f64 || VT == MVT::i64)
19000 return std::make_pair(x: 0U, y: &PPC::F8RCRegClass);
19001 }
19002 break;
19003 case 'v':
19004 if (Subtarget.hasAltivec() && VT.isVector())
19005 return std::make_pair(x: 0U, y: &PPC::VRRCRegClass);
19006 else if (Subtarget.hasVSX())
19007 // Scalars in Altivec registers only make sense with VSX.
19008 return std::make_pair(x: 0U, y: &PPC::VFRCRegClass);
19009 break;
19010 case 'y': // crrc
19011 return std::make_pair(x: 0U, y: &PPC::CRRCRegClass);
19012 }
19013 } else if (Constraint == "wc" && Subtarget.useCRBits()) {
19014 // An individual CR bit.
19015 return std::make_pair(x: 0U, y: &PPC::CRBITRCRegClass);
19016 } else if ((Constraint == "wa" || Constraint == "wd" ||
19017 Constraint == "wf" || Constraint == "wi") &&
19018 Subtarget.hasVSX()) {
19019 // A VSX register for either a scalar (FP) or vector. There is no
19020 // support for single precision scalars on subtargets prior to Power8.
19021 if (VT.isVector())
19022 return std::make_pair(x: 0U, y: &PPC::VSRCRegClass);
19023 if (VT == MVT::f32 && Subtarget.hasP8Vector())
19024 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
19025 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
19026 } else if ((Constraint == "ws" || Constraint == "ww") && Subtarget.hasVSX()) {
19027 if (VT == MVT::f32 && Subtarget.hasP8Vector())
19028 return std::make_pair(x: 0U, y: &PPC::VSSRCRegClass);
19029 else
19030 return std::make_pair(x: 0U, y: &PPC::VSFRCRegClass);
19031 } else if (Constraint == "lr") {
19032 if (VT == MVT::i64)
19033 return std::make_pair(x: 0U, y: &PPC::LR8RCRegClass);
19034 else
19035 return std::make_pair(x: 0U, y: &PPC::LRRCRegClass);
19036 }
19037
19038 // Handle special cases of physical registers that are not properly handled
19039 // by the base class.
19040 if (Constraint[0] == '{' && Constraint[Constraint.size() - 1] == '}') {
19041 // If we name a VSX register, we can't defer to the base class because it
19042 // will not recognize the correct register (their names will be VSL{0-31}
19043 // and V{0-31} so they won't match). So we match them here.
19044 if (Constraint.size() > 3 && Constraint[1] == 'v' && Constraint[2] == 's') {
19045 int VSNum = atoi(nptr: Constraint.data() + 3);
19046 assert(VSNum >= 0 && VSNum <= 63 &&
19047 "Attempted to access a vsr out of range");
19048 if (VSNum < 32)
19049 return std::make_pair(x: PPC::VSL0 + VSNum, y: &PPC::VSRCRegClass);
19050 return std::make_pair(x: PPC::V0 + VSNum - 32, y: &PPC::VSRCRegClass);
19051 }
19052
19053 // For float registers, we can't defer to the base class as it will match
19054 // the SPILLTOVSRRC class.
19055 if (Constraint.size() > 3 && Constraint[1] == 'f') {
19056 int RegNum = atoi(nptr: Constraint.data() + 2);
19057 if (RegNum > 31 || RegNum < 0)
19058 report_fatal_error(reason: "Invalid floating point register number");
19059 if (VT == MVT::f32 || VT == MVT::i32)
19060 return Subtarget.hasSPE()
19061 ? std::make_pair(x: PPC::R0 + RegNum, y: &PPC::GPRCRegClass)
19062 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F4RCRegClass);
19063 if (VT == MVT::f64 || VT == MVT::i64)
19064 return Subtarget.hasSPE()
19065 ? std::make_pair(x: PPC::S0 + RegNum, y: &PPC::SPERCRegClass)
19066 : std::make_pair(x: PPC::F0 + RegNum, y: &PPC::F8RCRegClass);
19067 }
19068 }
19069
19070 std::pair<unsigned, const TargetRegisterClass *> R =
19071 TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
19072
19073 // r[0-9]+ are used, on PPC64, to refer to the corresponding 64-bit registers
19074 // (which we call X[0-9]+). If a 64-bit value has been requested, and a
19075 // 32-bit GPR has been selected, then 'upgrade' it to the 64-bit parent
19076 // register.
19077 // FIXME: If TargetLowering::getRegForInlineAsmConstraint could somehow use
19078 // the AsmName field from *RegisterInfo.td, then this would not be necessary.
19079 if (R.first && VT == MVT::i64 && Subtarget.isPPC64() &&
19080 PPC::GPRCRegClass.contains(Reg: R.first))
19081 return std::make_pair(x: TRI->getMatchingSuperReg(Reg: R.first,
19082 SubIdx: PPC::sub_32, RC: &PPC::G8RCRegClass),
19083 y: &PPC::G8RCRegClass);
19084
19085 // GCC accepts 'cc' as an alias for 'cr0', and we need to do the same.
19086 if (!R.second && StringRef("{cc}").equals_insensitive(RHS: Constraint)) {
19087 R.first = PPC::CR0;
19088 R.second = &PPC::CRRCRegClass;
19089 }
19090 // FIXME: This warning should ideally be emitted in the front end.
19091 if (Subtarget.isAIXABI() && !Subtarget.isAIXExtendedAltivecABI()) {
19092 if (((R.first >= PPC::V20 && R.first <= PPC::V31) ||
19093 (R.first >= PPC::VF20 && R.first <= PPC::VF31)) &&
19094 (R.second == &PPC::VSRCRegClass || R.second == &PPC::VSFRCRegClass))
19095 errs() << "warning: vector registers 20 to 32 are reserved in the "
19096 "default AIX AltiVec ABI and cannot be used\n";
19097 }
19098
19099 return R;
19100}
19101
19102/// LowerAsmOperandForConstraint - Lower the specified operand into the Ops
19103/// vector. If it is invalid, don't add anything to Ops.
19104void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op,
19105 StringRef Constraint,
19106 std::vector<SDValue> &Ops,
19107 SelectionDAG &DAG) const {
19108 SDValue Result;
19109
19110 // Only support length 1 constraints.
19111 if (Constraint.size() > 1)
19112 return;
19113
19114 char Letter = Constraint[0];
19115 switch (Letter) {
19116 default: break;
19117 case 'I':
19118 case 'J':
19119 case 'K':
19120 case 'L':
19121 case 'M':
19122 case 'N':
19123 case 'O':
19124 case 'P': {
19125 ConstantSDNode *CST = dyn_cast<ConstantSDNode>(Val&: Op);
19126 if (!CST) return; // Must be an immediate to match.
19127 SDLoc dl(Op);
19128 int64_t Value = CST->getSExtValue();
19129 EVT TCVT = MVT::i64; // All constants taken to be 64 bits so that negative
19130 // numbers are printed as such.
19131 switch (Letter) {
19132 default: llvm_unreachable("Unknown constraint letter!");
19133 case 'I': // "I" is a signed 16-bit constant.
19134 if (isInt<16>(x: Value))
19135 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19136 break;
19137 case 'J': // "J" is a constant with only the high-order 16 bits nonzero.
19138 if (isShiftedUInt<16, 16>(x: Value))
19139 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19140 break;
19141 case 'L': // "L" is a signed 16-bit constant shifted left 16 bits.
19142 if (isShiftedInt<16, 16>(x: Value))
19143 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19144 break;
19145 case 'K': // "K" is a constant with only the low-order 16 bits nonzero.
19146 if (isUInt<16>(x: Value))
19147 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19148 break;
19149 case 'M': // "M" is a constant that is greater than 31.
19150 if (Value > 31)
19151 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19152 break;
19153 case 'N': // "N" is a positive constant that is an exact power of two.
19154 if (Value > 0 && isPowerOf2_64(Value))
19155 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19156 break;
19157 case 'O': // "O" is the constant zero.
19158 if (Value == 0)
19159 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19160 break;
19161 case 'P': // "P" is a constant whose negation is a signed 16-bit constant.
19162 if (isInt<16>(x: -Value))
19163 Result = DAG.getTargetConstant(Val: Value, DL: dl, VT: TCVT);
19164 break;
19165 }
19166 break;
19167 }
19168 }
19169
19170 if (Result.getNode()) {
19171 Ops.push_back(x: Result);
19172 return;
19173 }
19174
19175 // Handle standard constraint letters.
19176 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
19177}
19178
19179void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I,
19180 SmallVectorImpl<SDValue> &Ops,
19181 SelectionDAG &DAG) const {
19182 if (I.getNumOperands() <= 1)
19183 return;
19184 if (!isa<ConstantSDNode>(Val: Ops[1].getNode()))
19185 return;
19186 auto IntrinsicID = Ops[1].getNode()->getAsZExtVal();
19187 if (IntrinsicID != Intrinsic::ppc_tdw && IntrinsicID != Intrinsic::ppc_tw &&
19188 IntrinsicID != Intrinsic::ppc_trapd && IntrinsicID != Intrinsic::ppc_trap)
19189 return;
19190
19191 if (MDNode *MDN = I.getMetadata(KindID: LLVMContext::MD_annotation))
19192 Ops.push_back(Elt: DAG.getMDNode(MD: MDN));
19193}
19194
19195// isLegalAddressingMode - Return true if the addressing mode represented
19196// by AM is legal for this target, for a load/store of the specified type.
19197bool PPCTargetLowering::isLegalAddressingMode(const DataLayout &DL,
19198 const AddrMode &AM, Type *Ty,
19199 unsigned AS,
19200 Instruction *I) const {
19201 // Vector type r+i form is supported since power9 as DQ form. We don't check
19202 // the offset matching DQ form requirement(off % 16 == 0), because on PowerPC,
19203 // imm form is preferred and the offset can be adjusted to use imm form later
19204 // in pass PPCLoopInstrFormPrep. Also in LSR, for one LSRUse, it uses min and
19205 // max offset to check legal addressing mode, we should be a little aggressive
19206 // to contain other offsets for that LSRUse.
19207 if (Ty->isVectorTy() && AM.BaseOffs != 0 && !Subtarget.hasP9Vector())
19208 return false;
19209
19210 // PPC allows a sign-extended 16-bit immediate field.
19211 if (AM.BaseOffs <= -(1LL << 16) || AM.BaseOffs >= (1LL << 16)-1)
19212 return false;
19213
19214 // No global is ever allowed as a base.
19215 if (AM.BaseGV)
19216 return false;
19217
19218 // PPC only support r+r,
19219 switch (AM.Scale) {
19220 case 0: // "r+i" or just "i", depending on HasBaseReg.
19221 break;
19222 case 1:
19223 if (AM.HasBaseReg && AM.BaseOffs) // "r+r+i" is not allowed.
19224 return false;
19225 // Otherwise we have r+r or r+i.
19226 break;
19227 case 2:
19228 if (AM.HasBaseReg || AM.BaseOffs) // 2*r+r or 2*r+i is not allowed.
19229 return false;
19230 // Allow 2*r as r+r.
19231 break;
19232 default:
19233 // No other scales are supported.
19234 return false;
19235 }
19236
19237 return true;
19238}
19239
19240SDValue PPCTargetLowering::LowerRETURNADDR(SDValue Op,
19241 SelectionDAG &DAG) const {
19242 MachineFunction &MF = DAG.getMachineFunction();
19243 MachineFrameInfo &MFI = MF.getFrameInfo();
19244 MFI.setReturnAddressIsTaken(true);
19245
19246 SDLoc dl(Op);
19247 unsigned Depth = Op.getConstantOperandVal(i: 0);
19248
19249 // Make sure the function does not optimize away the store of the RA to
19250 // the stack.
19251 PPCFunctionInfo *FuncInfo = MF.getInfo<PPCFunctionInfo>();
19252 FuncInfo->setLRStoreRequired();
19253 auto PtrVT = getPointerTy(DL: MF.getDataLayout());
19254
19255 if (Depth > 0) {
19256 // The link register (return address) is saved in the caller's frame
19257 // not the callee's stack frame. So we must get the caller's frame
19258 // address and load the return address at the LR offset from there.
19259 SDValue FrameAddr =
19260 DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19261 Ptr: LowerFRAMEADDR(Op, DAG), PtrInfo: MachinePointerInfo());
19262 SDValue Offset =
19263 DAG.getConstant(Val: Subtarget.getFrameLowering()->getReturnSaveOffset(), DL: dl,
19264 VT: Subtarget.getScalarIntVT());
19265 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(),
19266 Ptr: DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: PtrVT, N1: FrameAddr, N2: Offset),
19267 PtrInfo: MachinePointerInfo());
19268 }
19269
19270 // Just load the return address off the stack.
19271 SDValue RetAddrFI = getReturnAddrFrameIndex(DAG);
19272 return DAG.getLoad(VT: PtrVT, dl, Chain: DAG.getEntryNode(), Ptr: RetAddrFI,
19273 PtrInfo: MachinePointerInfo());
19274}
19275
19276SDValue PPCTargetLowering::LowerFRAMEADDR(SDValue Op,
19277 SelectionDAG &DAG) const {
19278 SDLoc dl(Op);
19279 unsigned Depth = Op.getConstantOperandVal(i: 0);
19280
19281 MachineFunction &MF = DAG.getMachineFunction();
19282 MachineFrameInfo &MFI = MF.getFrameInfo();
19283 MFI.setFrameAddressIsTaken(true);
19284
19285 EVT PtrVT = getPointerTy(DL: MF.getDataLayout());
19286 bool isPPC64 = PtrVT == MVT::i64;
19287
19288 // Naked functions never have a frame pointer, and so we use r1. For all
19289 // other functions, this decision must be delayed until during PEI.
19290 unsigned FrameReg;
19291 if (MF.getFunction().hasFnAttribute(Kind: Attribute::Naked))
19292 FrameReg = isPPC64 ? PPC::X1 : PPC::R1;
19293 else
19294 FrameReg = isPPC64 ? PPC::FP8 : PPC::FP;
19295
19296 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl, Reg: FrameReg,
19297 VT: PtrVT);
19298 while (Depth--)
19299 FrameAddr = DAG.getLoad(VT: Op.getValueType(), dl, Chain: DAG.getEntryNode(),
19300 Ptr: FrameAddr, PtrInfo: MachinePointerInfo());
19301 return FrameAddr;
19302}
19303
19304#define GET_REGISTER_MATCHER
19305#include "PPCGenAsmMatcher.inc"
19306
19307Register PPCTargetLowering::getRegisterByName(const char *RegName, LLT VT,
19308 const MachineFunction &MF) const {
19309 bool IsPPC64 = Subtarget.isPPC64();
19310
19311 bool Is64Bit = IsPPC64 && VT == LLT::scalar(SizeInBits: 64);
19312 if (!Is64Bit && VT != LLT::scalar(SizeInBits: 32))
19313 report_fatal_error(reason: "Invalid register global variable type");
19314
19315 Register Reg = MatchRegisterName(Name: RegName);
19316 if (!Reg)
19317 return Reg;
19318
19319 // FIXME: Unable to generate code for `-O2` but okay for `-O0`.
19320 // Need followup investigation as to why.
19321 if ((IsPPC64 && Reg == PPC::R2) || Reg == PPC::R0)
19322 report_fatal_error(reason: Twine("Trying to reserve an invalid register \"" +
19323 StringRef(RegName) + "\"."));
19324
19325 // Convert GPR to GP8R register for 64bit.
19326 if (Is64Bit && StringRef(RegName).starts_with_insensitive(Prefix: "r"))
19327 Reg = Reg.id() - PPC::R0 + PPC::X0;
19328
19329 return Reg;
19330}
19331
19332bool PPCTargetLowering::isAccessedAsGotIndirect(SDValue GA) const {
19333 // 32-bit SVR4 ABI access everything as got-indirect.
19334 if (Subtarget.is32BitELFABI())
19335 return true;
19336
19337 // AIX accesses everything indirectly through the TOC, which is similar to
19338 // the GOT.
19339 if (Subtarget.isAIXABI())
19340 return true;
19341
19342 CodeModel::Model CModel = getTargetMachine().getCodeModel();
19343 // If it is small or large code model, module locals are accessed
19344 // indirectly by loading their address from .toc/.got.
19345 if (CModel == CodeModel::Small || CModel == CodeModel::Large)
19346 return true;
19347
19348 // JumpTable and BlockAddress are accessed as got-indirect.
19349 if (isa<JumpTableSDNode>(Val: GA) || isa<BlockAddressSDNode>(Val: GA))
19350 return true;
19351
19352 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(Val&: GA))
19353 return Subtarget.isGVIndirectSymbol(GV: G->getGlobal());
19354
19355 return false;
19356}
19357
19358bool
19359PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const {
19360 // The PowerPC target isn't yet aware of offsets.
19361 return false;
19362}
19363
19364void PPCTargetLowering::getTgtMemIntrinsic(
19365 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
19366 MachineFunction &MF, unsigned Intrinsic) const {
19367 IntrinsicInfo Info;
19368 switch (Intrinsic) {
19369 case Intrinsic::ppc_atomicrmw_xchg_i128:
19370 case Intrinsic::ppc_atomicrmw_add_i128:
19371 case Intrinsic::ppc_atomicrmw_sub_i128:
19372 case Intrinsic::ppc_atomicrmw_nand_i128:
19373 case Intrinsic::ppc_atomicrmw_and_i128:
19374 case Intrinsic::ppc_atomicrmw_or_i128:
19375 case Intrinsic::ppc_atomicrmw_xor_i128:
19376 case Intrinsic::ppc_cmpxchg_i128:
19377 Info.opc = ISD::INTRINSIC_W_CHAIN;
19378 Info.memVT = MVT::i128;
19379 Info.ptrVal = I.getArgOperand(i: 0);
19380 Info.offset = 0;
19381 Info.align = Align(16);
19382 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
19383 MachineMemOperand::MOVolatile;
19384 Infos.push_back(Elt: Info);
19385 return;
19386 case Intrinsic::ppc_atomic_load_i128:
19387 Info.opc = ISD::INTRINSIC_W_CHAIN;
19388 Info.memVT = MVT::i128;
19389 Info.ptrVal = I.getArgOperand(i: 0);
19390 Info.offset = 0;
19391 Info.align = Align(16);
19392 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOVolatile;
19393 Infos.push_back(Elt: Info);
19394 return;
19395 case Intrinsic::ppc_atomic_store_i128:
19396 Info.opc = ISD::INTRINSIC_VOID;
19397 Info.memVT = MVT::i128;
19398 Info.ptrVal = I.getArgOperand(i: 2);
19399 Info.offset = 0;
19400 Info.align = Align(16);
19401 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19402 Infos.push_back(Elt: Info);
19403 return;
19404 case Intrinsic::ppc_altivec_lvx:
19405 case Intrinsic::ppc_altivec_lvxl:
19406 case Intrinsic::ppc_altivec_lvebx:
19407 case Intrinsic::ppc_altivec_lvehx:
19408 case Intrinsic::ppc_altivec_lvewx:
19409 case Intrinsic::ppc_vsx_lxvd2x:
19410 case Intrinsic::ppc_vsx_lxvw4x:
19411 case Intrinsic::ppc_vsx_lxvd2x_be:
19412 case Intrinsic::ppc_vsx_lxvw4x_be:
19413 case Intrinsic::ppc_vsx_lxvl:
19414 case Intrinsic::ppc_vsx_lxvll: {
19415 EVT VT;
19416 switch (Intrinsic) {
19417 case Intrinsic::ppc_altivec_lvebx:
19418 VT = MVT::i8;
19419 break;
19420 case Intrinsic::ppc_altivec_lvehx:
19421 VT = MVT::i16;
19422 break;
19423 case Intrinsic::ppc_altivec_lvewx:
19424 VT = MVT::i32;
19425 break;
19426 case Intrinsic::ppc_vsx_lxvd2x:
19427 case Intrinsic::ppc_vsx_lxvd2x_be:
19428 VT = MVT::v2f64;
19429 break;
19430 default:
19431 VT = MVT::v4i32;
19432 break;
19433 }
19434
19435 Info.opc = ISD::INTRINSIC_W_CHAIN;
19436 Info.memVT = VT;
19437 Info.ptrVal = I.getArgOperand(i: 0);
19438 Info.offset = -VT.getStoreSize()+1;
19439 Info.size = 2*VT.getStoreSize()-1;
19440 Info.align = Align(1);
19441 Info.flags = MachineMemOperand::MOLoad;
19442 Infos.push_back(Elt: Info);
19443 return;
19444 }
19445 case Intrinsic::ppc_altivec_stvx:
19446 case Intrinsic::ppc_altivec_stvxl:
19447 case Intrinsic::ppc_altivec_stvebx:
19448 case Intrinsic::ppc_altivec_stvehx:
19449 case Intrinsic::ppc_altivec_stvewx:
19450 case Intrinsic::ppc_vsx_stxvd2x:
19451 case Intrinsic::ppc_vsx_stxvw4x:
19452 case Intrinsic::ppc_vsx_stxvd2x_be:
19453 case Intrinsic::ppc_vsx_stxvw4x_be:
19454 case Intrinsic::ppc_vsx_stxvl:
19455 case Intrinsic::ppc_vsx_stxvll: {
19456 EVT VT;
19457 switch (Intrinsic) {
19458 case Intrinsic::ppc_altivec_stvebx:
19459 VT = MVT::i8;
19460 break;
19461 case Intrinsic::ppc_altivec_stvehx:
19462 VT = MVT::i16;
19463 break;
19464 case Intrinsic::ppc_altivec_stvewx:
19465 VT = MVT::i32;
19466 break;
19467 case Intrinsic::ppc_vsx_stxvd2x:
19468 case Intrinsic::ppc_vsx_stxvd2x_be:
19469 VT = MVT::v2f64;
19470 break;
19471 default:
19472 VT = MVT::v4i32;
19473 break;
19474 }
19475
19476 Info.opc = ISD::INTRINSIC_VOID;
19477 Info.memVT = VT;
19478 Info.ptrVal = I.getArgOperand(i: 1);
19479 Info.offset = -VT.getStoreSize()+1;
19480 Info.size = 2*VT.getStoreSize()-1;
19481 Info.align = Align(1);
19482 Info.flags = MachineMemOperand::MOStore;
19483 Infos.push_back(Elt: Info);
19484 return;
19485 }
19486 case Intrinsic::ppc_stdcx:
19487 case Intrinsic::ppc_stwcx:
19488 case Intrinsic::ppc_sthcx:
19489 case Intrinsic::ppc_stbcx: {
19490 EVT VT;
19491 auto Alignment = Align(8);
19492 switch (Intrinsic) {
19493 case Intrinsic::ppc_stdcx:
19494 VT = MVT::i64;
19495 break;
19496 case Intrinsic::ppc_stwcx:
19497 VT = MVT::i32;
19498 Alignment = Align(4);
19499 break;
19500 case Intrinsic::ppc_sthcx:
19501 VT = MVT::i16;
19502 Alignment = Align(2);
19503 break;
19504 case Intrinsic::ppc_stbcx:
19505 VT = MVT::i8;
19506 Alignment = Align(1);
19507 break;
19508 }
19509 Info.opc = ISD::INTRINSIC_W_CHAIN;
19510 Info.memVT = VT;
19511 Info.ptrVal = I.getArgOperand(i: 0);
19512 Info.offset = 0;
19513 Info.align = Alignment;
19514 Info.flags = MachineMemOperand::MOStore | MachineMemOperand::MOVolatile;
19515 Infos.push_back(Elt: Info);
19516 return;
19517 }
19518 default:
19519 break;
19520 }
19521}
19522
19523/// It returns EVT::Other if the type should be determined using generic
19524/// target-independent logic.
19525EVT PPCTargetLowering::getOptimalMemOpType(
19526 LLVMContext &Context, const MemOp &Op,
19527 const AttributeList &FuncAttributes) const {
19528 if (getTargetMachine().getOptLevel() != CodeGenOptLevel::None) {
19529 // We should use Altivec/VSX loads and stores when available. For unaligned
19530 // addresses, unaligned VSX loads are only fast starting with the P8.
19531 if (Subtarget.hasAltivec() && Op.size() >= 16) {
19532 if (Op.isMemset() && Subtarget.hasVSX()) {
19533 uint64_t TailSize = Op.size() % 16;
19534 // For memset lowering, EXTRACT_VECTOR_ELT tries to return constant
19535 // element if vector element type matches tail store. For tail size
19536 // 3/4, the tail store is i32, v4i32 cannot be used, need a legal one.
19537 if (TailSize > 2 && TailSize <= 4) {
19538 return MVT::v8i16;
19539 }
19540 return MVT::v4i32;
19541 }
19542 if (Op.isAligned(AlignCheck: Align(16)) || Subtarget.hasP8Vector())
19543 return MVT::v4i32;
19544 }
19545 }
19546
19547 if (Subtarget.isPPC64()) {
19548 return MVT::i64;
19549 }
19550
19551 return MVT::i32;
19552}
19553
19554/// Returns true if it is beneficial to convert a load of a constant
19555/// to just the constant itself.
19556bool PPCTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
19557 Type *Ty) const {
19558 assert(Ty->isIntegerTy());
19559
19560 unsigned BitSize = Ty->getPrimitiveSizeInBits();
19561 return !(BitSize == 0 || BitSize > 64);
19562}
19563
19564bool PPCTargetLowering::isTruncateFree(Type *Ty1, Type *Ty2) const {
19565 if (!Ty1->isIntegerTy() || !Ty2->isIntegerTy())
19566 return false;
19567 unsigned NumBits1 = Ty1->getPrimitiveSizeInBits();
19568 unsigned NumBits2 = Ty2->getPrimitiveSizeInBits();
19569 return NumBits1 == 64 && NumBits2 == 32;
19570}
19571
19572bool PPCTargetLowering::isTruncateFree(EVT VT1, EVT VT2) const {
19573 if (!VT1.isInteger() || !VT2.isInteger())
19574 return false;
19575 unsigned NumBits1 = VT1.getSizeInBits();
19576 unsigned NumBits2 = VT2.getSizeInBits();
19577 return NumBits1 == 64 && NumBits2 == 32;
19578}
19579
19580bool PPCTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
19581 // Generally speaking, zexts are not free, but they are free when they can be
19582 // folded with other operations.
19583 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val)) {
19584 EVT MemVT = LD->getMemoryVT();
19585 if ((MemVT == MVT::i1 || MemVT == MVT::i8 || MemVT == MVT::i16 ||
19586 (Subtarget.isPPC64() && MemVT == MVT::i32)) &&
19587 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
19588 LD->getExtensionType() == ISD::ZEXTLOAD))
19589 return true;
19590 }
19591
19592 // FIXME: Add other cases...
19593 // - 32-bit shifts with a zext to i64
19594 // - zext after ctlz, bswap, etc.
19595 // - zext after and by a constant mask
19596
19597 return TargetLowering::isZExtFree(Val, VT2);
19598}
19599
19600bool PPCTargetLowering::isFPExtFree(EVT DestVT, EVT SrcVT) const {
19601 assert(DestVT.isFloatingPoint() && SrcVT.isFloatingPoint() &&
19602 "invalid fpext types");
19603 // Extending to float128 is not free.
19604 if (DestVT == MVT::f128)
19605 return false;
19606 return true;
19607}
19608
19609bool PPCTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
19610 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19611}
19612
19613bool PPCTargetLowering::isLegalAddImmediate(int64_t Imm) const {
19614 return isInt<16>(x: Imm) || isUInt<16>(x: Imm);
19615}
19616
19617bool PPCTargetLowering::allowsMisalignedMemoryAccesses(EVT VT, unsigned, Align,
19618 MachineMemOperand::Flags,
19619 unsigned *Fast) const {
19620 if (!EnablePPCUnaligned)
19621 return false;
19622
19623 // PowerPC supports unaligned memory access for simple non-vector types.
19624 // Although accessing unaligned addresses is not as efficient as accessing
19625 // aligned addresses, it is generally more efficient than manual expansion,
19626 // and generally only traps for software emulation when crossing page
19627 // boundaries.
19628
19629 if (!VT.isSimple())
19630 return false;
19631
19632 if (VT.isFloatingPoint() && !VT.isVector() &&
19633 !Subtarget.allowsUnalignedFPAccess())
19634 return false;
19635
19636 if (VT.getSimpleVT().isVector()) {
19637 if (Subtarget.hasVSX()) {
19638 if (VT != MVT::v2f64 && VT != MVT::v2i64 &&
19639 VT != MVT::v4f32 && VT != MVT::v4i32)
19640 return false;
19641 } else {
19642 return false;
19643 }
19644 }
19645
19646 if (VT == MVT::ppcf128)
19647 return false;
19648
19649 if (Fast)
19650 *Fast = 1;
19651
19652 return true;
19653}
19654
19655bool PPCTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
19656 SDValue C) const {
19657 // Check integral scalar types.
19658 if (!VT.isScalarInteger())
19659 return false;
19660 if (auto *ConstNode = dyn_cast<ConstantSDNode>(Val: C.getNode())) {
19661 if (!ConstNode->getAPIntValue().isSignedIntN(N: 64))
19662 return false;
19663 // This transformation will generate >= 2 operations. But the following
19664 // cases will generate <= 2 instructions during ISEL. So exclude them.
19665 // 1. If the constant multiplier fits 16 bits, it can be handled by one
19666 // HW instruction, ie. MULLI
19667 // 2. If the multiplier after shifted fits 16 bits, an extra shift
19668 // instruction is needed than case 1, ie. MULLI and RLDICR
19669 int64_t Imm = ConstNode->getSExtValue();
19670 unsigned Shift = llvm::countr_zero<uint64_t>(Val: Imm);
19671 Imm >>= Shift;
19672 if (isInt<16>(x: Imm))
19673 return false;
19674 uint64_t UImm = static_cast<uint64_t>(Imm);
19675 if (isPowerOf2_64(Value: UImm + 1) || isPowerOf2_64(Value: UImm - 1) ||
19676 isPowerOf2_64(Value: 1 - UImm) || isPowerOf2_64(Value: -1 - UImm))
19677 return true;
19678 }
19679 return false;
19680}
19681
19682bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
19683 EVT VT) const {
19684 return isFMAFasterThanFMulAndFAdd(
19685 F: MF.getFunction(), Ty: VT.getTypeForEVT(Context&: MF.getFunction().getContext()));
19686}
19687
19688bool PPCTargetLowering::isFMAFasterThanFMulAndFAdd(const Function &F,
19689 Type *Ty) const {
19690 if (Subtarget.hasSPE() || Subtarget.useSoftFloat())
19691 return false;
19692 switch (Ty->getScalarType()->getTypeID()) {
19693 case Type::FloatTyID:
19694 case Type::DoubleTyID:
19695 return true;
19696 case Type::FP128TyID:
19697 return Subtarget.hasP9Vector();
19698 default:
19699 return false;
19700 }
19701}
19702
19703// FIXME: add more patterns which are not profitable to hoist.
19704bool PPCTargetLowering::isProfitableToHoist(Instruction *I) const {
19705 if (!I->hasOneUse())
19706 return true;
19707
19708 Instruction *User = I->user_back();
19709 assert(User && "A single use instruction with no uses.");
19710
19711 switch (I->getOpcode()) {
19712 case Instruction::FMul: {
19713 // Don't break FMA, PowerPC prefers FMA.
19714 if (User->getOpcode() != Instruction::FSub &&
19715 User->getOpcode() != Instruction::FAdd)
19716 return true;
19717
19718 const Function *F = I->getFunction();
19719 const DataLayout &DL = F->getDataLayout();
19720 Type *Ty = User->getOperand(i: 0)->getType();
19721 bool AllowContract = I->getFastMathFlags().allowContract() &&
19722 User->getFastMathFlags().allowContract();
19723
19724 return !(isFMAFasterThanFMulAndFAdd(F: *F, Ty) &&
19725 isOperationLegalOrCustom(Op: ISD::FMA, VT: getValueType(DL, Ty)) &&
19726 AllowContract);
19727 }
19728 case Instruction::Load: {
19729 // Don't break "store (load float*)" pattern, this pattern will be combined
19730 // to "store (load int32)" in later InstCombine pass. See function
19731 // combineLoadToOperationType. On PowerPC, loading a float point takes more
19732 // cycles than loading a 32 bit integer.
19733 LoadInst *LI = cast<LoadInst>(Val: I);
19734 // For the loads that combineLoadToOperationType does nothing, like
19735 // ordered load, it should be profitable to hoist them.
19736 // For swifterror load, it can only be used for pointer to pointer type, so
19737 // later type check should get rid of this case.
19738 if (!LI->isUnordered())
19739 return true;
19740
19741 if (User->getOpcode() != Instruction::Store)
19742 return true;
19743
19744 if (I->getType()->getTypeID() != Type::FloatTyID)
19745 return true;
19746
19747 return false;
19748 }
19749 default:
19750 return true;
19751 }
19752 return true;
19753}
19754
19755const MCPhysReg *
19756PPCTargetLowering::getScratchRegisters(CallingConv::ID) const {
19757 // LR is a callee-save register, but we must treat it as clobbered by any call
19758 // site. Hence we include LR in the scratch registers, which are in turn added
19759 // as implicit-defs for stackmaps and patchpoints. The same reasoning applies
19760 // to CTR, which is used by any indirect call.
19761 static const MCPhysReg ScratchRegs[] = {
19762 PPC::X12, PPC::LR8, PPC::CTR8, 0
19763 };
19764
19765 return ScratchRegs;
19766}
19767
19768Register PPCTargetLowering::getExceptionPointerRegister(
19769 ExceptionHandling EH, const Constant *PersonalityFn) const {
19770 return Subtarget.isPPC64() ? PPC::X3 : PPC::R3;
19771}
19772
19773Register PPCTargetLowering::getExceptionSelectorRegister(
19774 ExceptionHandling EH, const Constant *PersonalityFn) const {
19775 return Subtarget.isPPC64() ? PPC::X4 : PPC::R4;
19776}
19777
19778bool
19779PPCTargetLowering::shouldExpandBuildVectorWithShuffles(
19780 EVT VT , unsigned DefinedValues) const {
19781 if (VT == MVT::v2i64)
19782 return Subtarget.hasDirectMove(); // Don't need stack ops with direct moves
19783
19784 if (Subtarget.hasVSX())
19785 return true;
19786
19787 return TargetLowering::shouldExpandBuildVectorWithShuffles(VT, DefinedValues);
19788}
19789
19790Sched::Preference PPCTargetLowering::getSchedulingPreference(SDNode *N) const {
19791 if (!EnableILPPref || Subtarget.enableMachineScheduler())
19792 return TargetLowering::getSchedulingPreference(N);
19793
19794 return Sched::ILP;
19795}
19796
19797// Create a fast isel object.
19798FastISel *PPCTargetLowering::createFastISel(
19799 FunctionLoweringInfo &FuncInfo, const TargetLibraryInfo *LibInfo,
19800 const LibcallLoweringInfo *LibcallLowering) const {
19801 return PPC::createFastISel(FuncInfo, LibInfo, LibcallLowering);
19802}
19803
19804// 'Inverted' means the FMA opcode after negating one multiplicand.
19805// For example, (fma -a b c) = (fnmsub a b c)
19806static unsigned invertFMAOpcode(unsigned Opc) {
19807 switch (Opc) {
19808 default:
19809 llvm_unreachable("Invalid FMA opcode for PowerPC!");
19810 case ISD::FMA:
19811 return PPCISD::FNMSUB;
19812 case PPCISD::FNMSUB:
19813 return ISD::FMA;
19814 }
19815}
19816
19817SDValue PPCTargetLowering::getNegatedExpression(SDValue Op, SelectionDAG &DAG,
19818 bool LegalOps, bool OptForSize,
19819 NegatibleCost &Cost,
19820 unsigned Depth) const {
19821 if (Depth > SelectionDAG::MaxRecursionDepth)
19822 return SDValue();
19823
19824 unsigned Opc = Op.getOpcode();
19825 EVT VT = Op.getValueType();
19826 SDNodeFlags Flags = Op.getNode()->getFlags();
19827
19828 switch (Opc) {
19829 case PPCISD::FNMSUB:
19830 if (!Op.hasOneUse() || !isTypeLegal(VT))
19831 break;
19832
19833 SDValue N0 = Op.getOperand(i: 0);
19834 SDValue N1 = Op.getOperand(i: 1);
19835 SDValue N2 = Op.getOperand(i: 2);
19836 SDLoc Loc(Op);
19837
19838 NegatibleCost N2Cost = NegatibleCost::Expensive;
19839 SDValue NegN2 =
19840 getNegatedExpression(Op: N2, DAG, LegalOps, OptForSize, Cost&: N2Cost, Depth: Depth + 1);
19841
19842 if (!NegN2)
19843 return SDValue();
19844
19845 // (fneg (fnmsub a b c)) => (fnmsub (fneg a) b (fneg c))
19846 // (fneg (fnmsub a b c)) => (fnmsub a (fneg b) (fneg c))
19847 // These transformations may change sign of zeroes. For example,
19848 // -(-ab-(-c))=-0 while -(-(ab-c))=+0 when a=b=c=1.
19849 if (Flags.hasNoSignedZeros()) {
19850 // Try and choose the cheaper one to negate.
19851 NegatibleCost N0Cost = NegatibleCost::Expensive;
19852 SDValue NegN0 = getNegatedExpression(Op: N0, DAG, LegalOps, OptForSize,
19853 Cost&: N0Cost, Depth: Depth + 1);
19854
19855 NegatibleCost N1Cost = NegatibleCost::Expensive;
19856 SDValue NegN1 = getNegatedExpression(Op: N1, DAG, LegalOps, OptForSize,
19857 Cost&: N1Cost, Depth: Depth + 1);
19858
19859 if (NegN0 && N0Cost <= N1Cost) {
19860 Cost = std::min(a: N0Cost, b: N2Cost);
19861 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: NegN0, N2: N1, N3: NegN2, Flags);
19862 } else if (NegN1) {
19863 Cost = std::min(a: N1Cost, b: N2Cost);
19864 return DAG.getNode(Opcode: Opc, DL: Loc, VT, N1: N0, N2: NegN1, N3: NegN2, Flags);
19865 }
19866 }
19867
19868 // (fneg (fnmsub a b c)) => (fma a b (fneg c))
19869 if (isOperationLegal(Op: ISD::FMA, VT)) {
19870 Cost = N2Cost;
19871 return DAG.getNode(Opcode: ISD::FMA, DL: Loc, VT, N1: N0, N2: N1, N3: NegN2, Flags);
19872 }
19873
19874 break;
19875 }
19876
19877 return TargetLowering::getNegatedExpression(Op, DAG, LegalOps, OptForSize,
19878 Cost, Depth);
19879}
19880
19881// Override to enable LOAD_STACK_GUARD lowering on Linux.
19882bool PPCTargetLowering::useLoadStackGuardNode(const Module &M) const {
19883 if (M.getStackProtectorGuard() == "tls" || Subtarget.isTargetLinux())
19884 return true;
19885 return TargetLowering::useLoadStackGuardNode(M);
19886}
19887
19888TargetLowering::FPImmAction
19889PPCTargetLowering::getFPImmAction(const APFloat &Imm, EVT VT,
19890 bool ForCodeSize) const {
19891 if (!VT.isSimple() || !Subtarget.hasVSX())
19892 return FPImmAction::ConstantPool;
19893
19894 switch(VT.getSimpleVT().SimpleTy) {
19895 default:
19896 // For FP types that are currently not supported by PPC backend, return
19897 // false. Examples: f16, f80.
19898 return FPImmAction::ConstantPool;
19899 case MVT::f32:
19900 case MVT::f64: {
19901 if (Subtarget.hasPrefixInstrs() && Subtarget.hasP10Vector()) {
19902 // we can materialize all immediatess via XXSPLTI32DX and XXSPLTIDP.
19903 return FPImmAction::Legal;
19904 }
19905 bool IsExact;
19906 APSInt IntResult(16, false);
19907 // The rounding mode doesn't really matter because we only care about floats
19908 // that can be converted to integers exactly.
19909 Imm.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
19910 // For exact values in the range [-16, 15] we can materialize the float.
19911 if (IsExact && IntResult <= 15 && IntResult >= -16)
19912 return FPImmAction::Legal;
19913 return Imm.isZero() ? FPImmAction::Legal : FPImmAction::ConstantPool;
19914 }
19915 case MVT::ppcf128:
19916 return Imm.isPosZero() ? FPImmAction::Legal : FPImmAction::ConstantPool;
19917 }
19918}
19919
19920// For vector shift operation op, fold
19921// (op x, (and y, ((1 << numbits(x)) - 1))) -> (target op x, y)
19922static SDValue stripModuloOnShift(const TargetLowering &TLI, SDNode *N,
19923 SelectionDAG &DAG) {
19924 SDValue N0 = N->getOperand(Num: 0);
19925 SDValue N1 = N->getOperand(Num: 1);
19926 EVT VT = N0.getValueType();
19927 unsigned OpSizeInBits = VT.getScalarSizeInBits();
19928 unsigned Opcode = N->getOpcode();
19929 unsigned TargetOpcode;
19930
19931 switch (Opcode) {
19932 default:
19933 llvm_unreachable("Unexpected shift operation");
19934 case ISD::SHL:
19935 TargetOpcode = PPCISD::SHL;
19936 break;
19937 case ISD::SRL:
19938 TargetOpcode = PPCISD::SRL;
19939 break;
19940 case ISD::SRA:
19941 TargetOpcode = PPCISD::SRA;
19942 break;
19943 }
19944
19945 if (VT.isVector() && TLI.isOperationLegal(Op: Opcode, VT) &&
19946 N1->getOpcode() == ISD::AND)
19947 if (ConstantSDNode *Mask = isConstOrConstSplat(N: N1->getOperand(Num: 1)))
19948 if (Mask->getZExtValue() == OpSizeInBits - 1)
19949 return DAG.getNode(Opcode: TargetOpcode, DL: SDLoc(N), VT, N1: N0, N2: N1->getOperand(Num: 0));
19950
19951 return SDValue();
19952}
19953
19954SDValue PPCTargetLowering::combineVectorShift(SDNode *N,
19955 DAGCombinerInfo &DCI) const {
19956 EVT VT = N->getValueType(ResNo: 0);
19957 assert(VT.isVector() && "Vector type expected.");
19958
19959 unsigned Opc = N->getOpcode();
19960 assert((Opc == ISD::SHL || Opc == ISD::SRL || Opc == ISD::SRA) &&
19961 "Unexpected opcode.");
19962
19963 if (!isOperationLegal(Op: Opc, VT))
19964 return SDValue();
19965
19966 EVT EltTy = VT.getScalarType();
19967 unsigned EltBits = EltTy.getSizeInBits();
19968 if (EltTy != MVT::i64 && EltTy != MVT::i32)
19969 return SDValue();
19970
19971 SDValue N1 = N->getOperand(Num: 1);
19972 uint64_t SplatBits = 0;
19973 bool AddSplatCase = false;
19974 unsigned OpcN1 = N1.getOpcode();
19975 if (OpcN1 == PPCISD::VADD_SPLAT &&
19976 N1.getConstantOperandVal(i: 1) == VT.getVectorNumElements()) {
19977 AddSplatCase = true;
19978 SplatBits = N1.getConstantOperandVal(i: 0);
19979 }
19980
19981 if (!AddSplatCase) {
19982 if (OpcN1 != ISD::BUILD_VECTOR)
19983 return SDValue();
19984
19985 unsigned SplatBitSize;
19986 bool HasAnyUndefs;
19987 APInt APSplatBits, APSplatUndef;
19988 BuildVectorSDNode *BVN = cast<BuildVectorSDNode>(Val&: N1);
19989 bool BVNIsConstantSplat =
19990 BVN->isConstantSplat(SplatValue&: APSplatBits, SplatUndef&: APSplatUndef, SplatBitSize,
19991 HasAnyUndefs, MinSplatBits: 0, isBigEndian: !Subtarget.isLittleEndian());
19992 if (!BVNIsConstantSplat || SplatBitSize != EltBits)
19993 return SDValue();
19994 SplatBits = APSplatBits.getZExtValue();
19995 }
19996
19997 SDLoc DL(N);
19998 SDValue N0 = N->getOperand(Num: 0);
19999 // PPC vector shifts by word/double look at only the low 5/6 bits of the
20000 // shift vector, which means the max value is 31/63. A shift vector of all
20001 // 1s will be truncated to 31/63, which is useful as vspltiw is limited to
20002 // -16 to 15 range.
20003 if (SplatBits == (EltBits - 1)) {
20004 unsigned NewOpc;
20005 switch (Opc) {
20006 case ISD::SHL:
20007 NewOpc = PPCISD::SHL;
20008 break;
20009 case ISD::SRL:
20010 NewOpc = PPCISD::SRL;
20011 break;
20012 case ISD::SRA:
20013 NewOpc = PPCISD::SRA;
20014 break;
20015 }
20016 SDValue SplatOnes = getCanonicalConstSplat(Val: 255, SplatSize: 1, VT, DAG&: DCI.DAG, dl: DL);
20017 return DCI.DAG.getNode(Opcode: NewOpc, DL, VT, N1: N0, N2: SplatOnes);
20018 }
20019
20020 if (Opc != ISD::SHL || !isOperationLegal(Op: ISD::ADD, VT))
20021 return SDValue();
20022
20023 // For 64-bit there is no splat immediate so we want to catch shift by 1 here
20024 // before the BUILD_VECTOR is replaced by a load.
20025 if (EltTy != MVT::i64 || SplatBits != 1)
20026 return SDValue();
20027
20028 return DCI.DAG.getNode(Opcode: ISD::ADD, DL: SDLoc(N), VT, N1: N0, N2: N0);
20029}
20030
20031SDValue PPCTargetLowering::combineSHL(SDNode *N, DAGCombinerInfo &DCI) const {
20032 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20033 return Value;
20034
20035 if (N->getValueType(ResNo: 0).isVector())
20036 return combineVectorShift(N, DCI);
20037
20038 SDValue N0 = N->getOperand(Num: 0);
20039 ConstantSDNode *CN1 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
20040 if (!Subtarget.isISA3_0() || !Subtarget.isPPC64() ||
20041 N0.getOpcode() != ISD::SIGN_EXTEND ||
20042 N0.getOperand(i: 0).getValueType() != MVT::i32 || CN1 == nullptr ||
20043 N->getValueType(ResNo: 0) != MVT::i64)
20044 return SDValue();
20045
20046 // We can't save an operation here if the value is already extended, and
20047 // the existing shift is easier to combine.
20048 SDValue ExtsSrc = N0.getOperand(i: 0);
20049 if (ExtsSrc.getOpcode() == ISD::TRUNCATE &&
20050 ExtsSrc.getOperand(i: 0).getOpcode() == ISD::AssertSext)
20051 return SDValue();
20052
20053 SDLoc DL(N0);
20054 SDValue ShiftBy = SDValue(CN1, 0);
20055 // We want the shift amount to be i32 on the extswli, but the shift could
20056 // have an i64.
20057 if (ShiftBy.getValueType() == MVT::i64)
20058 ShiftBy = DCI.DAG.getConstant(Val: CN1->getZExtValue(), DL, VT: MVT::i32);
20059
20060 return DCI.DAG.getNode(Opcode: PPCISD::EXTSWSLI, DL, VT: MVT::i64, N1: N0->getOperand(Num: 0),
20061 N2: ShiftBy);
20062}
20063
20064SDValue PPCTargetLowering::combineSRA(SDNode *N, DAGCombinerInfo &DCI) const {
20065 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20066 return Value;
20067
20068 if (N->getValueType(ResNo: 0).isVector())
20069 return combineVectorShift(N, DCI);
20070
20071 return SDValue();
20072}
20073
20074SDValue PPCTargetLowering::combineSRL(SDNode *N, DAGCombinerInfo &DCI) const {
20075 if (auto Value = stripModuloOnShift(TLI: *this, N, DAG&: DCI.DAG))
20076 return Value;
20077
20078 if (N->getValueType(ResNo: 0).isVector())
20079 return combineVectorShift(N, DCI);
20080
20081 return SDValue();
20082}
20083
20084// Transform (add X, (zext(setne Z, C))) -> (addze X, (addic (addi Z, -C), -1))
20085// Transform (add X, (zext(sete Z, C))) -> (addze X, (subfic (addi Z, -C), 0))
20086// When C is zero, the equation (addi Z, -C) can be simplified to Z
20087// Requirement: -C in [-32768, 32767], X and Z are MVT::i64 types
20088static SDValue combineADDToADDZE(SDNode *N, SelectionDAG &DAG,
20089 const PPCSubtarget &Subtarget) {
20090 if (!Subtarget.isPPC64())
20091 return SDValue();
20092
20093 SDValue LHS = N->getOperand(Num: 0);
20094 SDValue RHS = N->getOperand(Num: 1);
20095
20096 auto isZextOfCompareWithConstant = [](SDValue Op) {
20097 if (Op.getOpcode() != ISD::ZERO_EXTEND || !Op.hasOneUse() ||
20098 Op.getValueType() != MVT::i64)
20099 return false;
20100
20101 SDValue Cmp = Op.getOperand(i: 0);
20102 if (Cmp.getOpcode() != ISD::SETCC || !Cmp.hasOneUse() ||
20103 Cmp.getOperand(i: 0).getValueType() != MVT::i64)
20104 return false;
20105
20106 if (auto *Constant = dyn_cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1))) {
20107 int64_t NegConstant = 0 - Constant->getSExtValue();
20108 // Due to the limitations of the addi instruction,
20109 // -C is required to be [-32768, 32767].
20110 return isInt<16>(x: NegConstant);
20111 }
20112
20113 return false;
20114 };
20115
20116 bool LHSHasPattern = isZextOfCompareWithConstant(LHS);
20117 bool RHSHasPattern = isZextOfCompareWithConstant(RHS);
20118
20119 // If there is a pattern, canonicalize a zext operand to the RHS.
20120 if (LHSHasPattern && !RHSHasPattern)
20121 std::swap(a&: LHS, b&: RHS);
20122 else if (!LHSHasPattern && !RHSHasPattern)
20123 return SDValue();
20124
20125 SDLoc DL(N);
20126 EVT CarryType = Subtarget.useCRBits() ? MVT::i1 : MVT::i32;
20127 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: CarryType);
20128 SDValue Cmp = RHS.getOperand(i: 0);
20129 SDValue Z = Cmp.getOperand(i: 0);
20130 auto *Constant = cast<ConstantSDNode>(Val: Cmp.getOperand(i: 1));
20131 int64_t NegConstant = 0 - Constant->getSExtValue();
20132
20133 switch (Cmp.getOperandCondCode(i: 2)) {
20134 default: break;
20135 case ISD::SETNE: {
20136 // when C == 0
20137 // --> addze X, (addic Z, -1).carry
20138 // /
20139 // add X, (zext(setne Z, C))--
20140 // \ when -32768 <= -C <= 32767 && C != 0
20141 // --> addze X, (addic (addi Z, -C), -1).carry
20142 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20143 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20144 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20145 SDValue Addc =
20146 DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20147 N1: AddOrZ, N2: DAG.getAllOnesConstant(DL, VT: MVT::i64),
20148 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20149 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20150 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64),
20151 N3: SDValue(Addc.getNode(), 1));
20152 }
20153 case ISD::SETEQ: {
20154 // when C == 0
20155 // --> addze X, (subfic Z, 0).carry
20156 // /
20157 // add X, (zext(sete Z, C))--
20158 // \ when -32768 <= -C <= 32767 && C != 0
20159 // --> addze X, (subfic (addi Z, -C), 0).carry
20160 SDValue Add = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Z,
20161 N2: DAG.getConstant(Val: NegConstant, DL, VT: MVT::i64));
20162 SDValue AddOrZ = NegConstant != 0 ? Add : Z;
20163 SDValue Subc =
20164 DAG.getNode(Opcode: ISD::USUBO_CARRY, DL, VTList: DAG.getVTList(VT1: MVT::i64, VT2: CarryType),
20165 N1: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N2: AddOrZ,
20166 N3: DAG.getConstant(Val: 0, DL, VT: CarryType));
20167 SDValue Invert = DAG.getNode(Opcode: ISD::XOR, DL, VT: CarryType, N1: Subc.getValue(R: 1),
20168 N2: DAG.getConstant(Val: 1UL, DL, VT: CarryType));
20169 return DAG.getNode(Opcode: ISD::UADDO_CARRY, DL, VTList: VTs, N1: LHS,
20170 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i64), N3: Invert);
20171 }
20172 }
20173
20174 return SDValue();
20175}
20176
20177// Transform
20178// (add C1, (MAT_PCREL_ADDR GlobalAddr+C2)) to
20179// (MAT_PCREL_ADDR GlobalAddr+(C1+C2))
20180// In this case both C1 and C2 must be known constants.
20181// C1+C2 must fit into a 34 bit signed integer.
20182static SDValue combineADDToMAT_PCREL_ADDR(SDNode *N, SelectionDAG &DAG,
20183 const PPCSubtarget &Subtarget) {
20184 if (!Subtarget.isUsingPCRelativeCalls())
20185 return SDValue();
20186
20187 // Check both Operand 0 and Operand 1 of the ADD node for the PCRel node.
20188 // If we find that node try to cast the Global Address and the Constant.
20189 SDValue LHS = N->getOperand(Num: 0);
20190 SDValue RHS = N->getOperand(Num: 1);
20191
20192 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20193 std::swap(a&: LHS, b&: RHS);
20194
20195 if (LHS.getOpcode() != PPCISD::MAT_PCREL_ADDR)
20196 return SDValue();
20197
20198 // Operand zero of PPCISD::MAT_PCREL_ADDR is the GA node.
20199 GlobalAddressSDNode *GSDN = dyn_cast<GlobalAddressSDNode>(Val: LHS.getOperand(i: 0));
20200 ConstantSDNode* ConstNode = dyn_cast<ConstantSDNode>(Val&: RHS);
20201
20202 // Check that both casts succeeded.
20203 if (!GSDN || !ConstNode)
20204 return SDValue();
20205
20206 int64_t NewOffset = GSDN->getOffset() + ConstNode->getSExtValue();
20207 SDLoc DL(GSDN);
20208
20209 // The signed int offset needs to fit in 34 bits.
20210 if (!isInt<34>(x: NewOffset))
20211 return SDValue();
20212
20213 // The new global address is a copy of the old global address except
20214 // that it has the updated Offset.
20215 SDValue GA =
20216 DAG.getTargetGlobalAddress(GV: GSDN->getGlobal(), DL, VT: GSDN->getValueType(ResNo: 0),
20217 offset: NewOffset, TargetFlags: GSDN->getTargetFlags());
20218 SDValue MatPCRel =
20219 DAG.getNode(Opcode: PPCISD::MAT_PCREL_ADDR, DL, VT: GSDN->getValueType(ResNo: 0), Operand: GA);
20220 return MatPCRel;
20221}
20222
20223// Transform (add X, (build_vector (T 1), (T 1), ...)) -> (sub X, (XXLEQVOnes))
20224// XXLEQVOnes creates an all-1s vector (0xFFFFFFFF...) efficiently via xxleqv
20225// Mathematical identity: X + 1 = X - (-1)
20226// Applies to v4i32, v2i64, v8i16, v16i8 where all elements are constant 1
20227// Requirement: VSX feature for efficient xxleqv generation
20228static SDValue combineADDToSUB(SDNode *N, SelectionDAG &DAG,
20229 const PPCSubtarget &Subtarget) {
20230
20231 EVT VT = N->getValueType(ResNo: 0);
20232 if (!Subtarget.hasVSX())
20233 return SDValue();
20234
20235 // Handle v2i64, v4i32, v8i16 and v16i8 types
20236 if (!(VT == MVT::v8i16 || VT == MVT::v16i8 || VT == MVT::v4i32 ||
20237 VT == MVT::v2i64))
20238 return SDValue();
20239
20240 SDValue LHS = N->getOperand(Num: 0);
20241 SDValue RHS = N->getOperand(Num: 1);
20242
20243 // Check if RHS is BUILD_VECTOR
20244 if (RHS.getOpcode() != ISD::BUILD_VECTOR)
20245 return SDValue();
20246
20247 // Check if all the elements are 1
20248 unsigned NumOfEles = RHS.getNumOperands();
20249 for (unsigned i = 0; i < NumOfEles; ++i) {
20250 auto *CN = dyn_cast<ConstantSDNode>(Val: RHS.getOperand(i));
20251 if (!CN || CN->getSExtValue() != 1)
20252 return SDValue();
20253 }
20254 SDLoc DL(N);
20255
20256 SDValue MinusOne = DAG.getConstant(Val: APInt::getAllOnes(numBits: 32), DL, VT: MVT::i32);
20257 SmallVector<SDValue, 4> Ops(4, MinusOne);
20258 SDValue AllOnesVec = DAG.getBuildVector(VT: MVT::v4i32, DL, Ops);
20259
20260 // Bitcast to the target vector type
20261 SDValue Bitcast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: AllOnesVec);
20262
20263 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: LHS, N2: Bitcast);
20264}
20265
20266SDValue PPCTargetLowering::combineADD(SDNode *N, DAGCombinerInfo &DCI) const {
20267 if (auto Value = combineADDToADDZE(N, DAG&: DCI.DAG, Subtarget))
20268 return Value;
20269
20270 if (auto Value = combineADDToMAT_PCREL_ADDR(N, DAG&: DCI.DAG, Subtarget))
20271 return Value;
20272
20273 if (auto Value = combineADDToSUB(N, DAG&: DCI.DAG, Subtarget))
20274 return Value;
20275 return SDValue();
20276}
20277
20278// Detect TRUNCATE operations on bitcasts of float128 values.
20279// What we are looking for here is the situtation where we extract a subset
20280// of bits from a 128 bit float.
20281// This can be of two forms:
20282// 1) BITCAST of f128 feeding TRUNCATE
20283// 2) BITCAST of f128 feeding SRL (a shift) feeding TRUNCATE
20284// The reason this is required is because we do not have a legal i128 type
20285// and so we want to prevent having to store the f128 and then reload part
20286// of it.
20287SDValue PPCTargetLowering::combineTRUNCATE(SDNode *N,
20288 DAGCombinerInfo &DCI) const {
20289 // If we are using CRBits then try that first.
20290 if (Subtarget.useCRBits()) {
20291 // Check if CRBits did anything and return that if it did.
20292 if (SDValue CRTruncValue = DAGCombineTruncBoolExt(N, DCI))
20293 return CRTruncValue;
20294 }
20295
20296 SDLoc dl(N);
20297 SDValue Op0 = N->getOperand(Num: 0);
20298
20299 // Looking for a truncate of i128 to i64.
20300 if (Op0.getValueType() != MVT::i128 || N->getValueType(ResNo: 0) != MVT::i64)
20301 return SDValue();
20302
20303 int EltToExtract = DCI.DAG.getDataLayout().isBigEndian() ? 1 : 0;
20304
20305 // SRL feeding TRUNCATE.
20306 if (Op0.getOpcode() == ISD::SRL) {
20307 ConstantSDNode *ConstNode = dyn_cast<ConstantSDNode>(Val: Op0.getOperand(i: 1));
20308 // The right shift has to be by 64 bits.
20309 if (!ConstNode || ConstNode->getZExtValue() != 64)
20310 return SDValue();
20311
20312 // Switch the element number to extract.
20313 EltToExtract = EltToExtract ? 0 : 1;
20314 // Update Op0 past the SRL.
20315 Op0 = Op0.getOperand(i: 0);
20316 }
20317
20318 // BITCAST feeding a TRUNCATE possibly via SRL.
20319 if (Op0.getOpcode() == ISD::BITCAST &&
20320 Op0.getValueType() == MVT::i128 &&
20321 Op0.getOperand(i: 0).getValueType() == MVT::f128) {
20322 SDValue Bitcast = DCI.DAG.getBitcast(VT: MVT::v2i64, V: Op0.getOperand(i: 0));
20323 return DCI.DAG.getNode(
20324 Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i64, N1: Bitcast,
20325 N2: DCI.DAG.getTargetConstant(Val: EltToExtract, DL: dl, VT: MVT::i32));
20326 }
20327 return SDValue();
20328}
20329
20330SDValue PPCTargetLowering::combineMUL(SDNode *N, DAGCombinerInfo &DCI) const {
20331 SelectionDAG &DAG = DCI.DAG;
20332
20333 ConstantSDNode *ConstOpOrElement = isConstOrConstSplat(N: N->getOperand(Num: 1));
20334 if (!ConstOpOrElement)
20335 return SDValue();
20336
20337 // An imul is usually smaller than the alternative sequence for legal type.
20338 if (DAG.getMachineFunction().getFunction().hasMinSize() &&
20339 isOperationLegal(Op: ISD::MUL, VT: N->getValueType(ResNo: 0)))
20340 return SDValue();
20341
20342 auto IsProfitable = [this](bool IsNeg, bool IsAddOne, EVT VT) -> bool {
20343 switch (this->Subtarget.getCPUDirective()) {
20344 default:
20345 // TODO: enhance the condition for subtarget before pwr8
20346 return false;
20347 case PPC::DIR_PWR8:
20348 // type mul add shl
20349 // scalar 4 1 1
20350 // vector 7 2 2
20351 return true;
20352 case PPC::DIR_PWR9:
20353 case PPC::DIR_PWR10:
20354 case PPC::DIR_PWR11:
20355 case PPC::DIR_PWR_FUTURE:
20356 // type mul add shl
20357 // scalar 5 2 2
20358 // vector 7 2 2
20359
20360 // The cycle RATIO of related operations are showed as a table above.
20361 // Because mul is 5(scalar)/7(vector), add/sub/shl are all 2 for both
20362 // scalar and vector type. For 2 instrs patterns, add/sub + shl
20363 // are 4, it is always profitable; but for 3 instrs patterns
20364 // (mul x, -(2^N + 1)) => -(add (shl x, N), x), sub + add + shl are 6.
20365 // So we should only do it for vector type.
20366 return IsAddOne && IsNeg ? VT.isVector() : true;
20367 }
20368 };
20369
20370 EVT VT = N->getValueType(ResNo: 0);
20371 SDLoc DL(N);
20372
20373 const APInt &MulAmt = ConstOpOrElement->getAPIntValue();
20374 bool IsNeg = MulAmt.isNegative();
20375 APInt MulAmtAbs = MulAmt.abs();
20376
20377 if ((MulAmtAbs - 1).isPowerOf2()) {
20378 // (mul x, 2^N + 1) => (add (shl x, N), x)
20379 // (mul x, -(2^N + 1)) => -(add (shl x, N), x)
20380
20381 if (!IsProfitable(IsNeg, true, VT))
20382 return SDValue();
20383
20384 SDValue Op0 = N->getOperand(Num: 0);
20385 SDValue Op1 =
20386 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20387 N2: DAG.getConstant(Val: (MulAmtAbs - 1).logBase2(), DL, VT));
20388 SDValue Res = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Op0, N2: Op1);
20389
20390 if (!IsNeg)
20391 return Res;
20392
20393 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT), N2: Res);
20394 } else if ((MulAmtAbs + 1).isPowerOf2()) {
20395 // (mul x, 2^N - 1) => (sub (shl x, N), x)
20396 // (mul x, -(2^N - 1)) => (sub x, (shl x, N))
20397
20398 if (!IsProfitable(IsNeg, false, VT))
20399 return SDValue();
20400
20401 SDValue Op0 = N->getOperand(Num: 0);
20402 SDValue Op1 =
20403 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N->getOperand(Num: 0),
20404 N2: DAG.getConstant(Val: (MulAmtAbs + 1).logBase2(), DL, VT));
20405
20406 if (!IsNeg)
20407 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op1, N2: Op0);
20408 else
20409 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Op0, N2: Op1);
20410
20411 } else {
20412 return SDValue();
20413 }
20414}
20415
20416// Combine fma-like op (like fnmsub) with fnegs to appropriate op. Do this
20417// in combiner since we need to check SD flags and other subtarget features.
20418SDValue PPCTargetLowering::combineFMALike(SDNode *N,
20419 DAGCombinerInfo &DCI) const {
20420 SDValue N0 = N->getOperand(Num: 0);
20421 SDValue N1 = N->getOperand(Num: 1);
20422 SDValue N2 = N->getOperand(Num: 2);
20423 SDNodeFlags Flags = N->getFlags();
20424 EVT VT = N->getValueType(ResNo: 0);
20425 SelectionDAG &DAG = DCI.DAG;
20426 unsigned Opc = N->getOpcode();
20427 bool CodeSize = DAG.getMachineFunction().getFunction().hasOptSize();
20428 bool LegalOps = !DCI.isBeforeLegalizeOps();
20429 SDLoc Loc(N);
20430
20431 if (!isOperationLegal(Op: ISD::FMA, VT))
20432 return SDValue();
20433
20434 // Allowing transformation to FNMSUB may change sign of zeroes when ab-c=0
20435 // since (fnmsub a b c)=-0 while c-ab=+0.
20436 if (!Flags.hasNoSignedZeros())
20437 return SDValue();
20438
20439 // (fma (fneg a) b c) => (fnmsub a b c)
20440 // (fnmsub (fneg a) b c) => (fma a b c)
20441 if (SDValue NegN0 = getCheaperNegatedExpression(Op: N0, DAG, LegalOps, OptForSize: CodeSize))
20442 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: NegN0, N2: N1, N3: N2, Flags);
20443
20444 // (fma a (fneg b) c) => (fnmsub a b c)
20445 // (fnmsub a (fneg b) c) => (fma a b c)
20446 if (SDValue NegN1 = getCheaperNegatedExpression(Op: N1, DAG, LegalOps, OptForSize: CodeSize))
20447 return DAG.getNode(Opcode: invertFMAOpcode(Opc), DL: Loc, VT, N1: N0, N2: NegN1, N3: N2, Flags);
20448
20449 return SDValue();
20450}
20451
20452bool PPCTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
20453 // Only duplicate to increase tail-calls for the 64bit SysV ABIs.
20454 if (!Subtarget.is64BitELFABI())
20455 return false;
20456
20457 // If not a tail call then no need to proceed.
20458 if (!CI->isTailCall())
20459 return false;
20460
20461 // If sibling calls have been disabled and tail-calls aren't guaranteed
20462 // there is no reason to duplicate.
20463 auto &TM = getTargetMachine();
20464 if (!TM.Options.GuaranteedTailCallOpt && !EnableSCO)
20465 return false;
20466
20467 // Can't tail call a function called indirectly, or if it has variadic args.
20468 const Function *Callee = CI->getCalledFunction();
20469 if (!Callee || Callee->isVarArg())
20470 return false;
20471
20472 // Make sure the callee and caller calling conventions are eligible for tco.
20473 const Function *Caller = CI->getParent()->getParent();
20474 if (!areCallingConvEligibleForTCO_64SVR4(CallerCC: Caller->getCallingConv(),
20475 CalleeCC: CI->getCallingConv()))
20476 return false;
20477
20478 // If the function is local then we have a good chance at tail-calling it
20479 return getTargetMachine().shouldAssumeDSOLocal(GV: Callee);
20480}
20481
20482bool PPCTargetLowering::
20483isMaskAndCmp0FoldingBeneficial(const Instruction &AndI) const {
20484 const Value *Mask = AndI.getOperand(i: 1);
20485 // If the mask is suitable for andi. or andis. we should sink the and.
20486 if (const ConstantInt *CI = dyn_cast<ConstantInt>(Val: Mask)) {
20487 // Can't handle constants wider than 64-bits.
20488 if (CI->getBitWidth() > 64)
20489 return false;
20490 int64_t ConstVal = CI->getZExtValue();
20491 return isUInt<16>(x: ConstVal) ||
20492 (isUInt<16>(x: ConstVal >> 16) && !(ConstVal & 0xFFFF));
20493 }
20494
20495 // For non-constant masks, we can always use the record-form and.
20496 return true;
20497}
20498
20499/// getAddrModeForFlags - Based on the set of address flags, select the most
20500/// optimal instruction format to match by.
20501PPC::AddrMode PPCTargetLowering::getAddrModeForFlags(unsigned Flags) const {
20502 // This is not a node we should be handling here.
20503 if (Flags == PPC::MOF_None)
20504 return PPC::AM_None;
20505 // Unaligned D-Forms are tried first, followed by the aligned D-Forms.
20506 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DForm))
20507 if ((Flags & FlagSet) == FlagSet)
20508 return PPC::AM_DForm;
20509 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DSForm))
20510 if ((Flags & FlagSet) == FlagSet)
20511 return PPC::AM_DSForm;
20512 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_DQForm))
20513 if ((Flags & FlagSet) == FlagSet)
20514 return PPC::AM_DQForm;
20515 for (auto FlagSet : AddrModesMap.at(k: PPC::AM_PrefixDForm))
20516 if ((Flags & FlagSet) == FlagSet)
20517 return PPC::AM_PrefixDForm;
20518 // If no other forms are selected, return an X-Form as it is the most
20519 // general addressing mode.
20520 return PPC::AM_XForm;
20521}
20522
20523/// Set alignment flags based on whether or not the Frame Index is aligned.
20524/// Utilized when computing flags for address computation when selecting
20525/// load and store instructions.
20526static void setAlignFlagsForFI(SDValue N, unsigned &FlagSet,
20527 SelectionDAG &DAG) {
20528 bool IsAdd = ((N.getOpcode() == ISD::ADD) || (N.getOpcode() == ISD::OR));
20529 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: IsAdd ? N.getOperand(i: 0) : N);
20530 if (!FI)
20531 return;
20532 const MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo();
20533 unsigned FrameIndexAlign = MFI.getObjectAlign(ObjectIdx: FI->getIndex()).value();
20534 // If this is (add $FI, $S16Imm), the alignment flags are already set
20535 // based on the immediate. We just need to clear the alignment flags
20536 // if the FI alignment is weaker.
20537 if ((FrameIndexAlign % 4) != 0)
20538 FlagSet &= ~PPC::MOF_RPlusSImm16Mult4;
20539 if ((FrameIndexAlign % 16) != 0)
20540 FlagSet &= ~PPC::MOF_RPlusSImm16Mult16;
20541 // If the address is a plain FrameIndex, set alignment flags based on
20542 // FI alignment.
20543 if (!IsAdd) {
20544 if ((FrameIndexAlign % 4) == 0)
20545 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20546 if ((FrameIndexAlign % 16) == 0)
20547 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20548 }
20549}
20550
20551/// Given a node, compute flags that are used for address computation when
20552/// selecting load and store instructions. The flags computed are stored in
20553/// FlagSet. This function takes into account whether the node is a constant,
20554/// an ADD, OR, or a constant, and computes the address flags accordingly.
20555static void computeFlagsForAddressComputation(SDValue N, unsigned &FlagSet,
20556 SelectionDAG &DAG) {
20557 // Set the alignment flags for the node depending on if the node is
20558 // 4-byte or 16-byte aligned.
20559 auto SetAlignFlagsForImm = [&](uint64_t Imm) {
20560 if ((Imm & 0x3) == 0)
20561 FlagSet |= PPC::MOF_RPlusSImm16Mult4;
20562 if ((Imm & 0xf) == 0)
20563 FlagSet |= PPC::MOF_RPlusSImm16Mult16;
20564 };
20565
20566 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: N)) {
20567 // All 32-bit constants can be computed as LIS + Disp.
20568 const APInt &ConstImm = CN->getAPIntValue();
20569 if (ConstImm.isSignedIntN(N: 32)) { // Flag to handle 32-bit constants.
20570 FlagSet |= PPC::MOF_AddrIsSImm32;
20571 SetAlignFlagsForImm(ConstImm.getZExtValue());
20572 setAlignFlagsForFI(N, FlagSet, DAG);
20573 }
20574 if (ConstImm.isSignedIntN(N: 34)) // Flag to handle 34-bit constants.
20575 FlagSet |= PPC::MOF_RPlusSImm34;
20576 else // Let constant materialization handle large constants.
20577 FlagSet |= PPC::MOF_NotAddNorCst;
20578 } else if (N.getOpcode() == ISD::ADD || provablyDisjointOr(DAG, N)) {
20579 // This address can be represented as an addition of:
20580 // - Register + Imm16 (possibly a multiple of 4/16)
20581 // - Register + Imm34
20582 // - Register + PPCISD::Lo
20583 // - Register + Register
20584 // In any case, we won't have to match this as Base + Zero.
20585 SDValue RHS = N.getOperand(i: 1);
20586 if (ConstantSDNode *CN = dyn_cast<ConstantSDNode>(Val&: RHS)) {
20587 const APInt &ConstImm = CN->getAPIntValue();
20588 if (ConstImm.isSignedIntN(N: 16)) {
20589 FlagSet |= PPC::MOF_RPlusSImm16; // Signed 16-bit immediates.
20590 SetAlignFlagsForImm(ConstImm.getZExtValue());
20591 setAlignFlagsForFI(N, FlagSet, DAG);
20592 }
20593 if (ConstImm.isSignedIntN(N: 34))
20594 FlagSet |= PPC::MOF_RPlusSImm34; // Signed 34-bit immediates.
20595 else
20596 FlagSet |= PPC::MOF_RPlusR; // Register.
20597 } else if (RHS.getOpcode() == PPCISD::Lo && !RHS.getConstantOperandVal(i: 1))
20598 FlagSet |= PPC::MOF_RPlusLo; // PPCISD::Lo.
20599 else
20600 FlagSet |= PPC::MOF_RPlusR;
20601 } else { // The address computation is not a constant or an addition.
20602 setAlignFlagsForFI(N, FlagSet, DAG);
20603 FlagSet |= PPC::MOF_NotAddNorCst;
20604 }
20605}
20606
20607static bool isPCRelNode(SDValue N) {
20608 return (N.getOpcode() == PPCISD::MAT_PCREL_ADDR ||
20609 isValidPCRelNode<ConstantPoolSDNode>(N) ||
20610 isValidPCRelNode<GlobalAddressSDNode>(N) ||
20611 isValidPCRelNode<JumpTableSDNode>(N) ||
20612 isValidPCRelNode<BlockAddressSDNode>(N));
20613}
20614
20615/// computeMOFlags - Given a node N and it's Parent (a MemSDNode), compute
20616/// the address flags of the load/store instruction that is to be matched.
20617unsigned PPCTargetLowering::computeMOFlags(const SDNode *Parent, SDValue N,
20618 SelectionDAG &DAG) const {
20619 unsigned FlagSet = PPC::MOF_None;
20620
20621 // Compute subtarget flags.
20622 if (!Subtarget.hasP9Vector())
20623 FlagSet |= PPC::MOF_SubtargetBeforeP9;
20624 else
20625 FlagSet |= PPC::MOF_SubtargetP9;
20626
20627 if (Subtarget.hasPrefixInstrs())
20628 FlagSet |= PPC::MOF_SubtargetP10;
20629
20630 if (Subtarget.hasSPE())
20631 FlagSet |= PPC::MOF_SubtargetSPE;
20632
20633 // Check if we have a PCRel node and return early.
20634 if ((FlagSet & PPC::MOF_SubtargetP10) && isPCRelNode(N))
20635 return FlagSet;
20636
20637 // If the node is the paired load/store intrinsics, compute flags for
20638 // address computation and return early.
20639 unsigned ParentOp = Parent->getOpcode();
20640 if (Subtarget.isISA3_1() && ((ParentOp == ISD::INTRINSIC_W_CHAIN) ||
20641 (ParentOp == ISD::INTRINSIC_VOID))) {
20642 unsigned ID = Parent->getConstantOperandVal(Num: 1);
20643 if ((ID == Intrinsic::ppc_vsx_lxvp) || (ID == Intrinsic::ppc_vsx_stxvp)) {
20644 SDValue IntrinOp = (ID == Intrinsic::ppc_vsx_lxvp)
20645 ? Parent->getOperand(Num: 2)
20646 : Parent->getOperand(Num: 3);
20647 computeFlagsForAddressComputation(N: IntrinOp, FlagSet, DAG);
20648 FlagSet |= PPC::MOF_Vector;
20649 return FlagSet;
20650 }
20651 }
20652
20653 // Mark this as something we don't want to handle here if it is atomic
20654 // or pre-increment instruction.
20655 if (const LSBaseSDNode *LSB = dyn_cast<LSBaseSDNode>(Val: Parent))
20656 if (LSB->isIndexed())
20657 return PPC::MOF_None;
20658
20659 // Compute in-memory type flags. This is based on if there are scalars,
20660 // floats or vectors.
20661 const MemSDNode *MN = dyn_cast<MemSDNode>(Val: Parent);
20662 assert(MN && "Parent should be a MemSDNode!");
20663 EVT MemVT = MN->getMemoryVT();
20664 unsigned Size = MemVT.getSizeInBits();
20665 if (MemVT.isScalarInteger()) {
20666 assert(Size <= 128 &&
20667 "Not expecting scalar integers larger than 16 bytes!");
20668 if (Size < 32)
20669 FlagSet |= PPC::MOF_SubWordInt;
20670 else if (Size == 32)
20671 FlagSet |= PPC::MOF_WordInt;
20672 else
20673 FlagSet |= PPC::MOF_DoubleWordInt;
20674 } else if (MemVT.isVector() && !MemVT.isFloatingPoint()) { // Integer vectors.
20675 if (Size == 128)
20676 FlagSet |= PPC::MOF_Vector;
20677 else if (Size == 256) {
20678 assert(Subtarget.pairedVectorMemops() &&
20679 "256-bit vectors are only available when paired vector memops is "
20680 "enabled!");
20681 FlagSet |= PPC::MOF_Vector;
20682 } else
20683 llvm_unreachable("Not expecting illegal vectors!");
20684 } else { // Floating point type: can be scalar, f128 or vector types.
20685 if (Size == 32 || Size == 64)
20686 FlagSet |= PPC::MOF_ScalarFloat;
20687 else if (MemVT == MVT::f128 || MemVT.isVector())
20688 FlagSet |= PPC::MOF_Vector;
20689 else
20690 llvm_unreachable("Not expecting illegal scalar floats!");
20691 }
20692
20693 // Compute flags for address computation.
20694 computeFlagsForAddressComputation(N, FlagSet, DAG);
20695
20696 // Compute type extension flags.
20697 if (const LoadSDNode *LN = dyn_cast<LoadSDNode>(Val: Parent)) {
20698 switch (LN->getExtensionType()) {
20699 case ISD::SEXTLOAD:
20700 FlagSet |= PPC::MOF_SExt;
20701 break;
20702 case ISD::EXTLOAD:
20703 case ISD::ZEXTLOAD:
20704 FlagSet |= PPC::MOF_ZExt;
20705 break;
20706 case ISD::NON_EXTLOAD:
20707 FlagSet |= PPC::MOF_NoExt;
20708 break;
20709 }
20710 } else
20711 FlagSet |= PPC::MOF_NoExt;
20712
20713 // For integers, no extension is the same as zero extension.
20714 // We set the extension mode to zero extension so we don't have
20715 // to add separate entries in AddrModesMap for loads and stores.
20716 if (MemVT.isScalarInteger() && (FlagSet & PPC::MOF_NoExt)) {
20717 FlagSet |= PPC::MOF_ZExt;
20718 FlagSet &= ~PPC::MOF_NoExt;
20719 }
20720
20721 // If we don't have prefixed instructions, 34-bit constants should be
20722 // treated as PPC::MOF_NotAddNorCst so they can match D-Forms.
20723 bool IsNonP1034BitConst =
20724 ((PPC::MOF_RPlusSImm34 | PPC::MOF_AddrIsSImm32 | PPC::MOF_SubtargetP10) &
20725 FlagSet) == PPC::MOF_RPlusSImm34;
20726 if (N.getOpcode() != ISD::ADD && N.getOpcode() != ISD::OR &&
20727 IsNonP1034BitConst)
20728 FlagSet |= PPC::MOF_NotAddNorCst;
20729
20730 return FlagSet;
20731}
20732
20733/// SelectForceXFormMode - Given the specified address, force it to be
20734/// represented as an indexed [r+r] operation (an XForm instruction).
20735PPC::AddrMode PPCTargetLowering::SelectForceXFormMode(SDValue N, SDValue &Disp,
20736 SDValue &Base,
20737 SelectionDAG &DAG) const {
20738
20739 PPC::AddrMode Mode = PPC::AM_XForm;
20740 int16_t ForceXFormImm = 0;
20741 if (provablyDisjointOr(DAG, N) &&
20742 !isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm)) {
20743 Disp = N.getOperand(i: 0);
20744 Base = N.getOperand(i: 1);
20745 return Mode;
20746 }
20747
20748 // If the address is the result of an add, we will utilize the fact that the
20749 // address calculation includes an implicit add. However, we can reduce
20750 // register pressure if we do not materialize a constant just for use as the
20751 // index register. We only get rid of the add if it is not an add of a
20752 // value and a 16-bit signed constant and both have a single use.
20753 if (N.getOpcode() == ISD::ADD &&
20754 (!isIntS16Immediate(Op: N.getOperand(i: 1), Imm&: ForceXFormImm) ||
20755 !N.getOperand(i: 1).hasOneUse() || !N.getOperand(i: 0).hasOneUse())) {
20756 Disp = N.getOperand(i: 0);
20757 Base = N.getOperand(i: 1);
20758 return Mode;
20759 }
20760
20761 // Otherwise, use R0 as the base register.
20762 Disp = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
20763 VT: N.getValueType());
20764 Base = N;
20765
20766 return Mode;
20767}
20768
20769bool PPCTargetLowering::splitValueIntoRegisterParts(
20770 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
20771 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
20772 EVT ValVT = Val.getValueType();
20773 // If we are splitting a scalar integer into f64 parts (i.e. so they
20774 // can be placed into VFRC registers), we need to zero extend and
20775 // bitcast the values. This will ensure the value is placed into a
20776 // VSR using direct moves or stack operations as needed.
20777 if (PartVT == MVT::f64 &&
20778 (ValVT == MVT::i32 || ValVT == MVT::i16 || ValVT == MVT::i8)) {
20779 Val = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: Val);
20780 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::f64, Operand: Val);
20781 Parts[0] = Val;
20782 return true;
20783 }
20784 return false;
20785}
20786
20787SDValue PPCTargetLowering::lowerToLibCall(const char *LibCallName, SDValue Op,
20788 SelectionDAG &DAG) const {
20789 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20790 TargetLowering::CallLoweringInfo CLI(DAG);
20791 EVT RetVT = Op.getValueType();
20792 Type *RetTy = RetVT.getTypeForEVT(Context&: *DAG.getContext());
20793 SDValue Callee =
20794 DAG.getExternalSymbol(Sym: LibCallName, VT: TLI.getPointerTy(DL: DAG.getDataLayout()));
20795 bool SignExtend = TLI.shouldSignExtendTypeInLibCall(Ty: RetTy, IsSigned: false);
20796 TargetLowering::ArgListTy Args;
20797 for (const SDValue &N : Op->op_values()) {
20798 EVT ArgVT = N.getValueType();
20799 Type *ArgTy = ArgVT.getTypeForEVT(Context&: *DAG.getContext());
20800 TargetLowering::ArgListEntry Entry(N, ArgTy);
20801 Entry.IsSExt = TLI.shouldSignExtendTypeInLibCall(Ty: ArgTy, IsSigned: SignExtend);
20802 Entry.IsZExt = !Entry.IsSExt;
20803 Args.push_back(x: Entry);
20804 }
20805
20806 SDValue InChain = DAG.getEntryNode();
20807 SDValue TCChain = InChain;
20808 const Function &F = DAG.getMachineFunction().getFunction();
20809 bool isTailCall =
20810 TLI.isInTailCallPosition(DAG, Node: Op.getNode(), Chain&: TCChain) &&
20811 (RetTy == F.getReturnType() || F.getReturnType()->isVoidTy());
20812 if (isTailCall)
20813 InChain = TCChain;
20814 CLI.setDebugLoc(SDLoc(Op))
20815 .setChain(InChain)
20816 .setLibCallee(CC: CallingConv::C, ResultType: RetTy, Target: Callee, ArgsList: std::move(Args))
20817 .setTailCall(isTailCall)
20818 .setSExtResult(SignExtend)
20819 .setZExtResult(!SignExtend)
20820 .setIsPostTypeLegalization(true);
20821 return TLI.LowerCallTo(CLI).first;
20822}
20823
20824SDValue PPCTargetLowering::lowerLibCallBasedOnType(
20825 const char *LibCallFloatName, const char *LibCallDoubleName, SDValue Op,
20826 SelectionDAG &DAG) const {
20827 if (Op.getValueType() == MVT::f32)
20828 return lowerToLibCall(LibCallName: LibCallFloatName, Op, DAG);
20829
20830 if (Op.getValueType() == MVT::f64)
20831 return lowerToLibCall(LibCallName: LibCallDoubleName, Op, DAG);
20832
20833 return SDValue();
20834}
20835
20836bool PPCTargetLowering::isLowringToMASSFiniteSafe(SDValue Op) const {
20837 SDNodeFlags Flags = Op.getNode()->getFlags();
20838 return isLowringToMASSSafe(Op) && Flags.hasNoSignedZeros() &&
20839 Flags.hasNoNaNs() && Flags.hasNoInfs();
20840}
20841
20842bool PPCTargetLowering::isLowringToMASSSafe(SDValue Op) const {
20843 return Op.getNode()->getFlags().hasApproximateFuncs();
20844}
20845
20846bool PPCTargetLowering::isScalarMASSConversionEnabled() const {
20847 return getTargetMachine().getOptLevel() == CodeGenOptLevel::Aggressive &&
20848 EnablePPCGenScalarMASSEntries;
20849}
20850
20851SDValue PPCTargetLowering::lowerLibCallBase(const char *LibCallDoubleName,
20852 const char *LibCallFloatName,
20853 const char *LibCallDoubleNameFinite,
20854 const char *LibCallFloatNameFinite,
20855 SDValue Op,
20856 SelectionDAG &DAG) const {
20857 if (!isScalarMASSConversionEnabled() || !isLowringToMASSSafe(Op))
20858 return SDValue();
20859
20860 if (!isLowringToMASSFiniteSafe(Op))
20861 return lowerLibCallBasedOnType(LibCallFloatName, LibCallDoubleName, Op,
20862 DAG);
20863
20864 return lowerLibCallBasedOnType(LibCallFloatName: LibCallFloatNameFinite,
20865 LibCallDoubleName: LibCallDoubleNameFinite, Op, DAG);
20866}
20867
20868SDValue PPCTargetLowering::lowerPow(SDValue Op, SelectionDAG &DAG) const {
20869 return lowerLibCallBase(LibCallDoubleName: "__xl_pow", LibCallFloatName: "__xl_powf", LibCallDoubleNameFinite: "__xl_pow_finite",
20870 LibCallFloatNameFinite: "__xl_powf_finite", Op, DAG);
20871}
20872
20873SDValue PPCTargetLowering::lowerSin(SDValue Op, SelectionDAG &DAG) const {
20874 return lowerLibCallBase(LibCallDoubleName: "__xl_sin", LibCallFloatName: "__xl_sinf", LibCallDoubleNameFinite: "__xl_sin_finite",
20875 LibCallFloatNameFinite: "__xl_sinf_finite", Op, DAG);
20876}
20877
20878SDValue PPCTargetLowering::lowerCos(SDValue Op, SelectionDAG &DAG) const {
20879 return lowerLibCallBase(LibCallDoubleName: "__xl_cos", LibCallFloatName: "__xl_cosf", LibCallDoubleNameFinite: "__xl_cos_finite",
20880 LibCallFloatNameFinite: "__xl_cosf_finite", Op, DAG);
20881}
20882
20883SDValue PPCTargetLowering::lowerLog(SDValue Op, SelectionDAG &DAG) const {
20884 return lowerLibCallBase(LibCallDoubleName: "__xl_log", LibCallFloatName: "__xl_logf", LibCallDoubleNameFinite: "__xl_log_finite",
20885 LibCallFloatNameFinite: "__xl_logf_finite", Op, DAG);
20886}
20887
20888SDValue PPCTargetLowering::lowerLog10(SDValue Op, SelectionDAG &DAG) const {
20889 return lowerLibCallBase(LibCallDoubleName: "__xl_log10", LibCallFloatName: "__xl_log10f", LibCallDoubleNameFinite: "__xl_log10_finite",
20890 LibCallFloatNameFinite: "__xl_log10f_finite", Op, DAG);
20891}
20892
20893SDValue PPCTargetLowering::lowerExp(SDValue Op, SelectionDAG &DAG) const {
20894 return lowerLibCallBase(LibCallDoubleName: "__xl_exp", LibCallFloatName: "__xl_expf", LibCallDoubleNameFinite: "__xl_exp_finite",
20895 LibCallFloatNameFinite: "__xl_expf_finite", Op, DAG);
20896}
20897
20898// If we happen to match to an aligned D-Form, check if the Frame Index is
20899// adequately aligned. If it is not, reset the mode to match to X-Form.
20900static void setXFormForUnalignedFI(SDValue N, unsigned Flags,
20901 PPC::AddrMode &Mode) {
20902 if (!isa<FrameIndexSDNode>(Val: N))
20903 return;
20904 if ((Mode == PPC::AM_DSForm && !(Flags & PPC::MOF_RPlusSImm16Mult4)) ||
20905 (Mode == PPC::AM_DQForm && !(Flags & PPC::MOF_RPlusSImm16Mult16)))
20906 Mode = PPC::AM_XForm;
20907}
20908
20909/// SelectOptimalAddrMode - Based on a node N and it's Parent (a MemSDNode),
20910/// compute the address flags of the node, get the optimal address mode based
20911/// on the flags, and set the Base and Disp based on the address mode.
20912PPC::AddrMode PPCTargetLowering::SelectOptimalAddrMode(const SDNode *Parent,
20913 SDValue N, SDValue &Disp,
20914 SDValue &Base,
20915 SelectionDAG &DAG,
20916 MaybeAlign Align) const {
20917 SDLoc DL(Parent);
20918
20919 // Compute the address flags.
20920 unsigned Flags = computeMOFlags(Parent, N, DAG);
20921
20922 // Get the optimal address mode based on the Flags.
20923 PPC::AddrMode Mode = getAddrModeForFlags(Flags);
20924
20925 // If the address mode is DS-Form or DQ-Form, check if the FI is aligned.
20926 // Select an X-Form load if it is not.
20927 setXFormForUnalignedFI(N, Flags, Mode);
20928
20929 // Set the mode to PC-Relative addressing mode if we have a valid PC-Rel node.
20930 if ((Mode == PPC::AM_XForm) && isPCRelNode(N)) {
20931 assert(Subtarget.isUsingPCRelativeCalls() &&
20932 "Must be using PC-Relative calls when a valid PC-Relative node is "
20933 "present!");
20934 Mode = PPC::AM_PCRel;
20935 }
20936
20937 // Set Base and Disp accordingly depending on the address mode.
20938 switch (Mode) {
20939 case PPC::AM_DForm:
20940 case PPC::AM_DSForm:
20941 case PPC::AM_DQForm: {
20942 // This is a register plus a 16-bit immediate. The base will be the
20943 // register and the displacement will be the immediate unless it
20944 // isn't sufficiently aligned.
20945 if (Flags & PPC::MOF_RPlusSImm16) {
20946 SDValue Op0 = N.getOperand(i: 0);
20947 SDValue Op1 = N.getOperand(i: 1);
20948 int16_t Imm = Op1->getAsZExtVal();
20949 if (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm)) {
20950 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: N.getValueType());
20951 Base = Op0;
20952 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: Op0)) {
20953 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
20954 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
20955 }
20956 break;
20957 }
20958 }
20959 // This is a register plus the @lo relocation. The base is the register
20960 // and the displacement is the global address.
20961 else if (Flags & PPC::MOF_RPlusLo) {
20962 Disp = N.getOperand(i: 1).getOperand(i: 0); // The global address.
20963 assert(Disp.getOpcode() == ISD::TargetGlobalAddress ||
20964 Disp.getOpcode() == ISD::TargetGlobalTLSAddress ||
20965 Disp.getOpcode() == ISD::TargetConstantPool ||
20966 Disp.getOpcode() == ISD::TargetJumpTable);
20967 Base = N.getOperand(i: 0);
20968 break;
20969 }
20970 // This is a constant address at most 32 bits. The base will be
20971 // zero or load-immediate-shifted and the displacement will be
20972 // the low 16 bits of the address.
20973 else if (Flags & PPC::MOF_AddrIsSImm32) {
20974 auto *CN = cast<ConstantSDNode>(Val&: N);
20975 EVT CNType = CN->getValueType(ResNo: 0);
20976 uint64_t CNImm = CN->getZExtValue();
20977 // If this address fits entirely in a 16-bit sext immediate field, codegen
20978 // this as "d, 0".
20979 int16_t Imm;
20980 if (isIntS16Immediate(N: CN, Imm) && (!Align || isAligned(Lhs: *Align, SizeInBytes: Imm))) {
20981 Disp = DAG.getSignedTargetConstant(Val: Imm, DL, VT: CNType);
20982 Base = DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
20983 VT: CNType);
20984 break;
20985 }
20986 // Handle 32-bit sext immediate with LIS + Addr mode.
20987 if ((CNType == MVT::i32 || isInt<32>(x: CNImm)) &&
20988 (!Align || isAligned(Lhs: *Align, SizeInBytes: CNImm))) {
20989 int32_t Addr = (int32_t)CNImm;
20990 // Otherwise, break this down into LIS + Disp.
20991 Disp = DAG.getSignedTargetConstant(Val: (int16_t)Addr, DL, VT: MVT::i32);
20992 Base = DAG.getSignedTargetConstant(Val: (Addr - (int16_t)Addr) >> 16, DL,
20993 VT: MVT::i32);
20994 uint32_t LIS = CNType == MVT::i32 ? PPC::LIS : PPC::LIS8;
20995 Base = SDValue(DAG.getMachineNode(Opcode: LIS, dl: DL, VT: CNType, Op1: Base), 0);
20996 break;
20997 }
20998 }
20999 // Otherwise, the PPC:MOF_NotAdd flag is set. Load/Store is Non-foldable.
21000 Disp = DAG.getTargetConstant(Val: 0, DL, VT: getPointerTy(DL: DAG.getDataLayout()));
21001 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N)) {
21002 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
21003 fixupFuncForFI(DAG, FrameIdx: FI->getIndex(), VT: N.getValueType());
21004 } else
21005 Base = N;
21006 break;
21007 }
21008 case PPC::AM_PrefixDForm: {
21009 int64_t Imm34 = 0;
21010 unsigned Opcode = N.getOpcode();
21011 if (((Opcode == ISD::ADD) || (Opcode == ISD::OR)) &&
21012 (isIntS34Immediate(Op: N.getOperand(i: 1), Imm&: Imm34))) {
21013 // N is an Add/OR Node, and it's operand is a 34-bit signed immediate.
21014 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
21015 if (FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val: N.getOperand(i: 0)))
21016 Base = DAG.getTargetFrameIndex(FI: FI->getIndex(), VT: N.getValueType());
21017 else
21018 Base = N.getOperand(i: 0);
21019 } else if (isIntS34Immediate(Op: N, Imm&: Imm34)) {
21020 // The address is a 34-bit signed immediate.
21021 Disp = DAG.getSignedTargetConstant(Val: Imm34, DL, VT: N.getValueType());
21022 Base = DAG.getRegister(Reg: PPC::ZERO8, VT: N.getValueType());
21023 }
21024 break;
21025 }
21026 case PPC::AM_PCRel: {
21027 // When selecting PC-Relative instructions, "Base" is not utilized as
21028 // we select the address as [PC+imm].
21029 Disp = N;
21030 break;
21031 }
21032 case PPC::AM_None:
21033 break;
21034 default: { // By default, X-Form is always available to be selected.
21035 // When a frame index is not aligned, we also match by XForm.
21036 FrameIndexSDNode *FI = dyn_cast<FrameIndexSDNode>(Val&: N);
21037 Base = FI ? N : N.getOperand(i: 1);
21038 Disp = FI ? DAG.getRegister(Reg: Subtarget.isPPC64() ? PPC::ZERO8 : PPC::ZERO,
21039 VT: N.getValueType())
21040 : N.getOperand(i: 0);
21041 break;
21042 }
21043 }
21044 return Mode;
21045}
21046
21047CCAssignFn *PPCTargetLowering::ccAssignFnForCall(CallingConv::ID CC,
21048 bool Return,
21049 bool IsVarArg) const {
21050 switch (CC) {
21051 case CallingConv::Cold:
21052 return (Return ? RetCC_PPC_Cold : CC_PPC64_ELF);
21053 default:
21054 return CC_PPC64_ELF;
21055 }
21056}
21057
21058bool PPCTargetLowering::shouldInlineQuadwordAtomics() const {
21059 return Subtarget.isPPC64() && Subtarget.hasQuadwordAtomics();
21060}
21061
21062TargetLowering::AtomicExpansionKind
21063PPCTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
21064 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
21065 if (shouldInlineQuadwordAtomics() && Size == 128)
21066 return AtomicExpansionKind::MaskedIntrinsic;
21067
21068 switch (AI->getOperation()) {
21069 case AtomicRMWInst::UIncWrap:
21070 case AtomicRMWInst::UDecWrap:
21071 case AtomicRMWInst::USubCond:
21072 case AtomicRMWInst::USubSat:
21073 return AtomicExpansionKind::CmpXChg;
21074 default:
21075 return TargetLowering::shouldExpandAtomicRMWInIR(RMW: AI);
21076 }
21077
21078 llvm_unreachable("unreachable atomicrmw operation");
21079}
21080
21081TargetLowering::AtomicExpansionKind
21082PPCTargetLowering::shouldExpandAtomicCmpXchgInIR(
21083 const AtomicCmpXchgInst *AI) const {
21084 unsigned Size = AI->getNewValOperand()->getType()->getPrimitiveSizeInBits();
21085 if (shouldInlineQuadwordAtomics() && Size == 128)
21086 return AtomicExpansionKind::MaskedIntrinsic;
21087 return AtomicExpansionKind::LLSC;
21088}
21089
21090static Intrinsic::ID
21091getIntrinsicForAtomicRMWBinOp128(AtomicRMWInst::BinOp BinOp) {
21092 switch (BinOp) {
21093 default:
21094 llvm_unreachable("Unexpected AtomicRMW BinOp");
21095 case AtomicRMWInst::Xchg:
21096 return Intrinsic::ppc_atomicrmw_xchg_i128;
21097 case AtomicRMWInst::Add:
21098 return Intrinsic::ppc_atomicrmw_add_i128;
21099 case AtomicRMWInst::Sub:
21100 return Intrinsic::ppc_atomicrmw_sub_i128;
21101 case AtomicRMWInst::And:
21102 return Intrinsic::ppc_atomicrmw_and_i128;
21103 case AtomicRMWInst::Or:
21104 return Intrinsic::ppc_atomicrmw_or_i128;
21105 case AtomicRMWInst::Xor:
21106 return Intrinsic::ppc_atomicrmw_xor_i128;
21107 case AtomicRMWInst::Nand:
21108 return Intrinsic::ppc_atomicrmw_nand_i128;
21109 }
21110}
21111
21112Value *PPCTargetLowering::emitMaskedAtomicRMWIntrinsic(
21113 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
21114 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
21115 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21116 Module *M = Builder.getModule();
21117 Type *ValTy = Incr->getType();
21118 assert(ValTy->getPrimitiveSizeInBits() == 128);
21119 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21120 Value *IncrLo = Builder.CreateTrunc(V: Incr, DestTy: Int64Ty, Name: "incr_lo");
21121 Value *IncrHi =
21122 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: Incr, RHS: 64), DestTy: Int64Ty, Name: "incr_hi");
21123 Value *LoHi = Builder.CreateIntrinsic(
21124 ID: getIntrinsicForAtomicRMWBinOp128(BinOp: AI->getOperation()), OverloadTypes: {},
21125 Args: {AlignedAddr, IncrLo, IncrHi});
21126 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21127 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21128 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21129 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21130 return Builder.CreateOr(
21131 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21132}
21133
21134Value *PPCTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
21135 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
21136 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
21137 assert(shouldInlineQuadwordAtomics() && "Only support quadword now");
21138 Module *M = Builder.getModule();
21139 Type *ValTy = CmpVal->getType();
21140 assert(ValTy->getPrimitiveSizeInBits() == 128);
21141 Function *IntCmpXchg =
21142 Intrinsic::getOrInsertDeclaration(M, id: Intrinsic::ppc_cmpxchg_i128);
21143 Type *Int64Ty = Type::getInt64Ty(C&: M->getContext());
21144 Value *CmpLo = Builder.CreateTrunc(V: CmpVal, DestTy: Int64Ty, Name: "cmp_lo");
21145 Value *CmpHi =
21146 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: CmpVal, RHS: 64), DestTy: Int64Ty, Name: "cmp_hi");
21147 Value *NewLo = Builder.CreateTrunc(V: NewVal, DestTy: Int64Ty, Name: "new_lo");
21148 Value *NewHi =
21149 Builder.CreateTrunc(V: Builder.CreateLShr(LHS: NewVal, RHS: 64), DestTy: Int64Ty, Name: "new_hi");
21150 emitLeadingFence(Builder, Inst: CI, Ord);
21151 Value *LoHi =
21152 Builder.CreateCall(Callee: IntCmpXchg, Args: {AlignedAddr, CmpLo, CmpHi, NewLo, NewHi});
21153 emitTrailingFence(Builder, Inst: CI, Ord);
21154 Value *Lo = Builder.CreateExtractValue(Agg: LoHi, Idxs: 0, Name: "lo");
21155 Value *Hi = Builder.CreateExtractValue(Agg: LoHi, Idxs: 1, Name: "hi");
21156 Lo = Builder.CreateZExt(V: Lo, DestTy: ValTy, Name: "lo64");
21157 Hi = Builder.CreateZExt(V: Hi, DestTy: ValTy, Name: "hi64");
21158 return Builder.CreateOr(
21159 LHS: Lo, RHS: Builder.CreateShl(LHS: Hi, RHS: ConstantInt::get(Ty: ValTy, V: 64)), Name: "val64");
21160}
21161
21162bool PPCTargetLowering::hasMultipleConditionRegisters(EVT VT) const {
21163 return Subtarget.useCRBits();
21164}
21165
21166/// Shuffle masks for vectors of bits are not legal as such vectors are
21167/// reserved for MMA/DM.
21168bool PPCTargetLowering::isShuffleMaskLegal(ArrayRef<int> Mask, EVT VT) const {
21169 if (VT.getScalarType() == MVT::i1)
21170 return false;
21171 return TargetLowering::isShuffleMaskLegal(Mask, VT);
21172}
21173
21174// Optimize the following patterns using vbpermq/vbpermd:
21175// i16 = bitcast(v16i1 truncate(v16i8))
21176// i8 = bitcast(v8i1 truncate(v8i16))
21177// i8 = bitcast(v8i1 truncate(v8i8))
21178SDValue PPCTargetLowering::DAGCombineBitcast(SDNode *N,
21179 DAGCombinerInfo &DCI) const {
21180 SDValue Op0 = N->getOperand(Num: 0);
21181 if (Op0.getOpcode() != ISD::TRUNCATE)
21182 return SDValue();
21183 SDValue Src = Op0.getOperand(i: 0);
21184 EVT ResVT = N->getValueType(ResNo: 0);
21185 EVT TruncResVT = Op0.getValueType();
21186 EVT SrcVT = Src.getValueType();
21187 SDLoc dl(N);
21188 SelectionDAG &DAG = DCI.DAG;
21189 bool IsLittleEndian = Subtarget.isLittleEndian();
21190
21191 if (ResVT != MVT::i16 && ResVT != MVT::i8)
21192 return SDValue();
21193 SDValue VBPerm =
21194 GenerateVBPERM(DAG, dl, Src, SrcVT, ResVT: TruncResVT, IsLE: IsLittleEndian);
21195 if (!VBPerm)
21196 return SDValue();
21197 SDValue ForExtract = DAG.getBitcast(VT: MVT::v4i32, V: VBPerm);
21198 SDValue Extracted =
21199 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL: dl, VT: MVT::i32, N1: ForExtract,
21200 N2: DAG.getIntPtrConstant(Val: IsLittleEndian ? 2 : 1, DL: dl));
21201 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: ResVT, Operand: Extracted);
21202}
21203
21204SDValue PPCTargetLowering::GenerateVBPERM(SelectionDAG &DAG, SDLoc dl,
21205 SDValue Src, EVT SrcVT, EVT ResVT,
21206 bool IsLE) const {
21207 bool IsV16i8 = (ResVT == MVT::v16i1 && SrcVT == MVT::v16i8);
21208 bool IsV8i16 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i16);
21209 bool IsV8i8 = (ResVT == MVT::v8i1 && SrcVT == MVT::v8i8);
21210
21211 if (!IsV16i8 && !IsV8i16 && !IsV8i8)
21212 return SDValue();
21213
21214 if (IsV8i8) {
21215 Src = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL: dl, VT: MVT::v16i8,
21216 N1: DAG.getUNDEF(VT: MVT::v16i8), N2: Src,
21217 N3: DAG.getIntPtrConstant(Val: 0, DL: dl));
21218 }
21219 SmallVector<int, 16> BitIndices(16, 128);
21220 unsigned NumElts = SrcVT.getVectorNumElements();
21221 unsigned EltSize = SrcVT.getScalarType().getSizeInBits();
21222 for (int Idx = 0, End = SrcVT.getVectorNumElements(); Idx < End; Idx++) {
21223 BitIndices[Idx] = EltSize * (NumElts - Idx) - 1;
21224 if (IsV8i8 && IsLE)
21225 BitIndices[Idx] += 64;
21226 }
21227 if (!IsLE)
21228 std::reverse(first: BitIndices.begin(), last: BitIndices.end());
21229 SmallVector<SDValue, 16> BVOps;
21230 for (auto Idx : BitIndices)
21231 BVOps.push_back(Elt: DAG.getConstant(Val: Idx, DL: dl, VT: MVT::i8));
21232 SDValue VRB = DAG.getBuildVector(VT: MVT::v16i8, DL: dl, Ops: BVOps);
21233 return DAG.getNode(
21234 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v16i8,
21235 N1: DAG.getConstant(Val: Intrinsic::ppc_altivec_vbpermq, DL: dl, VT: MVT::i32),
21236 N2: DAG.getBitcast(VT: MVT::v16i8, V: Src), N3: VRB);
21237}
21238
21239// For Power8/9, optimize vec splats of small FP values that can be
21240// represented as integers. Use vspltisw + xvcvsxwdp/xvcvsxwsp instead of
21241// loading from constant pool.
21242SDValue PPCTargetLowering::LowerVecSplatSmallFP(SDValue Op, SelectionDAG &DAG,
21243 bool BVNIsConstantSplat,
21244 unsigned SplatBitSize) const {
21245
21246 if (!BVNIsConstantSplat || !Subtarget.hasVSX() || !Subtarget.hasP8Vector() ||
21247 Subtarget.hasP10Vector())
21248 return SDValue();
21249
21250 EVT VT = Op->getValueType(ResNo: 0);
21251 if (!((SplatBitSize == 64 && VT == MVT::v2f64) ||
21252 (SplatBitSize == 32 && VT == MVT::v4f32)))
21253 return SDValue();
21254
21255 auto *CN = dyn_cast<ConstantFPSDNode>(Val: Op.getOperand(i: 0));
21256 if (!CN)
21257 return SDValue();
21258
21259 APFloat APFloatVal = CN->getValueAPF();
21260 bool IsExact;
21261 APSInt IntResult(16, false);
21262 APFloatVal.convertToInteger(Result&: IntResult, RM: APFloat::rmTowardZero, IsExact: &IsExact);
21263
21264 if (!(IsExact && IntResult <= 15 && IntResult >= -16 && !APFloatVal.isZero()))
21265 return SDValue();
21266
21267 int64_t IntVal = IntResult.getSExtValue();
21268
21269 SDLoc dl(Op);
21270 SDValue IntSplat = getCanonicalConstSplat(Val: IntVal, SplatSize: 4, VT: MVT::v4i32, DAG, dl);
21271
21272 if (SplatBitSize == 64)
21273 return DAG.getNode(
21274 Opcode: ISD::INTRINSIC_WO_CHAIN, DL: dl, VT: MVT::v2f64,
21275 N1: DAG.getConstant(Val: Intrinsic::ppc_vsx_xvcvsxwdp, DL: dl, VT: MVT::i32), N2: IntSplat);
21276
21277 return DAG.getNode(Opcode: PPCISD::XVCVSXWSP, DL: dl, VT: MVT::v4f32, Operand: IntSplat);
21278}
21279