1//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISC-V uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVConstantPoolValue.h"
18#include "RISCVMachineFunctionInfo.h"
19#include "RISCVRegisterInfo.h"
20#include "RISCVSelectionDAGInfo.h"
21#include "RISCVSubtarget.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Analysis/MemoryLocation.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/Analysis/VectorUtils.h"
28#include "llvm/CodeGen/GlobalISel/GISelValueTracking.h"
29#include "llvm/CodeGen/MachineFrameInfo.h"
30#include "llvm/CodeGen/MachineFunction.h"
31#include "llvm/CodeGen/MachineInstrBuilder.h"
32#include "llvm/CodeGen/MachineJumpTableInfo.h"
33#include "llvm/CodeGen/MachineRegisterInfo.h"
34#include "llvm/CodeGen/SDPatternMatch.h"
35#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
36#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
37#include "llvm/CodeGen/ValueTypes.h"
38#include "llvm/IR/DiagnosticInfo.h"
39#include "llvm/IR/DiagnosticPrinter.h"
40#include "llvm/IR/IRBuilder.h"
41#include "llvm/IR/Instructions.h"
42#include "llvm/IR/IntrinsicInst.h"
43#include "llvm/IR/IntrinsicsRISCV.h"
44#include "llvm/MC/MCCodeEmitter.h"
45#include "llvm/MC/MCInstBuilder.h"
46#include "llvm/Support/CommandLine.h"
47#include "llvm/Support/Debug.h"
48#include "llvm/Support/ErrorHandling.h"
49#include "llvm/Support/InstructionCost.h"
50#include "llvm/Support/KnownBits.h"
51#include "llvm/Support/MathExtras.h"
52#include "llvm/Support/raw_ostream.h"
53#include <optional>
54
55using namespace llvm;
56
57#define DEBUG_TYPE "riscv-lower"
58
59STATISTIC(NumTailCalls, "Number of tail calls");
60
61static cl::opt<unsigned> ExtensionMaxWebSize(
62 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
63 cl::desc("Give the maximum size (in number of nodes) of the web of "
64 "instructions that we will consider for VW expansion"),
65 cl::init(Val: 18));
66
67static cl::opt<bool>
68 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
69 cl::desc("Allow the formation of VW_W operations (e.g., "
70 "VWADD_W) with splat constants"),
71 cl::init(Val: false));
72
73static cl::opt<unsigned> NumRepeatedDivisors(
74 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
75 cl::desc("Set the minimum number of repetitions of a divisor to allow "
76 "transformation to multiplications by the reciprocal"),
77 cl::init(Val: 2));
78
79static cl::opt<int>
80 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
81 cl::desc("Give the maximum number of instructions that we will "
82 "use for creating a floating-point immediate value"),
83 cl::init(Val: 3));
84
85static cl::opt<bool>
86 ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
87 cl::desc("Swap add and addi in cases where the add may "
88 "be combined with a shift"),
89 cl::init(Val: true));
90
91static cl::opt<int> BrMergingBaseCostThresh(
92 "riscv-br-merging-base-cost", cl::init(Val: 2),
93 cl::desc(
94 "Sets the cost threshold for when multiple conditionals will be merged "
95 "into one branch versus be split in multiple branches. Merging "
96 "conditionals saves branches at the cost of additional instructions. "
97 "This value sets the instruction cost limit, below which conditionals "
98 "will be merged, and above which conditionals will be split. Set to -1 "
99 "to never merge branches."),
100 cl::Hidden);
101
102static cl::opt<int> BrMergingLikelyBias(
103 "riscv-br-merging-likely-bias", cl::init(Val: 0),
104 cl::desc(
105 "Increases 'riscv-br-merging-base-cost' in cases that it is "
106 "likely that all conditionals will be executed. For example for "
107 "merging the conditionals (a == b && c > d), if its known that "
108 "a == b is likely, then it is likely that if the conditionals are "
109 "split both sides will be executed, so it may be desirable to "
110 "increase the instruction cost threshold. Set to -1 to never merge "
111 "likely branches."),
112 cl::Hidden);
113
114static cl::opt<int> BrMergingUnlikelyBias(
115 "riscv-br-merging-unlikely-bias", cl::init(Val: -1),
116 cl::desc(
117 "Decreases 'riscv-br-merging-base-cost' in cases that it is unlikely "
118 "that all conditionals will be executed. For example for merging "
119 "the conditionals (a == b && c > d), if its known that a == b is "
120 "unlikely, then it is unlikely that if the conditionals are split "
121 "both sides will be executed, so it may be desirable to decrease "
122 "the instruction cost threshold. Set to -1 to never merge unlikely "
123 "branches."),
124 cl::Hidden);
125
126// TODO: Support more ops
127static const unsigned ZvfbfaOps[] = {
128 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FADD,
129 ISD::FSUB, ISD::FMUL, ISD::FMINNUM, ISD::FMAXNUM,
130 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUM, ISD::FMAXIMUM,
131 ISD::FMA, ISD::IS_FPCLASS, ISD::STRICT_FADD, ISD::STRICT_FSUB,
132 ISD::STRICT_FMUL, ISD::STRICT_FMA, ISD::SETCC};
133
134RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
135 const RISCVSubtarget &STI)
136 : TargetLowering(TM, STI), Subtarget(STI) {
137
138 RISCVABI::ABI ABI = Subtarget.getTargetABI();
139 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
140
141 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
142 !Subtarget.hasStdExtF()) {
143 errs() << "Hard-float 'f' ABI can't be used for a target that "
144 "doesn't support the F instruction set extension (ignoring "
145 "target-abi)\n";
146 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
147 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
148 !Subtarget.hasStdExtD()) {
149 errs() << "Hard-float 'd' ABI can't be used for a target that "
150 "doesn't support the D instruction set extension (ignoring "
151 "target-abi)\n";
152 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
153 }
154
155 switch (ABI) {
156 default:
157 reportFatalUsageError(reason: "Don't know how to lower this ABI");
158 case RISCVABI::ABI_ILP32:
159 case RISCVABI::ABI_ILP32E:
160 case RISCVABI::ABI_LP64E:
161 case RISCVABI::ABI_ILP32F:
162 case RISCVABI::ABI_ILP32D:
163 case RISCVABI::ABI_LP64:
164 case RISCVABI::ABI_LP64F:
165 case RISCVABI::ABI_LP64D:
166 break;
167 }
168
169 MVT XLenVT = Subtarget.getXLenVT();
170
171 // Set up the register classes.
172 addRegisterClass(VT: XLenVT, RC: &RISCV::GPRRegClass);
173
174 if (Subtarget.hasStdExtZfhmin())
175 addRegisterClass(VT: MVT::f16, RC: &RISCV::FPR16RegClass);
176 if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
177 addRegisterClass(VT: MVT::bf16, RC: &RISCV::FPR16RegClass);
178 if (Subtarget.hasStdExtF())
179 addRegisterClass(VT: MVT::f32, RC: &RISCV::FPR32RegClass);
180 if (Subtarget.hasStdExtD())
181 addRegisterClass(VT: MVT::f64, RC: &RISCV::FPR64RegClass);
182 if (Subtarget.hasStdExtZhinxmin())
183 addRegisterClass(VT: MVT::f16, RC: &RISCV::GPRF16RegClass);
184 if (Subtarget.hasStdExtZfinx())
185 addRegisterClass(VT: MVT::f32, RC: &RISCV::GPRF32RegClass);
186 if (Subtarget.hasStdExtZdinx()) {
187 if (Subtarget.is64Bit())
188 addRegisterClass(VT: MVT::f64, RC: &RISCV::GPRRegClass);
189 else
190 addRegisterClass(VT: MVT::f64, RC: &RISCV::GPRPairRegClass);
191 }
192
193 static const MVT::SimpleValueType BoolVecVTs[] = {
194 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
195 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
196 static const MVT::SimpleValueType IntVecVTs[] = {
197 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
198 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
199 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
200 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
201 MVT::nxv4i64, MVT::nxv8i64};
202 static const MVT::SimpleValueType F16VecVTs[] = {
203 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
204 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
205 static const MVT::SimpleValueType BF16VecVTs[] = {
206 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
207 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
208 static const MVT::SimpleValueType F32VecVTs[] = {
209 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
210 static const MVT::SimpleValueType F64VecVTs[] = {
211 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
212 static const MVT::SimpleValueType VecTupleVTs[] = {
213 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
214 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
215 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
216 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
217 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
218 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
219 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
220 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
221 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
222 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
223 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
224
225 if (Subtarget.hasVInstructions()) {
226 auto addRegClassForRVV = [this](MVT VT) {
227 // Disable the smallest fractional LMUL types if ELEN is less than
228 // RVVBitsPerBlock.
229 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
230 if (VT.getVectorMinNumElements() < MinElts)
231 return;
232
233 unsigned Size = VT.getSizeInBits().getKnownMinValue();
234 const TargetRegisterClass *RC;
235 if (Size <= RISCV::RVVBitsPerBlock)
236 RC = &RISCV::VRRegClass;
237 else if (Size == 2 * RISCV::RVVBitsPerBlock)
238 RC = &RISCV::VRM2RegClass;
239 else if (Size == 4 * RISCV::RVVBitsPerBlock)
240 RC = &RISCV::VRM4RegClass;
241 else if (Size == 8 * RISCV::RVVBitsPerBlock)
242 RC = &RISCV::VRM8RegClass;
243 else
244 llvm_unreachable("Unexpected size");
245
246 addRegisterClass(VT, RC);
247 };
248
249 for (MVT VT : BoolVecVTs)
250 addRegClassForRVV(VT);
251 for (MVT VT : IntVecVTs) {
252 if (VT.getVectorElementType() == MVT::i64 &&
253 !Subtarget.hasVInstructionsI64())
254 continue;
255 addRegClassForRVV(VT);
256 }
257
258 if (Subtarget.hasVInstructionsF16Minimal() ||
259 Subtarget.hasVendorXAndesVPackFPH())
260 for (MVT VT : F16VecVTs)
261 addRegClassForRVV(VT);
262
263 if (Subtarget.hasVInstructionsBF16Minimal() ||
264 Subtarget.hasVendorXAndesVBFHCvt())
265 for (MVT VT : BF16VecVTs)
266 addRegClassForRVV(VT);
267
268 if (Subtarget.hasVInstructionsF32())
269 for (MVT VT : F32VecVTs)
270 addRegClassForRVV(VT);
271
272 if (Subtarget.hasVInstructionsF64())
273 for (MVT VT : F64VecVTs)
274 addRegClassForRVV(VT);
275
276 if (Subtarget.useRVVForFixedLengthVectors()) {
277 auto addRegClassForFixedVectors = [this](MVT VT) {
278 MVT ContainerVT = getContainerForFixedLengthVector(VT);
279 unsigned RCID = getRegClassIDForVecVT(VT: ContainerVT);
280 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
281 addRegisterClass(VT, RC: TRI.getRegClass(i: RCID));
282 };
283 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
284 if (useRVVForFixedLengthVectorVT(VT))
285 addRegClassForFixedVectors(VT);
286
287 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
288 if (useRVVForFixedLengthVectorVT(VT))
289 addRegClassForFixedVectors(VT);
290 }
291
292 addRegisterClass(VT: MVT::riscv_nxv1i8x2, RC: &RISCV::VRN2M1RegClass);
293 addRegisterClass(VT: MVT::riscv_nxv1i8x3, RC: &RISCV::VRN3M1RegClass);
294 addRegisterClass(VT: MVT::riscv_nxv1i8x4, RC: &RISCV::VRN4M1RegClass);
295 addRegisterClass(VT: MVT::riscv_nxv1i8x5, RC: &RISCV::VRN5M1RegClass);
296 addRegisterClass(VT: MVT::riscv_nxv1i8x6, RC: &RISCV::VRN6M1RegClass);
297 addRegisterClass(VT: MVT::riscv_nxv1i8x7, RC: &RISCV::VRN7M1RegClass);
298 addRegisterClass(VT: MVT::riscv_nxv1i8x8, RC: &RISCV::VRN8M1RegClass);
299 addRegisterClass(VT: MVT::riscv_nxv2i8x2, RC: &RISCV::VRN2M1RegClass);
300 addRegisterClass(VT: MVT::riscv_nxv2i8x3, RC: &RISCV::VRN3M1RegClass);
301 addRegisterClass(VT: MVT::riscv_nxv2i8x4, RC: &RISCV::VRN4M1RegClass);
302 addRegisterClass(VT: MVT::riscv_nxv2i8x5, RC: &RISCV::VRN5M1RegClass);
303 addRegisterClass(VT: MVT::riscv_nxv2i8x6, RC: &RISCV::VRN6M1RegClass);
304 addRegisterClass(VT: MVT::riscv_nxv2i8x7, RC: &RISCV::VRN7M1RegClass);
305 addRegisterClass(VT: MVT::riscv_nxv2i8x8, RC: &RISCV::VRN8M1RegClass);
306 addRegisterClass(VT: MVT::riscv_nxv4i8x2, RC: &RISCV::VRN2M1RegClass);
307 addRegisterClass(VT: MVT::riscv_nxv4i8x3, RC: &RISCV::VRN3M1RegClass);
308 addRegisterClass(VT: MVT::riscv_nxv4i8x4, RC: &RISCV::VRN4M1RegClass);
309 addRegisterClass(VT: MVT::riscv_nxv4i8x5, RC: &RISCV::VRN5M1RegClass);
310 addRegisterClass(VT: MVT::riscv_nxv4i8x6, RC: &RISCV::VRN6M1RegClass);
311 addRegisterClass(VT: MVT::riscv_nxv4i8x7, RC: &RISCV::VRN7M1RegClass);
312 addRegisterClass(VT: MVT::riscv_nxv4i8x8, RC: &RISCV::VRN8M1RegClass);
313 addRegisterClass(VT: MVT::riscv_nxv8i8x2, RC: &RISCV::VRN2M1RegClass);
314 addRegisterClass(VT: MVT::riscv_nxv8i8x3, RC: &RISCV::VRN3M1RegClass);
315 addRegisterClass(VT: MVT::riscv_nxv8i8x4, RC: &RISCV::VRN4M1RegClass);
316 addRegisterClass(VT: MVT::riscv_nxv8i8x5, RC: &RISCV::VRN5M1RegClass);
317 addRegisterClass(VT: MVT::riscv_nxv8i8x6, RC: &RISCV::VRN6M1RegClass);
318 addRegisterClass(VT: MVT::riscv_nxv8i8x7, RC: &RISCV::VRN7M1RegClass);
319 addRegisterClass(VT: MVT::riscv_nxv8i8x8, RC: &RISCV::VRN8M1RegClass);
320 addRegisterClass(VT: MVT::riscv_nxv16i8x2, RC: &RISCV::VRN2M2RegClass);
321 addRegisterClass(VT: MVT::riscv_nxv16i8x3, RC: &RISCV::VRN3M2RegClass);
322 addRegisterClass(VT: MVT::riscv_nxv16i8x4, RC: &RISCV::VRN4M2RegClass);
323 addRegisterClass(VT: MVT::riscv_nxv32i8x2, RC: &RISCV::VRN2M4RegClass);
324 }
325
326 // fixed vector is stored in GPRs for P extension packed operations
327 if (Subtarget.hasStdExtP()) {
328 if (Subtarget.is64Bit()) {
329 addRegisterClass(VT: MVT::v2i32, RC: &RISCV::GPRRegClass);
330 addRegisterClass(VT: MVT::v4i16, RC: &RISCV::GPRRegClass);
331 addRegisterClass(VT: MVT::v8i8, RC: &RISCV::GPRRegClass);
332 } else {
333 addRegisterClass(VT: MVT::v2i16, RC: &RISCV::GPRRegClass);
334 addRegisterClass(VT: MVT::v4i8, RC: &RISCV::GPRRegClass);
335
336 addRegisterClass(VT: MVT::v2i32, RC: &RISCV::GPRPairRegClass);
337 addRegisterClass(VT: MVT::v4i16, RC: &RISCV::GPRPairRegClass);
338 addRegisterClass(VT: MVT::v8i8, RC: &RISCV::GPRPairRegClass);
339 }
340 }
341
342 // Compute derived properties from the register classes.
343 computeRegisterProperties(TRI: STI.getRegisterInfo());
344
345 setStackPointerRegisterToSaveRestore(RISCV::X2);
346
347 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: XLenVT,
348 MemVT: MVT::i1, Action: Promote);
349 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
350 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
351 MemVT: MVT::i1, Action: Promote);
352
353 // TODO: add all necessary setOperationAction calls.
354 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: XLenVT, Action: Custom);
355
356 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
357 setOperationAction(Op: ISD::BR_CC, VT: XLenVT, Action: Expand);
358 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
359 setOperationAction(Op: ISD::SELECT_CC, VT: XLenVT, Action: Expand);
360
361 setCondCodeAction(CCs: ISD::SETGT, VT: XLenVT, Action: Custom);
362 setCondCodeAction(CCs: ISD::SETGE, VT: XLenVT, Action: Expand);
363 setCondCodeAction(CCs: ISD::SETUGT, VT: XLenVT, Action: Custom);
364 setCondCodeAction(CCs: ISD::SETUGE, VT: XLenVT, Action: Expand);
365 if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
366 setCondCodeAction(CCs: ISD::SETULE, VT: XLenVT, Action: Expand);
367 setCondCodeAction(CCs: ISD::SETLE, VT: XLenVT, Action: Expand);
368 }
369
370 setOperationAction(Ops: {ISD::STACKSAVE, ISD::STACKRESTORE}, VT: MVT::Other, Action: Expand);
371
372 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
373 setOperationAction(Ops: {ISD::VAARG, ISD::VACOPY, ISD::VAEND}, VT: MVT::Other, Action: Expand);
374
375 if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
376 !Subtarget.hasVendorXAndesPerf())
377 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
378
379 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
380
381 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
382 !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
383 !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
384 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::i8, MVT::i16}, Action: Expand);
385
386 if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
387 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
388 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
389 }
390
391 if (Subtarget.is64Bit()) {
392 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i64, Action: Custom);
393
394 setOperationAction(Op: ISD::LOAD, VT: MVT::i32, Action: Custom);
395 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
396 VT: MVT::i32, Action: Custom);
397 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Custom);
398 setOperationAction(Ops: {ISD::SADDO, ISD::SSUBO}, VT: MVT::i32, Action: Custom);
399 } else if (Subtarget.hasStdExtP()) {
400 // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
401 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT: MVT::i64, Action: Custom);
402 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VT: MVT::i64, Action: Custom);
403 }
404 if (!Subtarget.hasStdExtZmmul()) {
405 setOperationAction(Ops: {ISD::MUL, ISD::MULHS, ISD::MULHU}, VT: XLenVT, Action: Expand);
406 } else if (Subtarget.is64Bit()) {
407 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Custom);
408 setOperationAction(Op: ISD::MUL, VT: MVT::i32, Action: Custom);
409 } else {
410 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
411 }
412
413 if (!Subtarget.hasStdExtM()) {
414 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT: XLenVT,
415 Action: Expand);
416 } else if (Subtarget.is64Bit()) {
417 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::UREM},
418 VTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Custom);
419 }
420
421 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT: XLenVT, Action: Expand);
422
423 // On RV32, the P extension has a WMUL(U) instruction we can use for
424 // (S/U)MUL_LOHI.
425 // FIXME: Does P imply Zmmul?
426 if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
427 Subtarget.is64Bit())
428 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: XLenVT, Action: Expand);
429
430 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, VT: XLenVT,
431 Action: Custom);
432
433 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
434 if (Subtarget.is64Bit())
435 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: MVT::i32, Action: Custom);
436 } else if (Subtarget.hasVendorXTHeadBb()) {
437 if (Subtarget.is64Bit())
438 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: MVT::i32, Action: Custom);
439 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: XLenVT, Action: Custom);
440 } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
441 setOperationAction(Op: ISD::ROTL, VT: XLenVT, Action: Expand);
442 } else {
443 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: XLenVT, Action: Expand);
444 }
445
446 if (Subtarget.hasStdExtP())
447 setOperationAction(Ops: {ISD::FSHL, ISD::FSHR}, VT: XLenVT, Action: Legal);
448
449 setOperationAction(Op: ISD::BSWAP, VT: XLenVT,
450 Action: Subtarget.hasREV8Like() ? Legal : Expand);
451
452 if (Subtarget.hasREVLike()) {
453 setOperationAction(Op: ISD::BITREVERSE, VT: XLenVT, Action: Legal);
454 } else {
455 // Zbkb can use rev8+brev8 to implement bitreverse.
456 setOperationAction(Op: ISD::BITREVERSE, VT: XLenVT,
457 Action: Subtarget.hasStdExtZbkb() ? Custom : Expand);
458 if (Subtarget.hasStdExtZbkb())
459 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i8, Action: Custom);
460 }
461
462 if (Subtarget.hasStdExtZbb() ||
463 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
464 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT: XLenVT,
465 Action: Legal);
466 }
467
468 if (Subtarget.hasCTZLike()) {
469 if (Subtarget.is64Bit())
470 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
471 } else {
472 setOperationAction(Op: ISD::CTTZ, VT: XLenVT, Action: Expand);
473 }
474
475 if (!Subtarget.hasCPOPLike()) {
476 // TODO: These should be set to LibCall, but this currently breaks
477 // the Linux kernel build. See #101786. Lacks i128 tests, too.
478 if (Subtarget.is64Bit())
479 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Expand);
480 else
481 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
482 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
483 }
484
485 if (Subtarget.hasCLZLike()) {
486 // We need the custom lowering to make sure that the resulting sequence
487 // for the 32bit case is efficient on 64bit targets.
488 // Use default promotion for i32 without Zbb.
489 if (Subtarget.is64Bit() &&
490 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
491 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON}, VT: MVT::i32, Action: Custom);
492 } else {
493 if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit())
494 setOperationAction(Op: ISD::CTLZ_ZERO_POISON, VT: XLenVT, Action: Legal);
495 setOperationAction(Op: ISD::CTLZ, VT: XLenVT, Action: Expand);
496 }
497
498 if (Subtarget.hasStdExtP()) {
499 setOperationAction(Op: ISD::CTLS, VT: XLenVT, Action: Legal);
500 if (Subtarget.is64Bit())
501 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
502 }
503
504 if (Subtarget.hasStdExtP() ||
505 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
506 setOperationAction(Op: ISD::ABS, VT: XLenVT, Action: Legal);
507 if (Subtarget.is64Bit())
508 setOperationAction(Ops: {ISD::ABS, ISD::ABS_MIN_POISON}, VT: MVT::i32, Action: Custom);
509 } else if (Subtarget.hasShortForwardBranchIALU()) {
510 // We can use PseudoCCSUB to implement ABS.
511 setOperationAction(Op: ISD::ABS, VT: XLenVT, Action: Legal);
512 } else if (Subtarget.is64Bit()) {
513 setOperationAction(Ops: {ISD::ABS, ISD::ABS_MIN_POISON}, VT: MVT::i32, Action: Custom);
514 }
515
516 if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
517 setOperationAction(Op: ISD::SELECT, VT: XLenVT, Action: Custom);
518
519 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
520 !Subtarget.is64Bit()) {
521 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
522 VT: MVT::i32, Action: Legal);
523 } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
524 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
525 VT: MVT::i32, Action: Custom);
526 } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
527 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
528 VT: MVT::i32, Action: Custom);
529 }
530
531 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
532 !Subtarget.is64Bit()) {
533 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
534 // pssha.w/psshl.w and extracting.
535 setOperationAction(Op: ISD::SSHLSAT, VT: MVT::i32, Action: Legal);
536 setOperationAction(Op: ISD::USHLSAT, VT: MVT::i32, Action: Legal);
537 }
538
539 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
540 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
541 // paadd.w, paaddu.w and extracting.
542 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU}, VT: MVT::i32, Action: Legal);
543 }
544
545 if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc())
546 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT: XLenVT, Action: Legal);
547 if (Subtarget.hasStdExtZbc())
548 setOperationAction(Op: ISD::CLMULR, VT: XLenVT, Action: Legal);
549
550 static const unsigned FPLegalNodeTypes[] = {
551 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
552 ISD::FMAXIMUMNUM, ISD::LRINT, ISD::LLRINT,
553 ISD::LROUND, ISD::LLROUND, ISD::STRICT_LRINT,
554 ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
555 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
556 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
557 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
558
559 static const ISD::CondCode FPCCToExpand[] = {
560 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
561 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
562 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
563
564 static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
565 ISD::FPOW};
566 static const unsigned FPOpToLibCall[] = {ISD::FREM};
567
568 static const unsigned FPRndMode[] = {
569 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
570 ISD::FROUNDEVEN};
571
572 static const unsigned ZfhminZfbfminPromoteOps[] = {
573 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUM,
574 ISD::FMAXIMUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM,
575 ISD::FADD, ISD::FSUB, ISD::FMUL,
576 ISD::FMA, ISD::FDIV, ISD::FSQRT,
577 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
578 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
579 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::SETCC,
580 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC,
581 ISD::FRINT, ISD::FROUND, ISD::FROUNDEVEN,
582 ISD::FCANONICALIZE};
583
584 if (Subtarget.hasStdExtP()) {
585 static const MVT P32VecVTs[] = {MVT::v2i16, MVT::v4i8};
586 static const MVT P64VecVTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
587 ArrayRef<MVT> VTs;
588 if (Subtarget.is64Bit()) {
589 VTs = P64VecVTs;
590 // There's no instruction for vector shamt in P extension so we unroll to
591 // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
592 // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
593 // widen for those operations that will be unrolled.
594 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA},
595 VTs: {MVT::v2i16, MVT::v4i8}, Action: Custom);
596 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN, VTs: {MVT::v2i16, MVT::v4i8},
597 Action: Custom);
598 // Operand legalization queries the action using the illegal subvector.
599 setOperationAction(Ops: ISD::INSERT_SUBVECTOR, VTs: {MVT::v2i16, MVT::v4i8},
600 Action: Custom);
601 } else {
602 VTs = P32VecVTs;
603 }
604 // By default everything must be expanded.
605 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
606 setOperationAction(Ops: Op, VTs, Action: Expand);
607
608 for (MVT VT : VTs) {
609 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
610 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
611 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
612 MemVT: OtherVT, Action: Expand);
613 }
614 }
615
616 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs, Action: Legal);
617 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VTs, Action: Legal);
618 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VTs, Action: Legal);
619 setOperationAction(Ops: ISD::UADDSAT, VTs, Action: Legal);
620 setOperationAction(Ops: ISD::SADDSAT, VTs, Action: Legal);
621 setOperationAction(Ops: ISD::USUBSAT, VTs, Action: Legal);
622 setOperationAction(Ops: ISD::SSUBSAT, VTs, Action: Legal);
623 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Action: Legal);
624 setOperationAction(Ops: ISD::BITREVERSE, VTs, Action: Legal);
625 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs, Action: Custom);
626 setOperationAction(Ops: ISD::VECTOR_REVERSE, VTs, Action: Legal);
627 for (MVT VT : VTs) {
628 if (VT != MVT::v2i32)
629 setOperationAction(Ops: {ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Action: Legal);
630 if (VT.getVectorElementType() != MVT::i8) {
631 setOperationAction(Op: ISD::SSHLSAT, VT, Action: Custom);
632 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
633 }
634 }
635 setOperationAction(Ops: ISD::UNDEF, VTs, Action: Legal);
636 setOperationAction(Ops: ISD::SPLAT_VECTOR, VTs, Action: Legal);
637 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs, Action: Legal);
638 setOperationAction(Ops: ISD::SCALAR_TO_VECTOR, VTs, Action: Legal);
639 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Action: Custom);
640 setOperationAction(Ops: ISD::BITCAST, VTs, Action: Custom);
641 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
642 Action: Custom);
643 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
644 Action: Legal);
645 setOperationAction(Ops: ISD::SELECT, VTs, Action: Custom);
646 setOperationAction(Ops: ISD::VSELECT, VTs, Action: Legal);
647 setOperationAction(Ops: ISD::SETCC, VTs, Action: Legal);
648 setCondCodeAction(
649 CCs: {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE}, VTs,
650 Action: Expand);
651 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETGT}, VTs, Action: Custom);
652
653 if (!Subtarget.is64Bit())
654 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: {MVT::v2i16, MVT::v4i8}, Action: Custom);
655
656 // P extension vector comparisons produce all 1s for true, all 0s for false
657 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
658
659 if (!Subtarget.is64Bit()) {
660 // By default everything must be expanded.
661 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
662 setOperationAction(Ops: Op, VTs: P64VecVTs, Action: Expand);
663
664 for (MVT VT : P64VecVTs) {
665 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
666 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
667 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
668 MemVT: OtherVT, Action: Expand);
669 }
670 }
671
672 setOperationAction(Ops: ISD::UNDEF, VTs: P64VecVTs, Action: Legal);
673 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs: P64VecVTs, Action: Custom);
674 setOperationAction(Ops: ISD::BITCAST, VTs: P64VecVTs, Action: Custom);
675 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VTs: P64VecVTs, Action: Legal);
676 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VTs: {MVT::v4i16, MVT::v8i8},
677 Action: Custom);
678 setOperationAction(
679 Ops: {ISD::UADDSAT, ISD::SADDSAT, ISD::USUBSAT, ISD::SSUBSAT}, VTs: P64VecVTs,
680 Action: Legal);
681 setOperationAction(Ops: ISD::INTRINSIC_WO_CHAIN, VTs: P64VecVTs, Action: Legal);
682 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i64, Action: Custom);
683 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs: P64VecVTs, Action: Legal);
684 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX},
685 VTs: P64VecVTs, Action: Legal);
686 setOperationAction(Ops: {ISD::ABS, ISD::ABDS, ISD::ABDU},
687 VTs: {MVT::v4i16, MVT::v8i8}, Action: Legal);
688 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VTs: P64VecVTs, Action: Custom);
689 setOperationAction(Ops: ISD::SSHLSAT, VTs: {MVT::v2i32, MVT::v4i16}, Action: Custom);
690 setOperationAction(Op: ISD::BSWAP, VT: MVT::v4i16, Action: Legal);
691 setOperationAction(Ops: ISD::BITREVERSE, VTs: {MVT::v4i16, MVT::v8i8}, Action: Legal);
692 setOperationAction(Ops: ISD::VECTOR_SHUFFLE, VTs: P64VecVTs, Action: Custom);
693 setOperationAction(Ops: ISD::VECTOR_REVERSE, VTs: P64VecVTs, Action: Custom);
694 setOperationAction(Ops: ISD::SPLAT_VECTOR, VTs: P64VecVTs, Action: Legal);
695 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs: P64VecVTs, Action: Legal);
696 setOperationAction(Op: ISD::EXTRACT_VECTOR_ELT, VT: MVT::v2i32, Action: Legal);
697 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT},
698 VTs: {MVT::v4i16, MVT::v8i8}, Action: Custom);
699 setOperationAction(Ops: ISD::CONCAT_VECTORS, VTs: {MVT::v4i16, MVT::v8i8}, Action: Legal);
700 setOperationAction(Ops: ISD::EXTRACT_SUBVECTOR, VTs: {MVT::v2i16, MVT::v4i8},
701 Action: Legal);
702 setOperationAction(Ops: {ISD::SELECT, ISD::VSELECT}, VTs: {MVT::v4i16, MVT::v8i8},
703 Action: Custom);
704 setOperationAction(Ops: {ISD::MUL, ISD::MULHS, ISD::MULHU},
705 VTs: {MVT::v4i16, MVT::v8i8}, Action: Custom);
706 setOperationAction(Ops: ISD::MUL, VTs: P32VecVTs, Action: Custom);
707 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT: MVT::v4i8, Action: Custom);
708 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT: MVT::v2i16, Action: Legal);
709 setOperationAction(Ops: {ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
710 VTs: {MVT::v4i16, MVT::v2i32}, Action: Legal);
711 setOperationAction(Ops: ISD::TRUNCATE, VTs: {MVT::v4i8, MVT::v2i16}, Action: Legal);
712 setOperationAction(Ops: ISD::SETCC, VTs: P64VecVTs, Action: Legal);
713 setCondCodeAction(
714 CCs: {ISD::SETGE, ISD::SETUGT, ISD::SETUGE, ISD::SETULE, ISD::SETLE},
715 VTs: P64VecVTs, Action: Expand);
716 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETGT}, VTs: P64VecVTs, Action: Custom);
717 // Operation legalization queries the action using the result type.
718 setOperationAction(Ops: ISD::INSERT_SUBVECTOR, VTs: {MVT::v4i16, MVT::v8i8},
719 Action: Custom);
720 } else {
721 setOperationAction(Ops: {ISD::MUL, ISD::MULHS, ISD::MULHU}, VTs: P64VecVTs, Action: Legal);
722 setOperationAction(Ops: ISD::ZERO_EXTEND_VECTOR_INREG,
723 VTs: {MVT::v4i16, MVT::v2i32}, Action: Legal);
724 setOperationAction(Ops: ISD::ANY_EXTEND_VECTOR_INREG, VTs: {MVT::v4i16, MVT::v2i32},
725 Action: Custom);
726 // LegalizeVectorOps uses result VT, LegalizeDAG uses ExtVT.
727 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG,
728 VTs: {MVT::v2i16, MVT::v4i8, MVT::v2i32, MVT::v4i16},
729 Action: Legal);
730 }
731 }
732
733 if (Subtarget.hasStdExtZfbfmin()) {
734 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
735 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Expand);
736 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::bf16, Action: Expand);
737 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Custom);
738 setOperationAction(Op: ISD::BR_CC, VT: MVT::bf16, Action: Expand);
739 setOperationAction(Ops: ZfhminZfbfminPromoteOps, VT: MVT::bf16, Action: Promote);
740 setOperationAction(Op: ISD::FREM, VT: MVT::bf16, Action: Promote);
741 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Custom);
742 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Custom);
743 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Custom);
744 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: XLenVT, Action: Custom);
745 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: XLenVT, Action: Custom);
746 }
747
748 if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
749 if (Subtarget.hasStdExtZfhOrZhinx()) {
750 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f16, Action: Legal);
751 setOperationAction(Ops: FPRndMode, VT: MVT::f16,
752 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
753 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f16, Action: Custom);
754 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16,
755 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
756 if (Subtarget.hasStdExtZfa())
757 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Custom);
758 } else {
759 setOperationAction(Ops: ZfhminZfbfminPromoteOps, VT: MVT::f16, Action: Promote);
760 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16, Action: Promote);
761 for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
762 ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
763 ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
764 setOperationAction(Op, VT: MVT::f16, Action: Custom);
765 setOperationAction(Op: ISD::FABS, VT: MVT::f16, Action: Custom);
766 setOperationAction(Op: ISD::FNEG, VT: MVT::f16, Action: Custom);
767 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Custom);
768 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: XLenVT, Action: Custom);
769 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: XLenVT, Action: Custom);
770 }
771
772 if (!Subtarget.hasStdExtD()) {
773 // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
774 // instruction runs into double rounding issues, so this is wrong.
775 // Normally we'd use an f64 fma, but without the D extension the f64 type
776 // is not legal. This should probably be a libcall.
777 AddPromotedToType(Opc: ISD::FMA, OrigVT: MVT::f16, DestVT: MVT::f32);
778 AddPromotedToType(Opc: ISD::STRICT_FMA, OrigVT: MVT::f16, DestVT: MVT::f32);
779 }
780
781 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
782
783 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Legal);
784 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
785 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f16, Action: Expand);
786 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f16, Action: Expand);
787 setOperationAction(Op: ISD::SELECT, VT: MVT::f16, Action: Custom);
788 setOperationAction(Op: ISD::BR_CC, VT: MVT::f16, Action: Expand);
789
790 setOperationAction(
791 Op: ISD::FNEARBYINT, VT: MVT::f16,
792 Action: Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
793 setOperationAction(Ops: {ISD::FREM, ISD::FPOW, ISD::FPOWI,
794 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
795 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
796 ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
797 VT: MVT::f16, Action: Promote);
798
799 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
800 // complete support for all operations in LegalizeDAG.
801 setOperationAction(Ops: {ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
802 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
803 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
804 ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
805 VT: MVT::f16, Action: Promote);
806
807 // We need to custom promote this.
808 if (Subtarget.is64Bit())
809 setOperationAction(Op: ISD::FPOWI, VT: MVT::i32, Action: Custom);
810 }
811
812 if (Subtarget.hasStdExtFOrZfinx()) {
813 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f32, Action: Legal);
814 setOperationAction(Ops: FPRndMode, VT: MVT::f32,
815 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
816 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f32, Action: Expand);
817 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Expand);
818 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Custom);
819 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Expand);
820 setOperationAction(Ops: FPOpToExpand, VT: MVT::f32, Action: Expand);
821 setOperationAction(Ops: FPOpToLibCall, VT: MVT::f32, Action: LibCall);
822 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
823 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
824 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
825 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
826 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Custom);
827 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Custom);
828 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32,
829 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
830 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Custom);
831 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Custom);
832 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f32, Action: Custom);
833 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f32, Action: Custom);
834
835 if (Subtarget.hasStdExtZfa()) {
836 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
837 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f32, Action: Legal);
838 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
839 } else {
840 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Custom);
841 }
842 }
843
844 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
845 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
846
847 if (Subtarget.hasStdExtDOrZdinx()) {
848 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f64, Action: Legal);
849
850 if (!Subtarget.is64Bit())
851 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
852
853 if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
854 !Subtarget.is64Bit()) {
855 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Custom);
856 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Custom);
857 }
858
859 if (Subtarget.hasStdExtZfa()) {
860 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
861 setOperationAction(Ops: FPRndMode, VT: MVT::f64, Action: Legal);
862 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f64, Action: Legal);
863 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f64, Action: Legal);
864 } else {
865 if (Subtarget.is64Bit())
866 setOperationAction(Ops: FPRndMode, VT: MVT::f64, Action: Custom);
867
868 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f64, Action: Custom);
869 }
870
871 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
872 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
873 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f64, Action: Expand);
874 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Expand);
875 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Custom);
876 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Expand);
877 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
878 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
879 setOperationAction(Ops: FPOpToExpand, VT: MVT::f64, Action: Expand);
880 setOperationAction(Ops: FPOpToLibCall, VT: MVT::f64, Action: LibCall);
881 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
882 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
883 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
884 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
885 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Custom);
886 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Custom);
887 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64,
888 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
889 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Custom);
890 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
891 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f64, Action: Custom);
892 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f64, Action: Expand);
893 }
894
895 if (Subtarget.is64Bit()) {
896 setOperationAction(Ops: {ISD::FP_TO_UINT, ISD::FP_TO_SINT,
897 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
898 VT: MVT::i32, Action: Custom);
899 setOperationAction(Op: ISD::LROUND, VT: MVT::i32, Action: Custom);
900 }
901
902 if (Subtarget.hasStdExtFOrZfinx()) {
903 setOperationAction(Ops: {ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, VT: XLenVT,
904 Action: Custom);
905
906 // f16/bf16 require custom handling.
907 setOperationAction(Ops: {ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, VT: XLenVT,
908 Action: Custom);
909 setOperationAction(Ops: {ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, VT: XLenVT,
910 Action: Custom);
911
912 setOperationAction(Op: ISD::GET_ROUNDING, VT: XLenVT, Action: Custom);
913 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
914 setOperationAction(Op: ISD::GET_FPENV, VT: XLenVT, Action: Custom);
915 setOperationAction(Op: ISD::SET_FPENV, VT: XLenVT, Action: Custom);
916 setOperationAction(Op: ISD::RESET_FPENV, VT: MVT::Other, Action: Custom);
917 setOperationAction(Op: ISD::GET_FPMODE, VT: XLenVT, Action: Custom);
918 setOperationAction(Op: ISD::SET_FPMODE, VT: XLenVT, Action: Custom);
919 setOperationAction(Op: ISD::RESET_FPMODE, VT: MVT::Other, Action: Custom);
920 }
921
922 setOperationAction(Ops: {ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
923 ISD::JumpTable},
924 VT: XLenVT, Action: Custom);
925
926 setOperationAction(Op: ISD::GlobalTLSAddress, VT: XLenVT, Action: Custom);
927
928 if (Subtarget.is64Bit())
929 setOperationAction(Op: ISD::Constant, VT: MVT::i64, Action: Custom);
930
931 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
932 // Unfortunately this can't be determined just from the ISA naming string.
933 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64,
934 Action: Subtarget.is64Bit() ? Legal : Custom);
935 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64,
936 Action: Subtarget.is64Bit() ? Legal : Custom);
937
938 if (Subtarget.is64Bit()) {
939 setOperationAction(Op: ISD::INIT_TRAMPOLINE, VT: MVT::Other, Action: Custom);
940 setOperationAction(Op: ISD::ADJUST_TRAMPOLINE, VT: MVT::Other, Action: Custom);
941 }
942
943 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Legal);
944 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
945 if (Subtarget.is64Bit())
946 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i32, Action: Custom);
947
948 if (Subtarget.hasVendorXMIPSCBOP())
949 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
950 else
951 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Legal);
952
953 if (Subtarget.hasStdExtZalrsc()) {
954 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
955 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
956 setMinCmpXchgSizeInBits(8);
957 else
958 setMinCmpXchgSizeInBits(32);
959 } else if (Subtarget.hasForcedAtomics()) {
960 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
961 } else {
962 setMaxAtomicSizeInBitsSupported(0);
963 }
964
965 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
966
967 setBooleanContents(ZeroOrOneBooleanContent);
968
969 if (getTargetMachine().getTargetTriple().isOSLinux()) {
970 // Custom lowering of llvm.clear_cache.
971 setOperationAction(Op: ISD::CLEAR_CACHE, VT: MVT::Other, Action: Custom);
972 }
973
974 if (Subtarget.hasVInstructions()) {
975 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
976
977 setOperationAction(Op: ISD::VSCALE, VT: XLenVT, Action: Custom);
978
979 // RVV intrinsics may have illegal operands.
980 // We also need to custom legalize vmv.x.s.
981 setOperationAction(Ops: {ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
982 ISD::INTRINSIC_VOID},
983 VTs: {MVT::i8, MVT::i16}, Action: Custom);
984 if (Subtarget.is64Bit())
985 setOperationAction(Ops: {ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
986 VT: MVT::i32, Action: Custom);
987 else
988 setOperationAction(Ops: {ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
989 VT: MVT::i64, Action: Custom);
990
991 setOperationAction(Ops: {ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
992 VT: MVT::Other, Action: Custom);
993
994 static const unsigned IntegerVPOps[] = {
995 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
996 ISD::VP_UREM, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
997 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
998 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
999 ISD::VP_MERGE,
1000 ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
1001 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON};
1002
1003 static const unsigned FloatingPointVPOps[] = {
1004 ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
1005 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
1006 ISD::VP_REDUCE_FMINIMUM, ISD::VP_REDUCE_FMAXIMUM};
1007
1008 static const unsigned IntegerVecReduceOps[] = {
1009 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
1010 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
1011 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
1012
1013 static const unsigned FloatingPointVecReduceOps[] = {
1014 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
1015 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
1016
1017 static const unsigned FloatingPointLibCallOps[] = {
1018 ISD::FREM, ISD::FPOW, ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
1019 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
1020
1021 if (!Subtarget.is64Bit()) {
1022 // We must custom-lower certain vXi64 operations on RV32 due to the vector
1023 // element type being illegal.
1024 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1025 VT: MVT::i64, Action: Custom);
1026
1027 setOperationAction(Ops: IntegerVecReduceOps, VT: MVT::i64, Action: Custom);
1028
1029 setOperationAction(Ops: {ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
1030 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
1031 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
1032 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
1033 VT: MVT::i64, Action: Custom);
1034 }
1035
1036 for (MVT VT : BoolVecVTs) {
1037 if (!isTypeLegal(VT))
1038 continue;
1039
1040 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Custom);
1041
1042 // Mask VTs are custom-expanded into a series of standard nodes
1043 setOperationAction(Ops: {ISD::TRUNCATE, ISD::CONCAT_VECTORS,
1044 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1045 ISD::SCALAR_TO_VECTOR},
1046 VT, Action: Custom);
1047
1048 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1049 Action: Custom);
1050
1051 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1052 setOperationAction(Ops: {ISD::SELECT_CC, ISD::VSELECT}, VT,
1053 Action: Expand);
1054 setOperationAction(Op: ISD::VP_MERGE, VT, Action: Custom);
1055
1056 setOperationAction(Ops: {ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
1057 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_POISON},
1058 VT, Action: Custom);
1059
1060 setOperationAction(
1061 Ops: {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1062 Action: Custom);
1063
1064 setOperationAction(
1065 Ops: {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1066 Action: Custom);
1067
1068 // RVV has native int->float & float->int conversions where the
1069 // element type sizes are within one power-of-two of each other. Any
1070 // wider distances between type sizes have to be lowered as sequences
1071 // which progressively narrow the gap in stages.
1072 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1073 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1074 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1075 ISD::STRICT_FP_TO_UINT},
1076 VT, Action: Custom);
1077 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1078 Action: Custom);
1079
1080 // Expand all extending loads to types larger than this, and truncating
1081 // stores from types larger than this.
1082 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1083 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1084 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1085 MemVT: OtherVT, Action: Expand);
1086 }
1087
1088 setOperationAction(Op: ISD::VECTOR_DEINTERLEAVE, VT, Action: Custom);
1089 setOperationAction(Op: ISD::VECTOR_INTERLEAVE, VT, Action: Custom);
1090
1091 setOperationAction(Op: ISD::VECTOR_REVERSE, VT, Action: Custom);
1092
1093 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1094 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1095
1096 setOperationPromotedToType(
1097 Ops: {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, OrigVT: VT,
1098 DestVT: MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount()));
1099 }
1100
1101 for (MVT VT : IntVecVTs) {
1102 if (!isTypeLegal(VT))
1103 continue;
1104
1105 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1106 setOperationAction(Op: ISD::SPLAT_VECTOR_PARTS, VT, Action: Custom);
1107
1108 // Vectors implement MULHS/MULHU.
1109 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
1110
1111 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1112 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
1113 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT, Action: Expand);
1114
1115 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
1116 Action: Legal);
1117
1118 if (Subtarget.hasStdExtZvabd()) {
1119 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
1120 // Only SEW=8/16 are supported in Zvabd.
1121 if (VT.getVectorElementType() == MVT::i8 ||
1122 VT.getVectorElementType() == MVT::i16)
1123 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Legal);
1124 else
1125 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1126 } else
1127 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1128
1129 // Custom-lower extensions and truncations from/to mask types.
1130 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
1131 VT, Action: Custom);
1132
1133 // RVV has native int->float & float->int conversions where the
1134 // element type sizes are within one power-of-two of each other. Any
1135 // wider distances between type sizes have to be lowered as sequences
1136 // which progressively narrow the gap in stages.
1137 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1138 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1139 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1140 ISD::STRICT_FP_TO_UINT},
1141 VT, Action: Custom);
1142 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1143 Action: Custom);
1144 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1145 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1146 ISD::SSUBSAT, ISD::USUBSAT},
1147 VT, Action: Legal);
1148
1149 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
1150 // nodes which truncate by one power of two at a time.
1151 setOperationAction(
1152 Ops: {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1153 Action: Custom);
1154
1155 // Custom-lower insert/extract operations to simplify patterns.
1156 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1157 Action: Custom);
1158
1159 // Custom-lower reduction operations to set up the corresponding custom
1160 // nodes' operands.
1161 setOperationAction(Ops: IntegerVecReduceOps, VT, Action: Custom);
1162
1163 setOperationAction(Ops: IntegerVPOps, VT, Action: Custom);
1164
1165 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1166
1167 setOperationAction(Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1168 VT, Action: Custom);
1169
1170 setOperationAction(
1171 Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1172 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1173 VT, Action: Custom);
1174 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1175
1176 setOperationAction(Ops: {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1177 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1178 VT, Action: Custom);
1179
1180 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1181 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1182
1183 setOperationAction(Ops: {ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Action: Custom);
1184
1185 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1186 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1187 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1188 MemVT: OtherVT, Action: Expand);
1189 }
1190
1191 setOperationAction(Op: ISD::VECTOR_DEINTERLEAVE, VT, Action: Custom);
1192 setOperationAction(Op: ISD::VECTOR_INTERLEAVE, VT, Action: Custom);
1193
1194 setOperationAction(Ops: {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
1195 VT, Action: Custom);
1196
1197 if (Subtarget.hasStdExtZvkb()) {
1198 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
1199 } else {
1200 setOperationAction(Op: ISD::BSWAP, VT, Action: Expand);
1201 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT, Action: Expand);
1202 }
1203
1204 if (Subtarget.hasStdExtZvbb()) {
1205 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Legal);
1206 } else {
1207 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Expand);
1208 setOperationAction(Ops: {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Action: Expand);
1209
1210 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1211 // range of f32.
1212 EVT FloatVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1213 if (isTypeLegal(VT: FloatVT)) {
1214 setOperationAction(
1215 Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1216 Action: Custom);
1217 }
1218 }
1219
1220 if (VT.getVectorElementType() == MVT::i64) {
1221 if (Subtarget.hasStdExtZvbc())
1222 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Legal);
1223 } else {
1224 if (Subtarget.hasStdExtZvbc32e()) {
1225 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Legal);
1226 } else if (Subtarget.hasStdExtZvbc()) {
1227 // Promote to i64 if the lmul is small enough.
1228 // FIXME: Split if necessary to widen.
1229 // FIXME: Promote clmulh directly without legalizing to clmul first.
1230 MVT I64VecVT = MVT::getVectorVT(VT: MVT::i64, EC: VT.getVectorElementCount());
1231 if (isTypeLegal(VT: I64VecVT))
1232 setOperationAction(Op: ISD::CLMUL, VT, Action: Custom);
1233 }
1234 }
1235
1236 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1237 setOperationAction(Ops: {ISD::MASKED_UDIV, ISD::MASKED_SDIV, ISD::MASKED_UREM,
1238 ISD::MASKED_SREM},
1239 VT, Action: Legal);
1240 }
1241
1242 for (MVT VT : VecTupleVTs) {
1243 if (!isTypeLegal(VT))
1244 continue;
1245
1246 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1247 }
1248
1249 // Expand various CCs to best match the RVV ISA, which natively supports UNE
1250 // but no other unordered comparisons, and supports all ordered comparisons
1251 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
1252 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
1253 // and we pattern-match those back to the "original", swapping operands once
1254 // more. This way we catch both operations and both "vf" and "fv" forms with
1255 // fewer patterns.
1256 static const ISD::CondCode VFPCCToExpand[] = {
1257 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
1258 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
1259 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
1260 };
1261
1262 // TODO: support more ops.
1263 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
1264 ISD::FMINNUM,
1265 ISD::FMAXNUM,
1266 ISD::FMINIMUMNUM,
1267 ISD::FMAXIMUMNUM,
1268 ISD::FADD,
1269 ISD::FSUB,
1270 ISD::FMUL,
1271 ISD::FMA,
1272 ISD::FDIV,
1273 ISD::FSQRT,
1274 ISD::FCEIL,
1275 ISD::FTRUNC,
1276 ISD::FFLOOR,
1277 ISD::FROUND,
1278 ISD::FROUNDEVEN,
1279 ISD::FRINT,
1280 ISD::FNEARBYINT,
1281 ISD::IS_FPCLASS,
1282 ISD::SETCC,
1283 ISD::FMAXIMUM,
1284 ISD::FMINIMUM,
1285 ISD::STRICT_FADD,
1286 ISD::STRICT_FSUB,
1287 ISD::STRICT_FMUL,
1288 ISD::STRICT_FDIV,
1289 ISD::STRICT_FSQRT,
1290 ISD::STRICT_FMA,
1291 ISD::VECREDUCE_FADD,
1292 ISD::VECREDUCE_FMIN,
1293 ISD::VECREDUCE_FMAX,
1294 ISD::VECREDUCE_FMINIMUM,
1295 ISD::VECREDUCE_FMAXIMUM,
1296 ISD::FCANONICALIZE};
1297
1298 // TODO: Make more of these ops legal.
1299 static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
1300 ISD::FSQRT,
1301 ISD::FCEIL,
1302 ISD::FTRUNC,
1303 ISD::FFLOOR,
1304 ISD::FROUND,
1305 ISD::FROUNDEVEN,
1306 ISD::FRINT,
1307 ISD::FNEARBYINT,
1308 ISD::STRICT_FDIV,
1309 ISD::STRICT_FSQRT,
1310 ISD::VECREDUCE_FADD,
1311 ISD::VECREDUCE_FMIN,
1312 ISD::VECREDUCE_FMAX,
1313 ISD::VECREDUCE_FMINIMUM,
1314 ISD::VECREDUCE_FMAXIMUM};
1315
1316 // TODO: support more vp ops.
1317 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
1318 ISD::VP_REDUCE_FMIN,
1319 ISD::VP_REDUCE_FMAX,
1320 ISD::VP_REDUCE_FMINIMUM,
1321 ISD::VP_REDUCE_FMAXIMUM};
1322
1323 // Sets common operation actions on RVV floating-point vector types.
1324 const auto SetCommonVFPActions = [&](MVT VT) {
1325 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1326 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
1327 // sizes are within one power-of-two of each other. Therefore conversions
1328 // between vXf16 and vXf64 must be lowered as sequences which convert via
1329 // vXf32.
1330 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1331 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1332 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1333 // Custom-lower insert/extract operations to simplify patterns.
1334 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1335 Action: Custom);
1336 // Expand various condition codes (explained above).
1337 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1338
1339 setOperationAction(
1340 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1341 Action: Legal);
1342 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Action: Custom);
1343
1344 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1345 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
1346 ISD::IS_FPCLASS},
1347 VT, Action: Custom);
1348
1349 setOperationAction(Ops: FloatingPointVecReduceOps, VT, Action: Custom);
1350
1351 // Expand FP operations that need libcalls.
1352 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1353
1354 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Expand);
1355
1356 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Legal);
1357
1358 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1359
1360 setOperationAction(Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1361 VT, Action: Custom);
1362
1363 setOperationAction(
1364 Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1365 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1366 VT, Action: Custom);
1367 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1368
1369 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1370 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1371
1372 setOperationAction(Ops: {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1373 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1374 VT, Action: Custom);
1375
1376 setOperationAction(Op: ISD::VECTOR_DEINTERLEAVE, VT, Action: Custom);
1377 setOperationAction(Op: ISD::VECTOR_INTERLEAVE, VT, Action: Custom);
1378
1379 setOperationAction(Ops: {ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1380 ISD::VECTOR_SPLICE_RIGHT},
1381 VT, Action: Custom);
1382 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1383 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1384
1385 setOperationAction(Ops: FloatingPointVPOps, VT, Action: Custom);
1386
1387 setOperationAction(Ops: {ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1388 Action: Custom);
1389 setOperationAction(Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1390 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
1391 VT, Action: Legal);
1392 setOperationAction(Ops: {ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
1393 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
1394 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1395 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1396 VT, Action: Custom);
1397
1398 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1399 };
1400
1401 // Sets common extload/truncstore actions on RVV floating-point vector
1402 // types.
1403 const auto SetCommonVFPExtLoadTruncStoreActions =
1404 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
1405 for (auto SmallVT : SmallerVTs) {
1406 setTruncStoreAction(ValVT: VT, MemVT: SmallVT, Action: Expand);
1407 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: SmallVT, Action: Expand);
1408 }
1409 };
1410
1411 // Sets common actions for f16 and bf16 for when there's only
1412 // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
1413 const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
1414 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1415 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1416 Action: Custom);
1417 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1418 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1419 setOperationAction(Ops: {ISD::VP_MERGE, ISD::SELECT}, VT,
1420 Action: Custom);
1421 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1422 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
1423 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1424 ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
1425 ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1426 ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
1427 VT, Action: Custom);
1428 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1429 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1430 MVT EltVT = VT.getVectorElementType();
1431 if (isTypeLegal(VT: EltVT))
1432 setOperationAction(Ops: {ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
1433 VT, Action: Custom);
1434 else
1435 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: EltVT, Action: Custom);
1436 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1437 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1438 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1439 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1440 ISD::VP_SCATTER},
1441 VT, Action: Custom);
1442 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1443
1444 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
1445 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
1446 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Expand);
1447
1448 // Expand FP operations that need libcalls.
1449 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1450
1451 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Expand);
1452
1453 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1454 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1455 setOperationAction(Ops: ZvfhminZvfbfminPromoteOps, VT, Action: Custom);
1456 setOperationAction(Ops: ZvfhminZvfbfminPromoteVPOps, VT, Action: Custom);
1457 } else {
1458 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1459 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1460 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1461 }
1462 };
1463
1464 // Sets common actions for zvfbfa, some of instructions are supported
1465 // natively so that we don't need to promote them.
1466 const auto SetZvfbfaActions = [&](MVT VT) {
1467 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1468 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1469 Action: Custom);
1470 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1471 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1472 setOperationAction(Ops: {ISD::VP_MERGE, ISD::SELECT}, VT,
1473 Action: Custom);
1474 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1475 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1476 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1477 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE,
1478 ISD::VECTOR_INTERLEAVE, ISD::VECTOR_REVERSE,
1479 ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
1480 ISD::VECTOR_COMPRESS},
1481 VT, Action: Custom);
1482 setOperationAction(
1483 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1484 Action: Legal);
1485 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Action: Custom);
1486 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
1487 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1488 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1489
1490 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Legal);
1491 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1492 setOperationAction(Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1493 ISD::STRICT_FMA},
1494 VT, Action: Legal);
1495 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1496
1497 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1498 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1499 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1500 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1501 ISD::VP_SCATTER},
1502 VT, Action: Custom);
1503 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1504
1505 // Expand FP operations that need libcalls.
1506 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1507
1508 setOperationAction(Op: ISD::FCANONICALIZE, VT, Action: Expand);
1509
1510 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1511 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1512 setOperationAction(Ops: ZvfbfaPromoteOps, VT, Action: Custom);
1513 setOperationAction(Ops: ZvfhminZvfbfminPromoteVPOps, VT, Action: Custom);
1514 } else {
1515 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1516 setOperationPromotedToType(Ops: ZvfbfaPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1517 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1518 }
1519 };
1520
1521 if (Subtarget.hasVInstructionsF16()) {
1522 for (MVT VT : F16VecVTs) {
1523 if (!isTypeLegal(VT))
1524 continue;
1525 SetCommonVFPActions(VT);
1526 }
1527 } else if (Subtarget.hasVInstructionsF16Minimal()) {
1528 for (MVT VT : F16VecVTs) {
1529 if (!isTypeLegal(VT))
1530 continue;
1531 SetCommonPromoteToF32Actions(VT);
1532 }
1533 }
1534
1535 if (Subtarget.hasVInstructionsBF16()) {
1536 for (MVT VT : BF16VecVTs) {
1537 if (!isTypeLegal(VT))
1538 continue;
1539 SetZvfbfaActions(VT);
1540 }
1541 } else if (Subtarget.hasVInstructionsBF16Minimal()) {
1542 for (MVT VT : BF16VecVTs) {
1543 if (!isTypeLegal(VT))
1544 continue;
1545 SetCommonPromoteToF32Actions(VT);
1546 }
1547 }
1548
1549 if (Subtarget.hasVInstructionsF32()) {
1550 for (MVT VT : F32VecVTs) {
1551 if (!isTypeLegal(VT))
1552 continue;
1553 SetCommonVFPActions(VT);
1554 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1555 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1556 }
1557 }
1558
1559 if (Subtarget.hasVInstructionsF64()) {
1560 for (MVT VT : F64VecVTs) {
1561 if (!isTypeLegal(VT))
1562 continue;
1563 SetCommonVFPActions(VT);
1564 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1565 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1566 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1567 }
1568 }
1569
1570 if (Subtarget.useRVVForFixedLengthVectors()) {
1571 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1572 if (!useRVVForFixedLengthVectorVT(VT))
1573 continue;
1574
1575 // By default everything must be expanded.
1576 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1577 setOperationAction(Op, VT, Action: Expand);
1578 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
1579 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1580 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1581 MemVT: OtherVT, Action: Expand);
1582 }
1583
1584 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1585 // expansion to a build_vector of 0s.
1586 setOperationAction(Op: ISD::UNDEF, VT, Action: Custom);
1587
1588 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
1589 setOperationAction(Ops: {ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1590 Action: Custom);
1591
1592 setOperationAction(
1593 Ops: {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
1594 Action: Custom);
1595
1596 setOperationAction(Ops: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1597 VT, Action: Custom);
1598
1599 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1600 VT, Action: Custom);
1601
1602 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
1603
1604 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1605
1606 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1607
1608 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1609
1610 setOperationAction(
1611 Ops: {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1612 Action: Custom);
1613
1614 setOperationAction(Op: ISD::BITCAST, VT, Action: Custom);
1615
1616 setOperationAction(
1617 Ops: {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1618 Action: Custom);
1619
1620 setOperationAction(
1621 Ops: {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1622 Action: Custom);
1623
1624 setOperationAction(
1625 Ops: {
1626 ISD::SINT_TO_FP,
1627 ISD::UINT_TO_FP,
1628 ISD::FP_TO_SINT,
1629 ISD::FP_TO_UINT,
1630 ISD::STRICT_SINT_TO_FP,
1631 ISD::STRICT_UINT_TO_FP,
1632 ISD::STRICT_FP_TO_SINT,
1633 ISD::STRICT_FP_TO_UINT,
1634 },
1635 VT, Action: Custom);
1636 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1637 Action: Custom);
1638
1639 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
1640
1641 // Operations below are different for between masks and other vectors.
1642 if (VT.getVectorElementType() == MVT::i1) {
1643 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VT, Action: Custom);
1644
1645 setOperationAction(Op: ISD::VP_MERGE, VT, Action: Custom);
1646
1647 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1648 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1649
1650 setOperationAction(Ops: {ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
1651 Action: Custom);
1652 continue;
1653 }
1654
1655 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
1656 // it before type legalization for i64 vectors on RV32. It will then be
1657 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
1658 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
1659 // improvements first.
1660 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1661 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1662 setOperationAction(Op: ISD::SPLAT_VECTOR_PARTS, VT, Action: Custom);
1663
1664 // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
1665 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::i64, Action: Custom);
1666 }
1667
1668 setOperationAction(
1669 Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Action: Custom);
1670
1671 setOperationAction(Ops: {ISD::VP_LOAD, ISD::VP_STORE,
1672 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1673 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1674 ISD::VP_SCATTER},
1675 VT, Action: Custom);
1676 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1677
1678 setOperationAction(Ops: {ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
1679 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
1680 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
1681 VT, Action: Custom);
1682
1683 setOperationAction(
1684 Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Action: Custom);
1685
1686 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1687
1688 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1689 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1690 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Custom);
1691
1692 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1693 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1694 ISD::SSUBSAT, ISD::USUBSAT},
1695 VT, Action: Custom);
1696
1697 setOperationAction(Op: ISD::VSELECT, VT, Action: Custom);
1698
1699 setOperationAction(
1700 Ops: {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Action: Custom);
1701
1702 // Custom-lower reduction operations to set up the corresponding custom
1703 // nodes' operands.
1704 setOperationAction(Ops: {ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
1705 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1706 ISD::VECREDUCE_UMIN},
1707 VT, Action: Custom);
1708
1709 setOperationAction(Ops: IntegerVPOps, VT, Action: Custom);
1710
1711 if (Subtarget.hasStdExtZvkb())
1712 setOperationAction(Ops: {ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Action: Custom);
1713
1714 if (Subtarget.hasStdExtZvbb()) {
1715 setOperationAction(Ops: {ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_POISON,
1716 ISD::CTTZ, ISD::CTTZ_ZERO_POISON, ISD::CTPOP},
1717 VT, Action: Custom);
1718 } else {
1719 // Lower CTLZ_ZERO_POISON and CTTZ_ZERO_POISON if element of VT in the
1720 // range of f32.
1721 EVT FloatVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1722 if (isTypeLegal(VT: FloatVT))
1723 setOperationAction(
1724 Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_POISON, ISD::CTTZ_ZERO_POISON}, VT,
1725 Action: Custom);
1726 }
1727
1728 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1729 setOperationAction(Ops: {ISD::MASKED_UDIV, ISD::MASKED_SDIV,
1730 ISD::MASKED_UREM, ISD::MASKED_SREM},
1731 VT, Action: Custom);
1732 }
1733
1734 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1735 // There are no extending loads or truncating stores.
1736 for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
1737 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
1738 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
1739 }
1740
1741 if (!useRVVForFixedLengthVectorVT(VT))
1742 continue;
1743
1744 // By default everything must be expanded.
1745 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1746 setOperationAction(Op, VT, Action: Expand);
1747
1748 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1749 // expansion to a build_vector of 0s.
1750 setOperationAction(Op: ISD::UNDEF, VT, Action: Custom);
1751
1752 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1753 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1754 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1755 ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
1756 VT, Action: Custom);
1757 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1758 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1759
1760 setOperationAction(Ops: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1761 VT, Action: Custom);
1762
1763 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1764 ISD::MGATHER, ISD::MSCATTER},
1765 VT, Action: Custom);
1766 setOperationAction(Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
1767 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1768 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1769 VT, Action: Custom);
1770 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1771
1772 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1773 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1774 Action: Custom);
1775
1776 setOperationAction(Op: ISD::BITCAST, VT, Action: Custom);
1777
1778 if (VT.getVectorElementType() == MVT::f16 &&
1779 !Subtarget.hasVInstructionsF16()) {
1780 setOperationAction(
1781 Ops: {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1782 Action: Custom);
1783 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1784 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1785 if (Subtarget.hasStdExtZfhmin()) {
1786 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
1787 } else {
1788 // We need to custom legalize f16 build vectors if Zfhmin isn't
1789 // available.
1790 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
1791 }
1792 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
1793 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
1794 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Expand);
1795 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1796 // Don't promote f16 vector operations to f32 if f32 vector type is
1797 // not legal.
1798 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1799 // TODO: Support more operations.
1800 if (!isTypeLegal(VT: F32VecVT)) {
1801 setOperationAction(Ops: {ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1802 ISD::VECREDUCE_FMAXIMUM,
1803 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1804 VT, Action: Custom);
1805 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1806 continue;
1807 }
1808 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1809 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1810 continue;
1811 }
1812
1813 if (VT.getVectorElementType() == MVT::bf16) {
1814 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1815 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1816 if (Subtarget.hasStdExtZfbfmin()) {
1817 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
1818 } else {
1819 // We need to custom legalize bf16 build vectors if Zfbfmin isn't
1820 // available.
1821 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::bf16, Action: Custom);
1822 }
1823 if (Subtarget.hasVInstructionsBF16()) {
1824 setOperationAction(Ops: ZvfbfaOps, VT, Action: Custom);
1825 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1826 }
1827 setOperationAction(
1828 Ops: {ISD::VP_MERGE, ISD::VSELECT, ISD::SELECT}, VT,
1829 Action: Custom);
1830 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1831 // Don't promote bf16 vector operations to f32 if f32 vector type is
1832 // not legal.
1833 // Custom lower maximum LMUL case to split to 2 half LMUL operations.
1834 // TODO: Support more operations.
1835 if (!isTypeLegal(VT: F32VecVT)) {
1836 setOperationAction(Ops: {ISD::VECREDUCE_FMIN, ISD::VECREDUCE_FMAX,
1837 ISD::VECREDUCE_FMAXIMUM,
1838 ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FADD},
1839 VT, Action: Custom);
1840 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1841 continue;
1842 }
1843
1844 if (Subtarget.hasVInstructionsBF16())
1845 setOperationPromotedToType(Ops: ZvfbfaPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1846 else
1847 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1848 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1849 continue;
1850 }
1851
1852 setOperationAction(Ops: {ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
1853 Action: Custom);
1854
1855 setOperationAction(Ops: {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1856 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1857 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1858 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
1859 ISD::FMAXIMUM, ISD::FMINIMUM},
1860 VT, Action: Custom);
1861
1862 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1863 ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
1864 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
1865 ISD::FNEARBYINT, ISD::FCANONICALIZE},
1866 VT, Action: Custom);
1867
1868 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1869
1870 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1871 setOperationAction(Ops: {ISD::VSELECT, ISD::SELECT}, VT, Action: Custom);
1872
1873 setOperationAction(Ops: FloatingPointVecReduceOps, VT, Action: Custom);
1874
1875 setOperationAction(Ops: FloatingPointVPOps, VT, Action: Custom);
1876
1877 setOperationAction(
1878 Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1879 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
1880 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
1881 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1882 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1883 VT, Action: Custom);
1884 }
1885
1886 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1887 setOperationAction(Ops: ISD::BITCAST, VTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Custom);
1888 if (Subtarget.is64Bit())
1889 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
1890 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1891 setOperationAction(Op: ISD::BITCAST, VT: MVT::f16, Action: Custom);
1892 if (Subtarget.hasStdExtZfbfmin())
1893 setOperationAction(Op: ISD::BITCAST, VT: MVT::bf16, Action: Custom);
1894 if (Subtarget.hasStdExtFOrZfinx())
1895 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
1896 if (Subtarget.hasStdExtDOrZdinx())
1897 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Custom);
1898 }
1899 }
1900
1901 if (Subtarget.hasStdExtZaamo())
1902 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: XLenVT, Action: Expand);
1903
1904 if (Subtarget.hasForcedAtomics()) {
1905 // Force __sync libcalls to be emitted for atomic rmw/cas operations.
1906 setOperationAction(
1907 Ops: {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1908 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1909 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1910 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1911 VT: XLenVT, Action: LibCall);
1912 }
1913
1914 if (Subtarget.hasVendorXTHeadMemIdx()) {
1915 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1916 setIndexedLoadAction(IdxModes: im, VT: MVT::i8, Action: Legal);
1917 setIndexedStoreAction(IdxModes: im, VT: MVT::i8, Action: Legal);
1918 setIndexedLoadAction(IdxModes: im, VT: MVT::i16, Action: Legal);
1919 setIndexedStoreAction(IdxModes: im, VT: MVT::i16, Action: Legal);
1920 setIndexedLoadAction(IdxModes: im, VT: MVT::i32, Action: Legal);
1921 setIndexedStoreAction(IdxModes: im, VT: MVT::i32, Action: Legal);
1922
1923 if (Subtarget.is64Bit()) {
1924 setIndexedLoadAction(IdxModes: im, VT: MVT::i64, Action: Legal);
1925 setIndexedStoreAction(IdxModes: im, VT: MVT::i64, Action: Legal);
1926 }
1927 }
1928 }
1929
1930 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
1931 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i8, Action: Legal);
1932 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i16, Action: Legal);
1933 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
1934
1935 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i8, Action: Legal);
1936 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i16, Action: Legal);
1937 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
1938 }
1939
1940 // zve32x is broken for partial_reduce_umla, but let's not make it worse.
1941 if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
1942 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
1943 ISD::PARTIAL_REDUCE_UMLA,
1944 ISD::PARTIAL_REDUCE_SUMLA};
1945 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv1i32, InputVT: MVT::nxv4i8, Action: Custom);
1946 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv2i32, InputVT: MVT::nxv8i8, Action: Custom);
1947 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv4i32, InputVT: MVT::nxv16i8, Action: Custom);
1948 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv8i32, InputVT: MVT::nxv32i8, Action: Custom);
1949 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv16i32, InputVT: MVT::nxv64i8, Action: Custom);
1950
1951 if (Subtarget.useRVVForFixedLengthVectors()) {
1952 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1953 if (VT.getVectorElementType() != MVT::i32 ||
1954 !useRVVForFixedLengthVectorVT(VT))
1955 continue;
1956 ElementCount EC = VT.getVectorElementCount();
1957 MVT ArgVT = MVT::getVectorVT(VT: MVT::i8, EC: EC.multiplyCoefficientBy(RHS: 4));
1958 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: VT, InputVT: ArgVT, Action: Custom);
1959 }
1960 }
1961 }
1962
1963 // Customize load and store operation for bf16 if zfh isn't enabled.
1964 if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
1965 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Custom);
1966 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Custom);
1967 }
1968
1969 // Function alignments.
1970 const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
1971 setMinFunctionAlignment(FunctionAlignment);
1972 // Set preferred alignments.
1973 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
1974 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
1975
1976 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
1977 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
1978 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
1979 ISD::SRA});
1980 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1981
1982 if (Subtarget.hasStdExtFOrZfinx())
1983 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
1984
1985 // Allow scalar min/max to be combined with vector reductions.
1986 if (Subtarget.hasVInstructions())
1987 setTargetDAGCombine({ISD::UMAX, ISD::UMIN});
1988 if (Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
1989 setTargetDAGCombine({ISD::SMAX, ISD::SMIN});
1990
1991 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
1992 Subtarget.hasVInstructions() || Subtarget.hasStdExtP())
1993 setTargetDAGCombine(ISD::TRUNCATE);
1994
1995 if (Subtarget.hasStdExtZbkb())
1996 setTargetDAGCombine(ISD::BITREVERSE);
1997
1998 if (Subtarget.hasStdExtFOrZfinx())
1999 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
2000 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
2001 if (Subtarget.hasVInstructions())
2002 setTargetDAGCombine({ISD::FCOPYSIGN,
2003 ISD::MGATHER,
2004 ISD::MSCATTER,
2005 ISD::VP_GATHER,
2006 ISD::VP_SCATTER,
2007 ISD::SRL,
2008 ISD::SHL,
2009 ISD::STORE,
2010 ISD::SPLAT_VECTOR,
2011 ISD::BUILD_VECTOR,
2012 ISD::CONCAT_VECTORS,
2013 ISD::VP_STORE,
2014 ISD::EXPERIMENTAL_VP_REVERSE,
2015 ISD::SDIV,
2016 ISD::UDIV,
2017 ISD::SREM,
2018 ISD::UREM,
2019 ISD::INSERT_VECTOR_ELT,
2020 ISD::ABS,
2021 ISD::ABS_MIN_POISON,
2022 ISD::CTPOP,
2023 ISD::VECTOR_SHUFFLE,
2024 ISD::FMA,
2025 ISD::VSELECT,
2026 ISD::VECREDUCE_ADD,
2027 ISD::VECTOR_SPLICE_RIGHT});
2028
2029 if (Subtarget.hasVendorXTHeadMemPair())
2030 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
2031 if (Subtarget.useRVVForFixedLengthVectors() || Subtarget.hasStdExtP())
2032 setTargetDAGCombine(ISD::BITCAST);
2033
2034 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2035
2036 setMaxLargeFPConvertBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
2037
2038 setJumpIsExpensive(Subtarget.isJumpExpensive());
2039
2040 // Disable strict node mutation.
2041 IsStrictFPEnabled = true;
2042 EnableExtLdPromotion = true;
2043
2044 // Let the subtarget decide if a predictable select is more expensive than the
2045 // corresponding branch. This information is used in CGP/SelectOpt to decide
2046 // when to convert selects into branches.
2047 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
2048
2049 MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
2050 MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
2051
2052 MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
2053 MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
2054 MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
2055
2056 MaxStoresPerMemmoveOptSize =
2057 Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
2058 MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
2059
2060 MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
2061 MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
2062}
2063
2064TargetLoweringBase::LegalizeTypeAction
2065RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
2066 if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
2067 if (VT == MVT::v2i16 || VT == MVT::v4i8)
2068 return TypeWidenVector;
2069
2070 return TargetLoweringBase::getPreferredVectorAction(VT);
2071}
2072
2073EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
2074 LLVMContext &Context,
2075 EVT VT) const {
2076 if (!VT.isVector())
2077 return getPointerTy(DL);
2078 if (Subtarget.hasVInstructions() &&
2079 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
2080 return EVT::getVectorVT(Context, VT: MVT::i1, EC: VT.getVectorElementCount());
2081 return VT.changeVectorElementTypeToInteger();
2082}
2083
2084TargetLoweringBase::CondMergingParams
2085RISCVTargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
2086 const Value *LHS,
2087 const Value *RHS,
2088 const Function *F) const {
2089 if (F->hasOptSize())
2090 return TargetLowering::getJumpConditionMergingParams(Opc, LHS, RHS, F);
2091
2092 // Merging conditions eliminates a branch, so the budget we are willing to
2093 // spend eagerly computing the RHS condition should scale with how expensive a
2094 // mispredicted branch is. A branch only costs the full penalty when actually
2095 // mispredicted, so scale it down by an assumed misprediction rate (~25%).
2096 int BaseCost = Subtarget.getMispredictionPenalty() / 4;
2097 if (BrMergingBaseCostThresh.getNumOccurrences() > 1)
2098 BaseCost = BrMergingBaseCostThresh;
2099
2100 return {.BaseCost: BaseCost, .LikelyBias: BrMergingLikelyBias, .UnlikelyBias: BrMergingUnlikelyBias};
2101}
2102
2103MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
2104 return Subtarget.getXLenVT();
2105}
2106
2107// Return false if we can lower get_vector_length to a vsetvli intrinsic.
2108bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
2109 unsigned VF,
2110 bool IsScalable) const {
2111 if (!Subtarget.hasVInstructions())
2112 return true;
2113
2114 if (!IsScalable)
2115 return true;
2116
2117 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
2118 return true;
2119
2120 // Don't allow VF=1 if those types are't legal.
2121 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
2122 return true;
2123
2124 // VLEN=32 support is incomplete.
2125 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
2126 return true;
2127
2128 // The maximum VF is for the smallest element width with LMUL=8.
2129 // VF must be a power of 2.
2130 unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
2131 return VF > MaxVF || !isPowerOf2_32(Value: VF);
2132}
2133
2134bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
2135 return !Subtarget.hasVInstructions() ||
2136 VT.getVectorElementType() != MVT::i1 || !isTypeLegal(VT);
2137}
2138
2139void RISCVTargetLowering::getTgtMemIntrinsic(
2140 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2141 MachineFunction &MF, unsigned Intrinsic) const {
2142 IntrinsicInfo Info;
2143 auto &DL = I.getDataLayout();
2144
2145 auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
2146 bool IsUnitStrided, bool UsePtrVal = false) {
2147 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
2148 // We can't use ptrVal if the intrinsic can access memory before the
2149 // pointer. This means we can't use it for strided or indexed intrinsics.
2150 if (UsePtrVal)
2151 Info.ptrVal = I.getArgOperand(i: PtrOp);
2152 else
2153 Info.fallbackAddressSpace =
2154 I.getArgOperand(i: PtrOp)->getType()->getPointerAddressSpace();
2155 Type *MemTy;
2156 if (IsStore) {
2157 // Store value is the first operand.
2158 MemTy = I.getArgOperand(i: 0)->getType();
2159 } else {
2160 // Use return type. If it's segment load, return type is a struct.
2161 MemTy = I.getType();
2162 if (MemTy->isStructTy())
2163 MemTy = MemTy->getStructElementType(N: 0);
2164 }
2165 if (!IsUnitStrided)
2166 MemTy = MemTy->getScalarType();
2167
2168 Info.memVT = getValueType(DL, Ty: MemTy);
2169 if (MemTy->isTargetExtTy()) {
2170 // RISC-V vector tuple type's alignment type should be its element type.
2171 if (cast<TargetExtType>(Val: MemTy)->getName() == "riscv.vector.tuple")
2172 MemTy = Type::getIntNTy(
2173 C&: MemTy->getContext(),
2174 N: 1 << cast<ConstantInt>(Val: I.getArgOperand(i: I.arg_size() - 1))
2175 ->getZExtValue());
2176 Info.align = DL.getABITypeAlign(Ty: MemTy);
2177 } else {
2178 Info.align = Align(DL.getTypeStoreSize(Ty: MemTy->getScalarType()));
2179 }
2180 Info.size = MemoryLocation::UnknownSize;
2181 Info.flags |=
2182 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2183 Infos.push_back(Elt: Info);
2184 };
2185
2186 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
2187 Info.flags |= MachineMemOperand::MONonTemporal;
2188
2189 Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
2190 switch (Intrinsic) {
2191 default:
2192 return;
2193 case Intrinsic::riscv_masked_atomicrmw_xchg:
2194 case Intrinsic::riscv_masked_atomicrmw_add:
2195 case Intrinsic::riscv_masked_atomicrmw_sub:
2196 case Intrinsic::riscv_masked_atomicrmw_nand:
2197 case Intrinsic::riscv_masked_atomicrmw_max:
2198 case Intrinsic::riscv_masked_atomicrmw_min:
2199 case Intrinsic::riscv_masked_atomicrmw_umax:
2200 case Intrinsic::riscv_masked_atomicrmw_umin:
2201 case Intrinsic::riscv_masked_cmpxchg:
2202 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
2203 // narrow atomic operation. These will be expanded to an LR/SC loop that
2204 // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
2205 // will be used to modify the appropriate part of the 4 byte data and
2206 // preserve the rest.
2207 Info.opc = ISD::INTRINSIC_W_CHAIN;
2208 Info.memVT = MVT::i32;
2209 Info.ptrVal = I.getArgOperand(i: 0);
2210 Info.offset = 0;
2211 Info.align = Align(4);
2212 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2213 MachineMemOperand::MOVolatile;
2214 Infos.push_back(Elt: Info);
2215 return;
2216 case Intrinsic::riscv_seg2_load_mask:
2217 case Intrinsic::riscv_seg3_load_mask:
2218 case Intrinsic::riscv_seg4_load_mask:
2219 case Intrinsic::riscv_seg5_load_mask:
2220 case Intrinsic::riscv_seg6_load_mask:
2221 case Intrinsic::riscv_seg7_load_mask:
2222 case Intrinsic::riscv_seg8_load_mask:
2223 case Intrinsic::riscv_sseg2_load_mask:
2224 case Intrinsic::riscv_sseg3_load_mask:
2225 case Intrinsic::riscv_sseg4_load_mask:
2226 case Intrinsic::riscv_sseg5_load_mask:
2227 case Intrinsic::riscv_sseg6_load_mask:
2228 case Intrinsic::riscv_sseg7_load_mask:
2229 case Intrinsic::riscv_sseg8_load_mask:
2230 SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
2231 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2232 return;
2233 case Intrinsic::riscv_seg2_store_mask:
2234 case Intrinsic::riscv_seg3_store_mask:
2235 case Intrinsic::riscv_seg4_store_mask:
2236 case Intrinsic::riscv_seg5_store_mask:
2237 case Intrinsic::riscv_seg6_store_mask:
2238 case Intrinsic::riscv_seg7_store_mask:
2239 case Intrinsic::riscv_seg8_store_mask:
2240 // Operands are (vec, ..., vec, ptr, mask, vl)
2241 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2242 /*IsStore*/ true,
2243 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2244 return;
2245 case Intrinsic::riscv_sseg2_store_mask:
2246 case Intrinsic::riscv_sseg3_store_mask:
2247 case Intrinsic::riscv_sseg4_store_mask:
2248 case Intrinsic::riscv_sseg5_store_mask:
2249 case Intrinsic::riscv_sseg6_store_mask:
2250 case Intrinsic::riscv_sseg7_store_mask:
2251 case Intrinsic::riscv_sseg8_store_mask:
2252 // Operands are (vec, ..., vec, ptr, offset, mask, vl)
2253 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2254 /*IsStore*/ true,
2255 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2256 return;
2257 case Intrinsic::riscv_vlm:
2258 SetRVVLoadStoreInfo(/*PtrOp*/ 0,
2259 /*IsStore*/ false,
2260 /*IsUnitStrided*/ true,
2261 /*UsePtrVal*/ true);
2262 return;
2263 case Intrinsic::riscv_vle:
2264 case Intrinsic::riscv_vle_mask:
2265 case Intrinsic::riscv_vleff:
2266 case Intrinsic::riscv_vleff_mask:
2267 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2268 /*IsStore*/ false,
2269 /*IsUnitStrided*/ true,
2270 /*UsePtrVal*/ true);
2271 return;
2272 case Intrinsic::riscv_vsm:
2273 case Intrinsic::riscv_vse:
2274 case Intrinsic::riscv_vse_mask:
2275 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2276 /*IsStore*/ true,
2277 /*IsUnitStrided*/ true,
2278 /*UsePtrVal*/ true);
2279 return;
2280 case Intrinsic::riscv_vlse:
2281 case Intrinsic::riscv_vlse_mask:
2282 case Intrinsic::riscv_vloxei:
2283 case Intrinsic::riscv_vloxei_mask:
2284 case Intrinsic::riscv_vluxei:
2285 case Intrinsic::riscv_vluxei_mask:
2286 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2287 /*IsStore*/ false,
2288 /*IsUnitStrided*/ false);
2289 return;
2290 case Intrinsic::riscv_vsse:
2291 case Intrinsic::riscv_vsse_mask:
2292 case Intrinsic::riscv_vsoxei:
2293 case Intrinsic::riscv_vsoxei_mask:
2294 case Intrinsic::riscv_vsuxei:
2295 case Intrinsic::riscv_vsuxei_mask:
2296 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2297 /*IsStore*/ true,
2298 /*IsUnitStrided*/ false);
2299 return;
2300 case Intrinsic::riscv_vlseg2:
2301 case Intrinsic::riscv_vlseg3:
2302 case Intrinsic::riscv_vlseg4:
2303 case Intrinsic::riscv_vlseg5:
2304 case Intrinsic::riscv_vlseg6:
2305 case Intrinsic::riscv_vlseg7:
2306 case Intrinsic::riscv_vlseg8:
2307 case Intrinsic::riscv_vlseg2ff:
2308 case Intrinsic::riscv_vlseg3ff:
2309 case Intrinsic::riscv_vlseg4ff:
2310 case Intrinsic::riscv_vlseg5ff:
2311 case Intrinsic::riscv_vlseg6ff:
2312 case Intrinsic::riscv_vlseg7ff:
2313 case Intrinsic::riscv_vlseg8ff:
2314 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2315 /*IsStore*/ false,
2316 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2317 return;
2318 case Intrinsic::riscv_vlseg2_mask:
2319 case Intrinsic::riscv_vlseg3_mask:
2320 case Intrinsic::riscv_vlseg4_mask:
2321 case Intrinsic::riscv_vlseg5_mask:
2322 case Intrinsic::riscv_vlseg6_mask:
2323 case Intrinsic::riscv_vlseg7_mask:
2324 case Intrinsic::riscv_vlseg8_mask:
2325 case Intrinsic::riscv_vlseg2ff_mask:
2326 case Intrinsic::riscv_vlseg3ff_mask:
2327 case Intrinsic::riscv_vlseg4ff_mask:
2328 case Intrinsic::riscv_vlseg5ff_mask:
2329 case Intrinsic::riscv_vlseg6ff_mask:
2330 case Intrinsic::riscv_vlseg7ff_mask:
2331 case Intrinsic::riscv_vlseg8ff_mask:
2332 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2333 /*IsStore*/ false,
2334 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2335 return;
2336 case Intrinsic::riscv_vlsseg2:
2337 case Intrinsic::riscv_vlsseg3:
2338 case Intrinsic::riscv_vlsseg4:
2339 case Intrinsic::riscv_vlsseg5:
2340 case Intrinsic::riscv_vlsseg6:
2341 case Intrinsic::riscv_vlsseg7:
2342 case Intrinsic::riscv_vlsseg8:
2343 case Intrinsic::riscv_vloxseg2:
2344 case Intrinsic::riscv_vloxseg3:
2345 case Intrinsic::riscv_vloxseg4:
2346 case Intrinsic::riscv_vloxseg5:
2347 case Intrinsic::riscv_vloxseg6:
2348 case Intrinsic::riscv_vloxseg7:
2349 case Intrinsic::riscv_vloxseg8:
2350 case Intrinsic::riscv_vluxseg2:
2351 case Intrinsic::riscv_vluxseg3:
2352 case Intrinsic::riscv_vluxseg4:
2353 case Intrinsic::riscv_vluxseg5:
2354 case Intrinsic::riscv_vluxseg6:
2355 case Intrinsic::riscv_vluxseg7:
2356 case Intrinsic::riscv_vluxseg8:
2357 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2358 /*IsStore*/ false,
2359 /*IsUnitStrided*/ false);
2360 return;
2361 case Intrinsic::riscv_vlsseg2_mask:
2362 case Intrinsic::riscv_vlsseg3_mask:
2363 case Intrinsic::riscv_vlsseg4_mask:
2364 case Intrinsic::riscv_vlsseg5_mask:
2365 case Intrinsic::riscv_vlsseg6_mask:
2366 case Intrinsic::riscv_vlsseg7_mask:
2367 case Intrinsic::riscv_vlsseg8_mask:
2368 case Intrinsic::riscv_vloxseg2_mask:
2369 case Intrinsic::riscv_vloxseg3_mask:
2370 case Intrinsic::riscv_vloxseg4_mask:
2371 case Intrinsic::riscv_vloxseg5_mask:
2372 case Intrinsic::riscv_vloxseg6_mask:
2373 case Intrinsic::riscv_vloxseg7_mask:
2374 case Intrinsic::riscv_vloxseg8_mask:
2375 case Intrinsic::riscv_vluxseg2_mask:
2376 case Intrinsic::riscv_vluxseg3_mask:
2377 case Intrinsic::riscv_vluxseg4_mask:
2378 case Intrinsic::riscv_vluxseg5_mask:
2379 case Intrinsic::riscv_vluxseg6_mask:
2380 case Intrinsic::riscv_vluxseg7_mask:
2381 case Intrinsic::riscv_vluxseg8_mask:
2382 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
2383 /*IsStore*/ false,
2384 /*IsUnitStrided*/ false);
2385 return;
2386 case Intrinsic::riscv_vsseg2:
2387 case Intrinsic::riscv_vsseg3:
2388 case Intrinsic::riscv_vsseg4:
2389 case Intrinsic::riscv_vsseg5:
2390 case Intrinsic::riscv_vsseg6:
2391 case Intrinsic::riscv_vsseg7:
2392 case Intrinsic::riscv_vsseg8:
2393 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2394 /*IsStore*/ true,
2395 /*IsUnitStrided*/ false);
2396 return;
2397 case Intrinsic::riscv_vsseg2_mask:
2398 case Intrinsic::riscv_vsseg3_mask:
2399 case Intrinsic::riscv_vsseg4_mask:
2400 case Intrinsic::riscv_vsseg5_mask:
2401 case Intrinsic::riscv_vsseg6_mask:
2402 case Intrinsic::riscv_vsseg7_mask:
2403 case Intrinsic::riscv_vsseg8_mask:
2404 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2405 /*IsStore*/ true,
2406 /*IsUnitStrided*/ false);
2407 return;
2408 case Intrinsic::riscv_vssseg2:
2409 case Intrinsic::riscv_vssseg3:
2410 case Intrinsic::riscv_vssseg4:
2411 case Intrinsic::riscv_vssseg5:
2412 case Intrinsic::riscv_vssseg6:
2413 case Intrinsic::riscv_vssseg7:
2414 case Intrinsic::riscv_vssseg8:
2415 case Intrinsic::riscv_vsoxseg2:
2416 case Intrinsic::riscv_vsoxseg3:
2417 case Intrinsic::riscv_vsoxseg4:
2418 case Intrinsic::riscv_vsoxseg5:
2419 case Intrinsic::riscv_vsoxseg6:
2420 case Intrinsic::riscv_vsoxseg7:
2421 case Intrinsic::riscv_vsoxseg8:
2422 case Intrinsic::riscv_vsuxseg2:
2423 case Intrinsic::riscv_vsuxseg3:
2424 case Intrinsic::riscv_vsuxseg4:
2425 case Intrinsic::riscv_vsuxseg5:
2426 case Intrinsic::riscv_vsuxseg6:
2427 case Intrinsic::riscv_vsuxseg7:
2428 case Intrinsic::riscv_vsuxseg8:
2429 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2430 /*IsStore*/ true,
2431 /*IsUnitStrided*/ false);
2432 return;
2433 case Intrinsic::riscv_vssseg2_mask:
2434 case Intrinsic::riscv_vssseg3_mask:
2435 case Intrinsic::riscv_vssseg4_mask:
2436 case Intrinsic::riscv_vssseg5_mask:
2437 case Intrinsic::riscv_vssseg6_mask:
2438 case Intrinsic::riscv_vssseg7_mask:
2439 case Intrinsic::riscv_vssseg8_mask:
2440 case Intrinsic::riscv_vsoxseg2_mask:
2441 case Intrinsic::riscv_vsoxseg3_mask:
2442 case Intrinsic::riscv_vsoxseg4_mask:
2443 case Intrinsic::riscv_vsoxseg5_mask:
2444 case Intrinsic::riscv_vsoxseg6_mask:
2445 case Intrinsic::riscv_vsoxseg7_mask:
2446 case Intrinsic::riscv_vsoxseg8_mask:
2447 case Intrinsic::riscv_vsuxseg2_mask:
2448 case Intrinsic::riscv_vsuxseg3_mask:
2449 case Intrinsic::riscv_vsuxseg4_mask:
2450 case Intrinsic::riscv_vsuxseg5_mask:
2451 case Intrinsic::riscv_vsuxseg6_mask:
2452 case Intrinsic::riscv_vsuxseg7_mask:
2453 case Intrinsic::riscv_vsuxseg8_mask:
2454 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2455 /*IsStore*/ true,
2456 /*IsUnitStrided*/ false);
2457 return;
2458 case Intrinsic::riscv_sf_vlte8:
2459 case Intrinsic::riscv_sf_vlte16:
2460 case Intrinsic::riscv_sf_vlte32:
2461 case Intrinsic::riscv_sf_vlte64:
2462 Info.opc = ISD::INTRINSIC_VOID;
2463 Info.ptrVal = I.getArgOperand(i: 1);
2464 switch (Intrinsic) {
2465 case Intrinsic::riscv_sf_vlte8:
2466 Info.memVT = MVT::i8;
2467 Info.align = Align(1);
2468 break;
2469 case Intrinsic::riscv_sf_vlte16:
2470 Info.memVT = MVT::i16;
2471 Info.align = Align(2);
2472 break;
2473 case Intrinsic::riscv_sf_vlte32:
2474 Info.memVT = MVT::i32;
2475 Info.align = Align(4);
2476 break;
2477 case Intrinsic::riscv_sf_vlte64:
2478 Info.memVT = MVT::i64;
2479 Info.align = Align(8);
2480 break;
2481 }
2482 Info.size = MemoryLocation::UnknownSize;
2483 Info.flags |= MachineMemOperand::MOLoad;
2484 Infos.push_back(Elt: Info);
2485 return;
2486 case Intrinsic::riscv_sf_vste8:
2487 case Intrinsic::riscv_sf_vste16:
2488 case Intrinsic::riscv_sf_vste32:
2489 case Intrinsic::riscv_sf_vste64:
2490 Info.opc = ISD::INTRINSIC_VOID;
2491 Info.ptrVal = I.getArgOperand(i: 1);
2492 switch (Intrinsic) {
2493 case Intrinsic::riscv_sf_vste8:
2494 Info.memVT = MVT::i8;
2495 Info.align = Align(1);
2496 break;
2497 case Intrinsic::riscv_sf_vste16:
2498 Info.memVT = MVT::i16;
2499 Info.align = Align(2);
2500 break;
2501 case Intrinsic::riscv_sf_vste32:
2502 Info.memVT = MVT::i32;
2503 Info.align = Align(4);
2504 break;
2505 case Intrinsic::riscv_sf_vste64:
2506 Info.memVT = MVT::i64;
2507 Info.align = Align(8);
2508 break;
2509 }
2510 Info.size = MemoryLocation::UnknownSize;
2511 Info.flags |= MachineMemOperand::MOStore;
2512 Infos.push_back(Elt: Info);
2513 return;
2514 }
2515}
2516
2517bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
2518 const AddrMode &AM, Type *Ty,
2519 unsigned AS,
2520 Instruction *I) const {
2521 // No global is ever allowed as a base.
2522 if (AM.BaseGV)
2523 return false;
2524
2525 // None of our addressing modes allows a scalable offset
2526 if (AM.ScalableOffset)
2527 return false;
2528
2529 // RVV instructions only support register addressing.
2530 if (Subtarget.hasVInstructions() && isa<VectorType>(Val: Ty))
2531 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
2532
2533 // The Xqcilo extension provides load/store instructions with a 26-bit signed
2534 // offset.
2535 if (Subtarget.hasVendorXqcilo()) {
2536 if (!isInt<26>(x: AM.BaseOffs))
2537 return false;
2538 } else if (!isInt<12>(x: AM.BaseOffs)) {
2539 // Otherwise require a 12-bit signed offset.
2540 return false;
2541 }
2542
2543 switch (AM.Scale) {
2544 case 0: // "r+i" or just "i", depending on HasBaseReg.
2545 break;
2546 case 1:
2547 if (!AM.HasBaseReg) // allow "r+i".
2548 break;
2549 return false; // disallow "r+r" or "r+r+i".
2550 default:
2551 return false;
2552 }
2553
2554 return true;
2555}
2556
2557bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
2558 return isInt<12>(x: Imm);
2559}
2560
2561bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
2562 // The Xqcilia extension provides add-immediate instructions with a 26-bit
2563 // signed immediate.
2564 if (Subtarget.hasVendorXqcilia())
2565 return isInt<26>(x: Imm);
2566 return isInt<12>(x: Imm);
2567}
2568
2569// On RV32, 64-bit integers are split into their high and low parts and held
2570// in two different registers, so the trunc is free since the low register can
2571// just be used.
2572// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
2573// isTruncateFree?
2574bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
2575 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
2576 return false;
2577 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
2578 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
2579 return (SrcBits == 64 && DestBits == 32);
2580}
2581
2582bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
2583 // We consider i64->i32 free on RV64 since we have good selection of W
2584 // instructions that make promoting operations back to i64 free in many cases.
2585 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
2586 !DstVT.isInteger())
2587 return false;
2588 unsigned SrcBits = SrcVT.getSizeInBits();
2589 unsigned DestBits = DstVT.getSizeInBits();
2590 return (SrcBits == 64 && DestBits == 32);
2591}
2592
2593bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
2594 EVT SrcVT = Val.getValueType();
2595 // free truncate from vnsrl and vnsra
2596 if (Subtarget.hasVInstructions() &&
2597 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
2598 SrcVT.isVector() && VT2.isVector()) {
2599 unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
2600 unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
2601 if (SrcBits == DestBits * 2) {
2602 return true;
2603 }
2604 }
2605 return TargetLowering::isTruncateFree(Val, VT2);
2606}
2607
2608bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
2609 // Zexts are free if they can be combined with a load.
2610 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
2611 // poorly with type legalization of compares preferring sext.
2612 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
2613 EVT MemVT = LD->getMemoryVT();
2614 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
2615 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
2616 LD->getExtensionType() == ISD::ZEXTLOAD))
2617 return true;
2618 }
2619
2620 return TargetLowering::isZExtFree(Val, VT2);
2621}
2622
2623bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
2624 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
2625}
2626
2627bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
2628 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(BitWidth: 32);
2629}
2630
2631bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
2632 return Subtarget.hasCTZLike();
2633}
2634
2635bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
2636 return Subtarget.hasCLZLike();
2637}
2638
2639bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
2640 const Instruction &AndI) const {
2641 // We expect to be able to match a bit extraction instruction if the Zbs
2642 // extension is supported and the mask is a power of two. However, we
2643 // conservatively return false if the mask would fit in an ANDI instruction,
2644 // on the basis that it's possible the sinking+duplication of the AND in
2645 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
2646 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
2647 if (!Subtarget.hasBEXTILike())
2648 return false;
2649 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
2650 if (!Mask)
2651 return false;
2652 return !Mask->getValue().isSignedIntN(N: 12) && Mask->getValue().isPowerOf2();
2653}
2654
2655bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
2656 EVT VT = Y.getValueType();
2657
2658 if (VT.isVector())
2659 return false;
2660
2661 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2662 (!isa<ConstantSDNode>(Val: Y) || cast<ConstantSDNode>(Val&: Y)->isOpaque());
2663}
2664
2665bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
2666 EVT VT = Y.getValueType();
2667
2668 if (!VT.isVector())
2669 return hasAndNotCompare(Y);
2670
2671 return Subtarget.hasStdExtZvkb();
2672}
2673
2674bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2675 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
2676 if (Subtarget.hasStdExtZbs())
2677 return X.getValueType().isScalarInteger();
2678 auto *C = dyn_cast<ConstantSDNode>(Val&: Y);
2679 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
2680 if (Subtarget.hasVendorXTHeadBs())
2681 return C != nullptr;
2682 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
2683 return C && C->getAPIntValue().ule(RHS: 10);
2684}
2685
2686bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
2687 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
2688 SDValue Y) const {
2689 if (SelectOpcode != ISD::VSELECT)
2690 return false;
2691
2692 // Only enable for rvv.
2693 if (!VT.isVector() || !Subtarget.hasVInstructions())
2694 return false;
2695
2696 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
2697 return false;
2698
2699 return true;
2700}
2701
2702bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2703 Type *Ty) const {
2704 assert(Ty->isIntegerTy());
2705
2706 unsigned BitSize = Ty->getIntegerBitWidth();
2707 if (BitSize > Subtarget.getXLen())
2708 return false;
2709
2710 // Fast path, assume 32-bit immediates are cheap.
2711 int64_t Val = Imm.getSExtValue();
2712 if (isInt<32>(x: Val))
2713 return true;
2714
2715 // A constant pool entry may be more aligned than the load we're trying to
2716 // replace. If we don't support unaligned scalar mem, prefer the constant
2717 // pool.
2718 // TODO: Can the caller pass down the alignment?
2719 if (!Subtarget.enableUnalignedScalarMem())
2720 return true;
2721
2722 // Prefer to keep the load if it would require many instructions.
2723 // This uses the same threshold we use for constant pools but doesn't
2724 // check useConstantPoolForLargeInts.
2725 // TODO: Should we keep the load only when we're definitely going to emit a
2726 // constant pool?
2727
2728 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI: Subtarget);
2729 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
2730}
2731
2732bool RISCVTargetLowering::
2733 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
2734 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
2735 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
2736 SelectionDAG &DAG) const {
2737 // One interesting pattern that we'd want to form is 'bit extract':
2738 // ((1 >> Y) & 1) ==/!= 0
2739 // But we also need to be careful not to try to reverse that fold.
2740
2741 // Is this '((1 >> Y) & 1)'?
2742 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2743 return false; // Keep the 'bit extract' pattern.
2744
2745 // Will this be '((1 >> Y) & 1)' after the transform?
2746 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2747 return true; // Do form the 'bit extract' pattern.
2748
2749 // If 'X' is a constant, and we transform, then we will immediately
2750 // try to undo the fold, thus causing endless combine loop.
2751 // So only do the transform if X is not a constant. This matches the default
2752 // implementation of this function.
2753 return !XC;
2754}
2755
2756bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
2757 unsigned Opc = VecOp.getOpcode();
2758
2759 // Assume target opcodes can't be scalarized.
2760 // TODO - do we have any exceptions?
2761 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
2762 return false;
2763
2764 // If the vector op is not supported, try to convert to scalar.
2765 EVT VecVT = VecOp.getValueType();
2766 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
2767 return true;
2768
2769 // If the vector op is supported, but the scalar op is not, the transform may
2770 // not be worthwhile.
2771 // Permit a vector binary operation can be converted to scalar binary
2772 // operation which is custom lowered with illegal type.
2773 EVT ScalarVT = VecVT.getScalarType();
2774 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT) ||
2775 isOperationCustom(Op: Opc, VT: ScalarVT);
2776}
2777
2778bool RISCVTargetLowering::isOffsetFoldingLegal(
2779 const GlobalAddressSDNode *GA) const {
2780 // In order to maximise the opportunity for common subexpression elimination,
2781 // keep a separate ADD node for the global address offset instead of folding
2782 // it in the global address node. Later peephole optimisations may choose to
2783 // fold it back in when profitable.
2784 return false;
2785}
2786
2787// Returns 0-31 if the fli instruction is available for the type and this is
2788// legal FP immediate for the type. Returns -1 otherwise.
2789int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
2790 if (!Subtarget.hasStdExtZfa())
2791 return -1;
2792
2793 bool IsSupportedVT = false;
2794 if (VT == MVT::f16) {
2795 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2796 } else if (VT == MVT::f32) {
2797 IsSupportedVT = true;
2798 } else if (VT == MVT::f64) {
2799 assert(Subtarget.hasStdExtD() && "Expect D extension");
2800 IsSupportedVT = true;
2801 }
2802
2803 if (!IsSupportedVT)
2804 return -1;
2805
2806 return RISCVLoadFPImm::getLoadFPImm(FPImm: Imm);
2807}
2808
2809bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2810 bool ForCodeSize) const {
2811 bool IsLegalVT = false;
2812 if (VT == MVT::f16)
2813 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
2814 else if (VT == MVT::f32)
2815 IsLegalVT = Subtarget.hasStdExtFOrZfinx();
2816 else if (VT == MVT::f64)
2817 IsLegalVT = Subtarget.hasStdExtDOrZdinx();
2818 else if (VT == MVT::bf16)
2819 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2820
2821 if (!IsLegalVT)
2822 return false;
2823
2824 if (getLegalZfaFPImm(Imm, VT) >= 0)
2825 return true;
2826
2827 // Some constants can be produced by fli+fneg.
2828 if (Imm.isNegative() && getLegalZfaFPImm(Imm: -Imm, VT) >= 0)
2829 return true;
2830
2831 // Cannot create a 64 bit floating-point immediate value for rv32.
2832 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2833 // td can handle +0.0 or -0.0 already.
2834 // -0.0 can be created by fmv + fneg.
2835 return Imm.isZero();
2836 }
2837
2838 // Special case: fmv + fneg
2839 if (Imm.isNegZero())
2840 return true;
2841
2842 // Building an integer and then converting requires a fmv at the end of
2843 // the integer sequence. The fmv is not required for Zfinx.
2844 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2845 const int Cost =
2846 FmvCost + RISCVMatInt::getIntMatCost(Val: Imm.bitcastToAPInt(),
2847 Size: Subtarget.getXLen(), STI: Subtarget);
2848 return Cost <= FPImmCost;
2849}
2850
2851// TODO: This is very conservative.
2852bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2853 unsigned Index) const {
2854 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
2855 (ResVT == MVT::v4i8 || ResVT == MVT::v2i16))
2856 return (Index % ResVT.getVectorNumElements()) == 0;
2857
2858 if (!Subtarget.hasVInstructions())
2859 return false;
2860
2861 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2862 return false;
2863
2864 // Extracts from index 0 are just subreg extracts.
2865 if (Index == 0)
2866 return true;
2867
2868 // Only support extracting a fixed from a fixed vector for now.
2869 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
2870 return false;
2871
2872 EVT EltVT = ResVT.getVectorElementType();
2873 assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
2874
2875 // The smallest type we can slide is i8.
2876 if (EltVT == MVT::i1)
2877 return false;
2878
2879 unsigned ResElts = ResVT.getVectorNumElements();
2880 unsigned SrcElts = SrcVT.getVectorNumElements();
2881
2882 unsigned MinVLen = Subtarget.getRealMinVLen();
2883 unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
2884
2885 // If we're extracting only data from the first VLEN bits of the source
2886 // then we can always do this with an m1 vslidedown.vx. Restricting the
2887 // Index ensures we can use a vslidedown.vi.
2888 // TODO: We can generalize this when the exact VLEN is known.
2889 if (Index + ResElts <= MinVLMAX && Index < 31)
2890 return true;
2891
2892 // Convervatively only handle extracting half of a vector.
2893 // TODO: We can do arbitrary slidedowns, but for now only support extracting
2894 // the upper half of a vector until we have more test coverage.
2895 // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
2896 // a cheap extract. However, this case is important in practice for
2897 // shuffled extracts of longer vectors. How resolve?
2898 return (ResElts * 2) == SrcElts && Index == ResElts;
2899}
2900
2901MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
2902 CallingConv::ID CC,
2903 EVT VT) const {
2904 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2905 // We might still end up using a GPR but that will be decided based on ABI.
2906 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2907 !Subtarget.hasStdExtZfhminOrZhinxmin())
2908 return MVT::f32;
2909
2910 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2911}
2912
2913unsigned
2914RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
2915 std::optional<MVT> RegisterVT) const {
2916 // Pair inline assembly operand
2917 if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2918 *RegisterVT == MVT::Untyped)
2919 return 1;
2920
2921 return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
2922}
2923
2924unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
2925 CallingConv::ID CC,
2926 EVT VT) const {
2927 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2928 // We might still end up using a GPR but that will be decided based on ABI.
2929 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2930 !Subtarget.hasStdExtZfhminOrZhinxmin())
2931 return 1;
2932
2933 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2934}
2935
2936// Changes the condition code and swaps operands if necessary, so the SetCC
2937// operation matches one of the comparisons supported directly by branches
2938// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2939// with 1/-1.
2940static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2941 ISD::CondCode &CC, SelectionDAG &DAG,
2942 const RISCVSubtarget &Subtarget) {
2943 // If this is a single bit test that can't be handled by ANDI, shift the
2944 // bit to be tested to the MSB and perform a signed compare with 0.
2945 if (isIntEqualitySetCC(Code: CC) && isNullConstant(V: RHS) &&
2946 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
2947 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
2948 // XAndesPerf supports branch on test bit.
2949 !Subtarget.hasVendorXAndesPerf()) {
2950 uint64_t Mask = LHS.getConstantOperandVal(i: 1);
2951 if ((isPowerOf2_64(Value: Mask) || isMask_64(Value: Mask)) && !isInt<12>(x: Mask)) {
2952 unsigned ShAmt = 0;
2953 if (isPowerOf2_64(Value: Mask)) {
2954 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
2955 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Value: Mask);
2956 } else {
2957 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Value: Mask);
2958 }
2959
2960 LHS = LHS.getOperand(i: 0);
2961 if (ShAmt != 0)
2962 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS,
2963 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
2964 return;
2965 }
2966 }
2967
2968 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
2969 int64_t C = RHSC->getSExtValue();
2970 switch (CC) {
2971 default: break;
2972 case ISD::SETGT:
2973 // Convert X > -1 to X >= 0.
2974 if (C == -1) {
2975 RHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
2976 CC = ISD::SETGE;
2977 return;
2978 }
2979 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
2980 C != INT64_MAX && isInt<5>(x: C + 1)) {
2981 // We have a conditional move instruction for SETGE but not SETGT.
2982 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
2983 RHS = DAG.getSignedConstant(Val: C + 1, DL, VT: RHS.getValueType());
2984 CC = ISD::SETGE;
2985 return;
2986 }
2987 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(x: C + 1)) {
2988 // We have a branch immediate instruction for SETGE but not SETGT.
2989 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
2990 RHS = DAG.getSignedConstant(Val: C + 1, DL, VT: RHS.getValueType());
2991 CC = ISD::SETGE;
2992 return;
2993 }
2994 break;
2995 case ISD::SETLT:
2996 // Convert X < 1 to 0 >= X.
2997 if (C == 1) {
2998 RHS = LHS;
2999 LHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
3000 CC = ISD::SETGE;
3001 return;
3002 }
3003 break;
3004 case ISD::SETUGT:
3005 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
3006 C != INT64_MAX && isUInt<5>(x: C + 1)) {
3007 // We have a conditional move instruction for SETUGE but not SETUGT.
3008 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
3009 RHS = DAG.getConstant(Val: C + 1, DL, VT: RHS.getValueType());
3010 CC = ISD::SETUGE;
3011 return;
3012 }
3013 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(x: C + 1)) {
3014 // We have a branch immediate instruction for SETUGE but not SETUGT.
3015 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
3016 // immediate.
3017 RHS = DAG.getConstant(Val: C + 1, DL, VT: RHS.getValueType());
3018 CC = ISD::SETUGE;
3019 return;
3020 }
3021 break;
3022 }
3023 }
3024
3025 switch (CC) {
3026 default:
3027 break;
3028 case ISD::SETGT:
3029 case ISD::SETLE:
3030 case ISD::SETUGT:
3031 case ISD::SETULE:
3032 CC = ISD::getSetCCSwappedOperands(Operation: CC);
3033 std::swap(a&: LHS, b&: RHS);
3034 break;
3035 }
3036}
3037
3038RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
3039 if (VT.isRISCVVectorTuple()) {
3040 if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
3041 VT.SimpleTy <= MVT::riscv_nxv1i8x8)
3042 return RISCVVType::LMUL_F8;
3043 if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
3044 VT.SimpleTy <= MVT::riscv_nxv2i8x8)
3045 return RISCVVType::LMUL_F4;
3046 if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
3047 VT.SimpleTy <= MVT::riscv_nxv4i8x8)
3048 return RISCVVType::LMUL_F2;
3049 if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
3050 VT.SimpleTy <= MVT::riscv_nxv8i8x8)
3051 return RISCVVType::LMUL_1;
3052 if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
3053 VT.SimpleTy <= MVT::riscv_nxv16i8x4)
3054 return RISCVVType::LMUL_2;
3055 if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
3056 return RISCVVType::LMUL_4;
3057 llvm_unreachable("Invalid vector tuple type LMUL.");
3058 }
3059
3060 assert(VT.isScalableVector() && "Expecting a scalable vector type");
3061 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
3062 if (VT.getVectorElementType() == MVT::i1)
3063 KnownSize *= 8;
3064
3065 switch (KnownSize) {
3066 default:
3067 llvm_unreachable("Invalid LMUL.");
3068 case 8:
3069 return RISCVVType::LMUL_F8;
3070 case 16:
3071 return RISCVVType::LMUL_F4;
3072 case 32:
3073 return RISCVVType::LMUL_F2;
3074 case 64:
3075 return RISCVVType::LMUL_1;
3076 case 128:
3077 return RISCVVType::LMUL_2;
3078 case 256:
3079 return RISCVVType::LMUL_4;
3080 case 512:
3081 return RISCVVType::LMUL_8;
3082 }
3083}
3084
3085unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
3086 switch (LMul) {
3087 default:
3088 llvm_unreachable("Invalid LMUL.");
3089 case RISCVVType::LMUL_F8:
3090 case RISCVVType::LMUL_F4:
3091 case RISCVVType::LMUL_F2:
3092 case RISCVVType::LMUL_1:
3093 return RISCV::VRRegClassID;
3094 case RISCVVType::LMUL_2:
3095 return RISCV::VRM2RegClassID;
3096 case RISCVVType::LMUL_4:
3097 return RISCV::VRM4RegClassID;
3098 case RISCVVType::LMUL_8:
3099 return RISCV::VRM8RegClassID;
3100 }
3101}
3102
3103unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
3104 RISCVVType::VLMUL LMUL = getLMUL(VT);
3105 if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
3106 LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
3107 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
3108 "Unexpected subreg numbering");
3109 return RISCV::sub_vrm1_0 + Index;
3110 }
3111 if (LMUL == RISCVVType::LMUL_2) {
3112 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
3113 "Unexpected subreg numbering");
3114 return RISCV::sub_vrm2_0 + Index;
3115 }
3116 if (LMUL == RISCVVType::LMUL_4) {
3117 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
3118 "Unexpected subreg numbering");
3119 return RISCV::sub_vrm4_0 + Index;
3120 }
3121 llvm_unreachable("Invalid vector type.");
3122}
3123
3124unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
3125 if (VT.isRISCVVectorTuple()) {
3126 unsigned NF = VT.getRISCVVectorTupleNumFields();
3127 unsigned RegsPerField =
3128 std::max(a: 1U, b: (unsigned)VT.getSizeInBits().getKnownMinValue() /
3129 (NF * RISCV::RVVBitsPerBlock));
3130 switch (RegsPerField) {
3131 case 1:
3132 if (NF == 2)
3133 return RISCV::VRN2M1RegClassID;
3134 if (NF == 3)
3135 return RISCV::VRN3M1RegClassID;
3136 if (NF == 4)
3137 return RISCV::VRN4M1RegClassID;
3138 if (NF == 5)
3139 return RISCV::VRN5M1RegClassID;
3140 if (NF == 6)
3141 return RISCV::VRN6M1RegClassID;
3142 if (NF == 7)
3143 return RISCV::VRN7M1RegClassID;
3144 if (NF == 8)
3145 return RISCV::VRN8M1RegClassID;
3146 break;
3147 case 2:
3148 if (NF == 2)
3149 return RISCV::VRN2M2RegClassID;
3150 if (NF == 3)
3151 return RISCV::VRN3M2RegClassID;
3152 if (NF == 4)
3153 return RISCV::VRN4M2RegClassID;
3154 break;
3155 case 4:
3156 assert(NF == 2);
3157 return RISCV::VRN2M4RegClassID;
3158 default:
3159 break;
3160 }
3161 llvm_unreachable("Invalid vector tuple type RegClass.");
3162 }
3163
3164 if (VT.getVectorElementType() == MVT::i1)
3165 return RISCV::VRRegClassID;
3166 return getRegClassIDForLMUL(LMul: getLMUL(VT));
3167}
3168
3169// Attempt to decompose a subvector insert/extract between VecVT and
3170// SubVecVT via subregister indices. Returns the subregister index that
3171// can perform the subvector insert/extract with the given element index, as
3172// well as the index corresponding to any leftover subvectors that must be
3173// further inserted/extracted within the register class for SubVecVT.
3174std::pair<unsigned, unsigned>
3175RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3176 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
3177 const RISCVRegisterInfo *TRI) {
3178 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
3179 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
3180 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
3181 "Register classes not ordered");
3182 unsigned VecRegClassID = getRegClassIDForVecVT(VT: VecVT);
3183 unsigned SubRegClassID = getRegClassIDForVecVT(VT: SubVecVT);
3184
3185 // If VecVT is a vector tuple type, either it's the tuple type with same
3186 // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
3187 if (VecVT.isRISCVVectorTuple()) {
3188 if (VecRegClassID == SubRegClassID)
3189 return {RISCV::NoSubRegister, 0};
3190
3191 assert(SubVecVT.isScalableVector() &&
3192 "Only allow scalable vector subvector.");
3193 assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
3194 "Invalid vector tuple insert/extract for vector and subvector with "
3195 "different LMUL.");
3196 return {getSubregIndexByMVT(VT: VecVT, Index: InsertExtractIdx), 0};
3197 }
3198
3199 // Try to compose a subregister index that takes us from the incoming
3200 // LMUL>1 register class down to the outgoing one. At each step we half
3201 // the LMUL:
3202 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
3203 // Note that this is not guaranteed to find a subregister index, such as
3204 // when we are extracting from one VR type to another.
3205 unsigned SubRegIdx = RISCV::NoSubRegister;
3206 for (const unsigned RCID :
3207 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
3208 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
3209 VecVT = VecVT.getHalfNumVectorElementsVT();
3210 bool IsHi =
3211 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
3212 SubRegIdx = TRI->composeSubRegIndices(a: SubRegIdx,
3213 b: getSubregIndexByMVT(VT: VecVT, Index: IsHi));
3214 if (IsHi)
3215 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
3216 }
3217 return {SubRegIdx, InsertExtractIdx};
3218}
3219
3220// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
3221// stores for those types.
3222bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
3223 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
3224 (VT == MVT::i32 || VT == MVT::v2i16 || VT == MVT::v4i8))
3225 return false;
3226
3227 return !Subtarget.useRVVForFixedLengthVectors() ||
3228 VT.isFixedLengthVectorOf(EltVT: MVT::i1);
3229}
3230
3231bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
3232 if (!ScalarTy.isSimple())
3233 return false;
3234 switch (ScalarTy.getSimpleVT().SimpleTy) {
3235 case MVT::iPTR:
3236 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
3237 case MVT::i8:
3238 case MVT::i16:
3239 case MVT::i32:
3240 return Subtarget.hasVInstructions();
3241 case MVT::i64:
3242 return Subtarget.hasVInstructionsI64();
3243 case MVT::f16:
3244 return Subtarget.hasVInstructionsF16Minimal();
3245 case MVT::bf16:
3246 return Subtarget.hasVInstructionsBF16Minimal();
3247 case MVT::f32:
3248 return Subtarget.hasVInstructionsF32();
3249 case MVT::f64:
3250 return Subtarget.hasVInstructionsF64();
3251 default:
3252 return false;
3253 }
3254}
3255
3256
3257unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
3258 return NumRepeatedDivisors;
3259}
3260
3261static SDValue getVLOperand(SDValue Op) {
3262 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3263 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
3264 "Unexpected opcode");
3265 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
3266 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
3267 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
3268 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
3269 if (!II)
3270 return SDValue();
3271 return Op.getOperand(i: II->VLOperand + 1 + HasChain);
3272}
3273
3274static bool useRVVForFixedLengthVectorVT(MVT VT,
3275 const RISCVSubtarget &Subtarget) {
3276 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
3277 if (!Subtarget.useRVVForFixedLengthVectors())
3278 return false;
3279
3280 // We only support a set of vector types with a consistent maximum fixed size
3281 // across all supported vector element types to avoid legalization issues.
3282 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
3283 // fixed-length vector type we support is 1024 bytes.
3284 if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
3285 return false;
3286
3287 unsigned MinVLen = Subtarget.getRealMinVLen();
3288
3289 MVT EltVT = VT.getVectorElementType();
3290
3291 // Don't use RVV for vectors we cannot scalarize if required.
3292 switch (EltVT.SimpleTy) {
3293 // i1 is supported but has different rules.
3294 default:
3295 return false;
3296 case MVT::i1:
3297 // Masks can only use a single register.
3298 if (VT.getVectorNumElements() > MinVLen)
3299 return false;
3300 MinVLen /= 8;
3301 break;
3302 case MVT::i8:
3303 case MVT::i16:
3304 case MVT::i32:
3305 break;
3306 case MVT::i64:
3307 if (!Subtarget.hasVInstructionsI64())
3308 return false;
3309 break;
3310 case MVT::f16:
3311 if (!Subtarget.hasVInstructionsF16Minimal())
3312 return false;
3313 break;
3314 case MVT::bf16:
3315 if (!Subtarget.hasVInstructionsBF16Minimal())
3316 return false;
3317 break;
3318 case MVT::f32:
3319 if (!Subtarget.hasVInstructionsF32())
3320 return false;
3321 break;
3322 case MVT::f64:
3323 if (!Subtarget.hasVInstructionsF64())
3324 return false;
3325 break;
3326 }
3327
3328 // Reject elements larger than ELEN.
3329 if (EltVT.getSizeInBits() > Subtarget.getELen())
3330 return false;
3331
3332 unsigned LMul = divideCeil(Numerator: VT.getSizeInBits(), Denominator: MinVLen);
3333 // Don't use RVV for types that don't fit.
3334 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
3335 return false;
3336
3337 // TODO: Perhaps an artificial restriction, but worth having whilst getting
3338 // the base fixed length RVV support in place.
3339 if (!VT.isPow2VectorType())
3340 return false;
3341
3342 return true;
3343}
3344
3345bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
3346 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
3347}
3348
3349// Return the largest legal scalable vector type that matches VT's element type.
3350static MVT getContainerForFixedLengthVector(MVT VT,
3351 const RISCVSubtarget &Subtarget) {
3352 // This may be called before legal types are setup.
3353 assert(((VT.isFixedLengthVector() &&
3354 Subtarget.getTargetLowering()->isTypeLegal(VT)) ||
3355 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
3356 "Expected legal fixed length vector!");
3357
3358 unsigned MinVLen = Subtarget.getRealMinVLen();
3359 unsigned MaxELen = Subtarget.getELen();
3360
3361 MVT EltVT = VT.getVectorElementType();
3362 switch (EltVT.SimpleTy) {
3363 default:
3364 llvm_unreachable("unexpected element type for RVV container");
3365 case MVT::i1:
3366 case MVT::i8:
3367 case MVT::i16:
3368 case MVT::i32:
3369 case MVT::i64:
3370 case MVT::bf16:
3371 case MVT::f16:
3372 case MVT::f32:
3373 case MVT::f64: {
3374 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
3375 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
3376 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
3377 unsigned NumElts =
3378 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
3379 NumElts = std::max(a: NumElts, b: RISCV::RVVBitsPerBlock / MaxELen);
3380 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
3381 return MVT::getScalableVectorVT(VT: EltVT, NumElements: NumElts);
3382 }
3383 }
3384}
3385
3386MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
3387 return ::getContainerForFixedLengthVector(VT, Subtarget: getSubtarget());
3388}
3389
3390// Grow V to consume an entire RVV register.
3391static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3392 const RISCVSubtarget &Subtarget) {
3393 assert(VT.isScalableVector() &&
3394 "Expected to convert into a scalable vector!");
3395 assert(V.getValueType().isFixedLengthVector() &&
3396 "Expected a fixed length vector operand!");
3397 SDLoc DL(V);
3398 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: V, Idx: 0);
3399}
3400
3401// Shrink V so it's just big enough to maintain a VT's worth of data.
3402static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3403 const RISCVSubtarget &Subtarget) {
3404 assert(VT.isFixedLengthVector() &&
3405 "Expected to convert into a fixed length vector!");
3406 assert(V.getValueType().isScalableVector() &&
3407 "Expected a scalable vector operand!");
3408 SDLoc DL(V);
3409 return DAG.getExtractSubvector(DL, VT, Vec: V, Idx: 0);
3410}
3411
3412/// Return the type of the mask type suitable for masking the provided
3413/// vector type. This is simply an i1 element type vector of the same
3414/// (possibly scalable) length.
3415static MVT getMaskTypeFor(MVT VecVT) {
3416 assert(VecVT.isVector());
3417 ElementCount EC = VecVT.getVectorElementCount();
3418 return MVT::getVectorVT(VT: MVT::i1, EC);
3419}
3420
3421/// Creates an all ones mask suitable for masking a vector of type VecTy with
3422/// vector length VL. .
3423static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
3424 SelectionDAG &DAG) {
3425 MVT MaskVT = getMaskTypeFor(VecVT);
3426 return DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: MaskVT, Operand: VL);
3427}
3428
3429static std::pair<SDValue, SDValue>
3430getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
3431 const RISCVSubtarget &Subtarget) {
3432 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
3433 SDValue VL = DAG.getRegister(Reg: RISCV::X0, VT: Subtarget.getXLenVT());
3434 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
3435 return {Mask, VL};
3436}
3437
3438static std::pair<SDValue, SDValue>
3439getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
3440 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
3441 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3442 SDValue VL = DAG.getConstant(Val: NumElts, DL, VT: Subtarget.getXLenVT());
3443 SDValue Mask = getAllOnesMask(VecVT: ContainerVT, VL, DL, DAG);
3444 return {Mask, VL};
3445}
3446
3447// Gets the two common "VL" operands: an all-ones mask and the vector length.
3448// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
3449// the vector type that the fixed-length vector is contained in. Otherwise if
3450// VecVT is scalable, then ContainerVT should be the same as VecVT.
3451static std::pair<SDValue, SDValue>
3452getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
3453 const RISCVSubtarget &Subtarget) {
3454 if (VecVT.isFixedLengthVector())
3455 return getDefaultVLOps(NumElts: VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
3456 Subtarget);
3457 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3458 return getDefaultScalableVLOps(VecVT: ContainerVT, DL, DAG, Subtarget);
3459}
3460
3461SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
3462 SelectionDAG &DAG) const {
3463 assert(VecVT.isScalableVector() && "Expected scalable vector");
3464 return DAG.getElementCount(DL, VT: Subtarget.getXLenVT(),
3465 EC: VecVT.getVectorElementCount());
3466}
3467
3468std::pair<unsigned, unsigned>
3469RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
3470 const RISCVSubtarget &Subtarget) {
3471 assert(VecVT.isScalableVector() && "Expected scalable vector");
3472
3473 unsigned EltSize = VecVT.getScalarSizeInBits();
3474 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
3475
3476 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
3477 unsigned MaxVLMAX =
3478 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
3479
3480 unsigned VectorBitsMin = Subtarget.getRealMinVLen();
3481 unsigned MinVLMAX =
3482 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMin, EltSize, MinSize);
3483
3484 return std::make_pair(x&: MinVLMAX, y&: MaxVLMAX);
3485}
3486
3487// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
3488// of either is (currently) supported. This can get us into an infinite loop
3489// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
3490// as a ..., etc.
3491// Until either (or both) of these can reliably lower any node, reporting that
3492// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
3493// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
3494// which is not desirable.
3495bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
3496 EVT VT, unsigned DefinedValues) const {
3497 return false;
3498}
3499
3500InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
3501 // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
3502 // implementation-defined.
3503 if (!VT.isVector())
3504 return InstructionCost::getInvalid();
3505 unsigned DLenFactor = Subtarget.getDLenFactor();
3506 unsigned Cost;
3507 if (VT.isScalableVector()) {
3508 unsigned LMul;
3509 bool Fractional;
3510 std::tie(args&: LMul, args&: Fractional) =
3511 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT));
3512 if (Fractional)
3513 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
3514 else
3515 Cost = (LMul * DLenFactor);
3516 } else {
3517 Cost = divideCeil(Numerator: VT.getSizeInBits(), Denominator: Subtarget.getRealMinVLen() / DLenFactor);
3518 }
3519 return Cost;
3520}
3521
3522
3523/// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
3524/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
3525/// be by default. VRGatherCostModel reflects available options. Note that
3526/// operand (index and possibly mask) are handled separately.
3527InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
3528 auto LMULCost = getLMULCost(VT);
3529 bool Log2CostModel =
3530 Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
3531 if (Log2CostModel && LMULCost.isValid()) {
3532 unsigned Log = Log2_64(Value: LMULCost.getValue());
3533 if (Log > 0)
3534 return LMULCost * Log;
3535 }
3536 return LMULCost * LMULCost;
3537}
3538
3539/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
3540/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
3541/// or may track the vrgather.vv cost. It is implementation-dependent.
3542InstructionCost RISCVTargetLowering::getVRGatherVICost(MVT VT) const {
3543 return getLMULCost(VT);
3544}
3545
3546/// Return the cost of a vslidedown.vx or vslideup.vx instruction
3547/// for the type VT. (This does not cover the vslide1up or vslide1down
3548/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3549/// or may track the vrgather.vv cost. It is implementation-dependent.
3550InstructionCost RISCVTargetLowering::getVSlideVXCost(MVT VT) const {
3551 return getLMULCost(VT);
3552}
3553
3554/// Return the cost of a vslidedown.vi or vslideup.vi instruction
3555/// for the type VT. (This does not cover the vslide1up or vslide1down
3556/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3557/// or may track the vrgather.vv cost. It is implementation-dependent.
3558InstructionCost RISCVTargetLowering::getVSlideVICost(MVT VT) const {
3559 return getLMULCost(VT);
3560}
3561
3562static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
3563 const RISCVSubtarget &Subtarget) {
3564 // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
3565 // bf16 conversions are always promoted to f32.
3566 if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3567 Op.getValueType() == MVT::bf16) {
3568 bool IsStrict = Op->isStrictFPOpcode();
3569
3570 SDLoc DL(Op);
3571 if (IsStrict) {
3572 SDValue Val = DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {MVT::f32, MVT::Other},
3573 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
3574 return DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL,
3575 ResultTys: {Op.getValueType(), MVT::Other},
3576 Ops: {Val.getValue(R: 1), Val.getValue(R: 0),
3577 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
3578 }
3579 return DAG.getNode(
3580 Opcode: ISD::FP_ROUND, DL, VT: Op.getValueType(),
3581 N1: DAG.getNode(Opcode: Op.getOpcode(), DL, VT: MVT::f32, Operand: Op.getOperand(i: 0)),
3582 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3583 }
3584
3585 // Other operations are legal.
3586 return Op;
3587}
3588
3589static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
3590 const RISCVSubtarget &Subtarget) {
3591 // RISC-V FP-to-int conversions saturate to the destination register size, but
3592 // don't produce 0 for nan. We can use a conversion instruction and fix the
3593 // nan case with a compare and a select.
3594 SDValue Src = Op.getOperand(i: 0);
3595
3596 MVT DstVT = Op.getSimpleValueType();
3597 EVT SatVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3598
3599 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
3600
3601 if (!DstVT.isVector()) {
3602 // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
3603 // the result.
3604 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3605 Src.getValueType() == MVT::bf16) {
3606 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(Op), VT: MVT::f32, Operand: Src);
3607 }
3608
3609 unsigned Opc;
3610 if (SatVT == DstVT)
3611 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
3612 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
3613 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
3614 else
3615 return SDValue();
3616 // FIXME: Support other SatVTs by clamping before or after the conversion.
3617
3618 SDLoc DL(Op);
3619 SDValue FpToInt = DAG.getNode(
3620 Opcode: Opc, DL, VT: DstVT, N1: Src,
3621 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: Subtarget.getXLenVT()));
3622
3623 if (Opc == RISCVISD::FCVT_WU_RV64)
3624 FpToInt = DAG.getZeroExtendInReg(Op: FpToInt, DL, VT: MVT::i32);
3625
3626 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: DstVT);
3627 return DAG.getSelectCC(DL, LHS: Src, RHS: Src, True: ZeroInt, False: FpToInt,
3628 Cond: ISD::CondCode::SETUO);
3629 }
3630
3631 // Vectors.
3632
3633 MVT DstEltVT = DstVT.getVectorElementType();
3634 MVT SrcVT = Src.getSimpleValueType();
3635 MVT SrcEltVT = SrcVT.getVectorElementType();
3636 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3637 unsigned DstEltSize = DstEltVT.getSizeInBits();
3638
3639 // Only handle saturating to the destination type.
3640 if (SatVT != DstEltVT)
3641 return SDValue();
3642
3643 MVT DstContainerVT = DstVT;
3644 MVT SrcContainerVT = SrcVT;
3645 if (DstVT.isFixedLengthVector()) {
3646 DstContainerVT = getContainerForFixedLengthVector(VT: DstVT, Subtarget);
3647 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
3648 assert(DstContainerVT.getVectorElementCount() ==
3649 SrcContainerVT.getVectorElementCount() &&
3650 "Expected same element count");
3651 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
3652 }
3653
3654 SDLoc DL(Op);
3655
3656 auto [Mask, VL] = getDefaultVLOps(VecVT: DstVT, ContainerVT: DstContainerVT, DL, DAG, Subtarget);
3657
3658 SDValue IsNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
3659 Ops: {Src, Src, DAG.getCondCode(Cond: ISD::SETNE),
3660 DAG.getUNDEF(VT: Mask.getValueType()), Mask, VL});
3661
3662 // Need to widen by more than 1 step, promote the FP type, then do a widening
3663 // convert.
3664 if (DstEltSize > (2 * SrcEltSize)) {
3665 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
3666 MVT InterVT = SrcContainerVT.changeVectorElementType(EltVT: MVT::f32);
3667 Src = DAG.getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT: InterVT, N1: Src, N2: Mask, N3: VL);
3668 }
3669
3670 MVT CvtContainerVT = DstContainerVT;
3671 MVT CvtEltVT = DstEltVT;
3672 if (SrcEltSize > (2 * DstEltSize)) {
3673 CvtEltVT = MVT::getIntegerVT(BitWidth: SrcEltVT.getSizeInBits() / 2);
3674 CvtContainerVT = CvtContainerVT.changeVectorElementType(EltVT: CvtEltVT);
3675 }
3676
3677 unsigned RVVOpc =
3678 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
3679 SDValue Res = DAG.getNode(Opcode: RVVOpc, DL, VT: CvtContainerVT, N1: Src, N2: Mask, N3: VL);
3680
3681 while (CvtContainerVT != DstContainerVT) {
3682 CvtEltVT = MVT::getIntegerVT(BitWidth: CvtEltVT.getSizeInBits() / 2);
3683 CvtContainerVT = CvtContainerVT.changeVectorElementType(EltVT: CvtEltVT);
3684 // Rounding mode here is arbitrary since we aren't shifting out any bits.
3685 unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
3686 : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
3687 Res = DAG.getNode(Opcode: ClipOpc, DL, VT: CvtContainerVT, N1: Res, N2: Mask, N3: VL);
3688 }
3689
3690 SDValue SplatZero = DAG.getNode(
3691 Opcode: RISCVISD::VMV_V_X_VL, DL, VT: DstContainerVT, N1: DAG.getUNDEF(VT: DstContainerVT),
3692 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()), N3: VL);
3693 Res = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: DstContainerVT, N1: IsNan, N2: SplatZero,
3694 N3: Res, N4: DAG.getUNDEF(VT: DstContainerVT), N5: VL);
3695
3696 if (DstVT.isFixedLengthVector())
3697 Res = convertFromScalableVector(VT: DstVT, V: Res, DAG, Subtarget);
3698
3699 return Res;
3700}
3701
3702static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
3703 const RISCVSubtarget &Subtarget) {
3704 bool IsStrict = Op->isStrictFPOpcode();
3705 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
3706
3707 // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
3708 // bf16 conversions are always promoted to f32.
3709 if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3710 SrcVal.getValueType() == MVT::bf16) {
3711 SDLoc DL(Op);
3712 if (IsStrict) {
3713 SDValue Ext =
3714 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
3715 Ops: {Op.getOperand(i: 0), SrcVal});
3716 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
3717 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
3718 }
3719 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
3720 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: SrcVal));
3721 }
3722
3723 // Other operations are legal.
3724 return Op;
3725}
3726
3727static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
3728 switch (Opc) {
3729 case ISD::FROUNDEVEN:
3730 case ISD::STRICT_FROUNDEVEN:
3731 return RISCVFPRndMode::RNE;
3732 case ISD::FTRUNC:
3733 case ISD::STRICT_FTRUNC:
3734 return RISCVFPRndMode::RTZ;
3735 case ISD::FFLOOR:
3736 case ISD::STRICT_FFLOOR:
3737 return RISCVFPRndMode::RDN;
3738 case ISD::FCEIL:
3739 case ISD::STRICT_FCEIL:
3740 return RISCVFPRndMode::RUP;
3741 case ISD::FROUND:
3742 case ISD::LROUND:
3743 case ISD::LLROUND:
3744 case ISD::STRICT_FROUND:
3745 case ISD::STRICT_LROUND:
3746 case ISD::STRICT_LLROUND:
3747 return RISCVFPRndMode::RMM;
3748 case ISD::FRINT:
3749 case ISD::LRINT:
3750 case ISD::LLRINT:
3751 case ISD::STRICT_FRINT:
3752 case ISD::STRICT_LRINT:
3753 case ISD::STRICT_LLRINT:
3754 return RISCVFPRndMode::DYN;
3755 }
3756
3757 return RISCVFPRndMode::Invalid;
3758}
3759
3760// Expand vector FTRUNC, FCEIL, FFLOOR and FROUND by converting to
3761// the integer domain and back. Taking care to avoid converting values that are
3762// nan or already correct.
3763static SDValue
3764lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3765 const RISCVSubtarget &Subtarget) {
3766 MVT VT = Op.getSimpleValueType();
3767 assert(VT.isVector() && "Unexpected type");
3768
3769 SDLoc DL(Op);
3770
3771 SDValue Src = Op.getOperand(i: 0);
3772
3773 // Freeze the source since we are increasing the number of uses.
3774 Src = DAG.getFreeze(V: Src);
3775
3776 MVT ContainerVT = VT;
3777 if (VT.isFixedLengthVector()) {
3778 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3779 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
3780 }
3781
3782 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
3783
3784 // We do the conversion on the absolute value and fix the sign at the end.
3785 SDValue Abs = DAG.getNode(Opcode: RISCVISD::FABS_VL, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
3786
3787 // Determine the largest integer that can be represented exactly. This and
3788 // values larger than it don't have any fractional bits so don't need to
3789 // be converted.
3790 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3791 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3792 APFloat MaxVal = APFloat(FltSem);
3793 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3794 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3795 SDValue MaxValNode =
3796 DAG.getConstantFP(Val: MaxVal, DL, VT: ContainerVT.getVectorElementType());
3797 SDValue MaxValSplat = DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: ContainerVT,
3798 N1: DAG.getUNDEF(VT: ContainerVT), N2: MaxValNode, N3: VL);
3799
3800 // If abs(Src) was larger than MaxVal or nan, keep it.
3801 MVT SetccVT = MVT::getVectorVT(VT: MVT::i1, EC: ContainerVT.getVectorElementCount());
3802 Mask =
3803 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: SetccVT,
3804 Ops: {Abs, MaxValSplat, DAG.getCondCode(Cond: ISD::SETOLT),
3805 Mask, Mask, VL});
3806
3807 // Truncate to integer and convert back to FP.
3808 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3809 MVT XLenVT = Subtarget.getXLenVT();
3810 SDValue Truncated;
3811
3812 switch (Op.getOpcode()) {
3813 default:
3814 llvm_unreachable("Unexpected opcode");
3815 case ISD::FRINT:
3816 case ISD::FCEIL:
3817 case ISD::FFLOOR:
3818 case ISD::FROUND:
3819 case ISD::FROUNDEVEN: {
3820 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3821 assert(FRM != RISCVFPRndMode::Invalid);
3822 Truncated = DAG.getNode(Opcode: RISCVISD::VFCVT_RM_X_F_VL, DL, VT: IntVT, N1: Src, N2: Mask,
3823 N3: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), N4: VL);
3824 break;
3825 }
3826 case ISD::FTRUNC:
3827 Truncated = DAG.getNode(Opcode: RISCVISD::VFCVT_RTZ_X_F_VL, DL, VT: IntVT, N1: Src,
3828 N2: Mask, N3: VL);
3829 break;
3830 case ISD::FNEARBYINT:
3831 Truncated = DAG.getNode(Opcode: RISCVISD::VFROUND_NOEXCEPT_VL, DL, VT: ContainerVT, N1: Src,
3832 N2: Mask, N3: VL);
3833 break;
3834 }
3835
3836 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3837 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
3838 Truncated = DAG.getNode(Opcode: RISCVISD::SINT_TO_FP_VL, DL, VT: ContainerVT, N1: Truncated,
3839 N2: Mask, N3: VL);
3840
3841 // Restore the original sign so that -0.0 is preserved.
3842 Truncated = DAG.getNode(Opcode: RISCVISD::FCOPYSIGN_VL, DL, VT: ContainerVT, N1: Truncated,
3843 N2: Src, N3: Src, N4: Mask, N5: VL);
3844
3845 if (!VT.isFixedLengthVector())
3846 return Truncated;
3847
3848 return convertFromScalableVector(VT, V: Truncated, DAG, Subtarget);
3849}
3850
3851// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
3852// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
3853// qNan and converting the new source to integer and back to FP.
3854static SDValue
3855lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3856 const RISCVSubtarget &Subtarget) {
3857 SDLoc DL(Op);
3858 MVT VT = Op.getSimpleValueType();
3859 SDValue Chain = Op.getOperand(i: 0);
3860 SDValue Src = Op.getOperand(i: 1);
3861
3862 MVT ContainerVT = VT;
3863 if (VT.isFixedLengthVector()) {
3864 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
3865 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
3866 }
3867
3868 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
3869
3870 // Freeze the source since we are increasing the number of uses.
3871 Src = DAG.getFreeze(V: Src);
3872
3873 // Convert sNan to qNan by executing x + x for all unordered element x in Src.
3874 MVT MaskVT = Mask.getSimpleValueType();
3875 SDValue Unorder = DAG.getNode(Opcode: RISCVISD::STRICT_FSETCC_VL, DL,
3876 VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
3877 Ops: {Chain, Src, Src, DAG.getCondCode(Cond: ISD::SETUNE),
3878 DAG.getUNDEF(VT: MaskVT), Mask, VL});
3879 Chain = Unorder.getValue(R: 1);
3880 Src = DAG.getNode(Opcode: RISCVISD::STRICT_FADD_VL, DL,
3881 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
3882 Ops: {Chain, Src, Src, Src, Unorder, VL});
3883 Chain = Src.getValue(R: 1);
3884
3885 // We do the conversion on the absolute value and fix the sign at the end.
3886 SDValue Abs = DAG.getNode(Opcode: RISCVISD::FABS_VL, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
3887
3888 // Determine the largest integer that can be represented exactly. This and
3889 // values larger than it don't have any fractional bits so don't need to
3890 // be converted.
3891 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3892 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3893 APFloat MaxVal = APFloat(FltSem);
3894 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3895 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3896 SDValue MaxValNode =
3897 DAG.getConstantFP(Val: MaxVal, DL, VT: ContainerVT.getVectorElementType());
3898 SDValue MaxValSplat = DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: ContainerVT,
3899 N1: DAG.getUNDEF(VT: ContainerVT), N2: MaxValNode, N3: VL);
3900
3901 // If abs(Src) was larger than MaxVal or nan, keep it.
3902 Mask = DAG.getNode(
3903 Opcode: RISCVISD::SETCC_VL, DL, VT: MaskVT,
3904 Ops: {Abs, MaxValSplat, DAG.getCondCode(Cond: ISD::SETOLT), Mask, Mask, VL});
3905
3906 // Truncate to integer and convert back to FP.
3907 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3908 MVT XLenVT = Subtarget.getXLenVT();
3909 SDValue Truncated;
3910
3911 switch (Op.getOpcode()) {
3912 default:
3913 llvm_unreachable("Unexpected opcode");
3914 case ISD::STRICT_FCEIL:
3915 case ISD::STRICT_FFLOOR:
3916 case ISD::STRICT_FROUND:
3917 case ISD::STRICT_FROUNDEVEN: {
3918 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3919 assert(FRM != RISCVFPRndMode::Invalid);
3920 Truncated = DAG.getNode(
3921 Opcode: RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, VTList: DAG.getVTList(VT1: IntVT, VT2: MVT::Other),
3922 Ops: {Chain, Src, Mask, DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), VL});
3923 break;
3924 }
3925 case ISD::STRICT_FTRUNC:
3926 Truncated =
3927 DAG.getNode(Opcode: RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3928 VTList: DAG.getVTList(VT1: IntVT, VT2: MVT::Other), N1: Chain, N2: Src, N3: Mask, N4: VL);
3929 break;
3930 case ISD::STRICT_FNEARBYINT:
3931 Truncated = DAG.getNode(Opcode: RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3932 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), N1: Chain, N2: Src,
3933 N3: Mask, N4: VL);
3934 break;
3935 }
3936 Chain = Truncated.getValue(R: 1);
3937
3938 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3939 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3940 Truncated = DAG.getNode(Opcode: RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3941 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), N1: Chain,
3942 N2: Truncated, N3: Mask, N4: VL);
3943 Chain = Truncated.getValue(R: 1);
3944 }
3945
3946 // Restore the original sign so that -0.0 is preserved.
3947 Truncated = DAG.getNode(Opcode: RISCVISD::FCOPYSIGN_VL, DL, VT: ContainerVT, N1: Truncated,
3948 N2: Src, N3: Src, N4: Mask, N5: VL);
3949
3950 if (VT.isFixedLengthVector())
3951 Truncated = convertFromScalableVector(VT, V: Truncated, DAG, Subtarget);
3952 return DAG.getMergeValues(Ops: {Truncated, Chain}, dl: DL);
3953}
3954
3955static SDValue
3956lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3957 const RISCVSubtarget &Subtarget) {
3958 MVT VT = Op.getSimpleValueType();
3959 if (VT.isVector())
3960 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
3961
3962 if (DAG.shouldOptForSize())
3963 return SDValue();
3964
3965 SDLoc DL(Op);
3966 SDValue Src = Op.getOperand(i: 0);
3967
3968 // Create an integer the size of the mantissa with the MSB set. This and all
3969 // values larger than it don't have any fractional bits so don't need to be
3970 // converted.
3971 const fltSemantics &FltSem = VT.getFltSemantics();
3972 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3973 APFloat MaxVal = APFloat(FltSem);
3974 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3975 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3976 SDValue MaxValNode = DAG.getConstantFP(Val: MaxVal, DL, VT);
3977
3978 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3979 return DAG.getNode(Opcode: RISCVISD::FROUND, DL, VT, N1: Src, N2: MaxValNode,
3980 N3: DAG.getTargetConstant(Val: FRM, DL, VT: Subtarget.getXLenVT()));
3981}
3982
3983// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
3984static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
3985 const RISCVSubtarget &Subtarget) {
3986 SDLoc DL(Op);
3987 MVT DstVT = Op.getSimpleValueType();
3988 SDValue Src = Op.getOperand(i: 0);
3989 MVT SrcVT = Src.getSimpleValueType();
3990 assert(SrcVT.isVector() && DstVT.isVector() &&
3991 !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
3992 "Unexpected type");
3993
3994 MVT DstContainerVT = DstVT;
3995 MVT SrcContainerVT = SrcVT;
3996
3997 if (DstVT.isFixedLengthVector()) {
3998 DstContainerVT = getContainerForFixedLengthVector(VT: DstVT, Subtarget);
3999 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
4000 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
4001 }
4002
4003 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT: SrcContainerVT, DL, DAG, Subtarget);
4004
4005 // [b]f16 -> f32
4006 MVT SrcElemType = SrcVT.getVectorElementType();
4007 if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
4008 MVT F32VT = SrcContainerVT.changeVectorElementType(EltVT: MVT::f32);
4009 Src = DAG.getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT: F32VT, N1: Src, N2: Mask, N3: VL);
4010 }
4011
4012 SDValue Res =
4013 DAG.getNode(Opcode: RISCVISD::VFCVT_RM_X_F_VL, DL, VT: DstContainerVT, N1: Src, N2: Mask,
4014 N3: DAG.getTargetConstant(Val: matchRoundingOp(Opc: Op.getOpcode()), DL,
4015 VT: Subtarget.getXLenVT()),
4016 N4: VL);
4017
4018 if (!DstVT.isFixedLengthVector())
4019 return Res;
4020
4021 return convertFromScalableVector(VT: DstVT, V: Res, DAG, Subtarget);
4022}
4023
4024static SDValue
4025getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
4026 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
4027 SDValue Offset, SDValue Mask, SDValue VL,
4028 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4029 if (Passthru.isUndef())
4030 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4031 SDValue PolicyOp = DAG.getTargetConstant(Val: Policy, DL, VT: Subtarget.getXLenVT());
4032 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4033 return DAG.getNode(Opcode: RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
4034}
4035
4036static SDValue
4037getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
4038 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
4039 SDValue VL,
4040 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
4041 if (Passthru.isUndef())
4042 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4043 SDValue PolicyOp = DAG.getTargetConstant(Val: Policy, DL, VT: Subtarget.getXLenVT());
4044 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
4045 return DAG.getNode(Opcode: RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
4046}
4047
4048struct VIDSequence {
4049 int64_t StepNumerator;
4050 unsigned StepDenominator;
4051 int64_t Addend;
4052};
4053
4054static std::optional<APInt> getExactInteger(const APFloat &APF,
4055 uint32_t BitWidth) {
4056 // We will use a SINT_TO_FP to materialize this constant so we should use a
4057 // signed APSInt here.
4058 APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
4059 // We use an arbitrary rounding mode here. If a floating-point is an exact
4060 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
4061 // the rounding mode changes the output value, then it is not an exact
4062 // integer.
4063 RoundingMode ArbitraryRM = RoundingMode::TowardZero;
4064 bool IsExact;
4065 // If it is out of signed integer range, it will return an invalid operation.
4066 // If it is not an exact integer, IsExact is false.
4067 if ((APF.convertToInteger(Result&: ValInt, RM: ArbitraryRM, IsExact: &IsExact) ==
4068 APFloatBase::opInvalidOp) ||
4069 !IsExact)
4070 return std::nullopt;
4071 return ValInt.extractBits(numBits: BitWidth, bitPosition: 0);
4072}
4073
4074// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
4075// to the (non-zero) step S and start value X. This can be then lowered as the
4076// RVV sequence (VID * S) + X, for example.
4077// The step S is represented as an integer numerator divided by a positive
4078// denominator. Note that the implementation currently only identifies
4079// sequences in which either the numerator is +/- 1 or the denominator is 1. It
4080// cannot detect 2/3, for example.
4081// Note that this method will also match potentially unappealing index
4082// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
4083// determine whether this is worth generating code for.
4084//
4085// EltSizeInBits is the size of the type that the sequence will be calculated
4086// in, i.e. SEW for build_vectors or XLEN for address calculations.
4087static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
4088 unsigned EltSizeInBits) {
4089 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
4090 if (!cast<BuildVectorSDNode>(Val&: Op)->isConstant())
4091 return std::nullopt;
4092 bool IsInteger = Op.getValueType().isInteger();
4093
4094 std::optional<unsigned> SeqStepDenom;
4095 std::optional<APInt> SeqStepNum;
4096 std::optional<APInt> SeqAddend;
4097 std::optional<std::pair<APInt, unsigned>> PrevElt;
4098 assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
4099
4100 // First extract the ops into a list of constant integer values. This may not
4101 // be possible for floats if they're not all representable as integers.
4102 SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
4103 const unsigned OpSize = Op.getScalarValueSizeInBits();
4104 for (auto [Idx, Elt] : enumerate(First: Op->op_values())) {
4105 if (Elt.isUndef()) {
4106 Elts[Idx] = std::nullopt;
4107 continue;
4108 }
4109 if (IsInteger) {
4110 Elts[Idx] = Elt->getAsAPIntVal().trunc(width: OpSize).zext(width: EltSizeInBits);
4111 } else {
4112 auto ExactInteger =
4113 getExactInteger(APF: cast<ConstantFPSDNode>(Val: Elt)->getValueAPF(), BitWidth: OpSize);
4114 if (!ExactInteger)
4115 return std::nullopt;
4116 Elts[Idx] = *ExactInteger;
4117 }
4118 }
4119
4120 for (auto [Idx, Elt] : enumerate(First&: Elts)) {
4121 // Assume undef elements match the sequence; we just have to be careful
4122 // when interpolating across them.
4123 if (!Elt)
4124 continue;
4125
4126 if (PrevElt) {
4127 // Calculate the step since the last non-undef element, and ensure
4128 // it's consistent across the entire sequence.
4129 unsigned IdxDiff = Idx - PrevElt->second;
4130 APInt ValDiff = *Elt - PrevElt->first;
4131
4132 // A zero-value value difference means that we're somewhere in the middle
4133 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
4134 // step change before evaluating the sequence.
4135 if (ValDiff == 0)
4136 continue;
4137
4138 int64_t Remainder = ValDiff.srem(RHS: IdxDiff);
4139 // Normalize the step if it's greater than 1.
4140 if (Remainder != ValDiff.getSExtValue()) {
4141 // The difference must cleanly divide the element span.
4142 if (Remainder != 0)
4143 return std::nullopt;
4144 ValDiff = ValDiff.sdiv(RHS: IdxDiff);
4145 IdxDiff = 1;
4146 }
4147
4148 if (!SeqStepNum)
4149 SeqStepNum = ValDiff;
4150 else if (ValDiff != SeqStepNum)
4151 return std::nullopt;
4152
4153 if (!SeqStepDenom)
4154 SeqStepDenom = IdxDiff;
4155 else if (IdxDiff != *SeqStepDenom)
4156 return std::nullopt;
4157 }
4158
4159 // Record this non-undef element for later.
4160 if (!PrevElt || PrevElt->first != *Elt)
4161 PrevElt = std::make_pair(x&: *Elt, y&: Idx);
4162 }
4163
4164 // We need to have logged a step for this to count as a legal index sequence.
4165 if (!SeqStepNum || !SeqStepDenom)
4166 return std::nullopt;
4167
4168 // Loop back through the sequence and validate elements we might have skipped
4169 // while waiting for a valid step. While doing this, log any sequence addend.
4170 for (auto [Idx, Elt] : enumerate(First&: Elts)) {
4171 if (!Elt)
4172 continue;
4173 APInt ExpectedVal =
4174 (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
4175 *SeqStepNum)
4176 .sdiv(RHS: *SeqStepDenom);
4177
4178 APInt Addend = *Elt - ExpectedVal;
4179 if (!SeqAddend)
4180 SeqAddend = Addend;
4181 else if (Addend != SeqAddend)
4182 return std::nullopt;
4183 }
4184
4185 assert(SeqAddend && "Must have an addend if we have a step");
4186
4187 return VIDSequence{.StepNumerator: SeqStepNum->getSExtValue(), .StepDenominator: *SeqStepDenom,
4188 .Addend: SeqAddend->getSExtValue()};
4189}
4190
4191// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
4192// and lower it as a VRGATHER_VX_VL from the source vector.
4193static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
4194 SelectionDAG &DAG,
4195 const RISCVSubtarget &Subtarget) {
4196 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
4197 return SDValue();
4198 SDValue Src = SplatVal.getOperand(i: 0);
4199 // Don't perform this optimization for i1 vectors, or if the element types are
4200 // different
4201 // FIXME: Support i1 vectors, maybe by promoting to i8?
4202 MVT EltTy = VT.getVectorElementType();
4203 if (EltTy == MVT::i1 ||
4204 !DAG.getTargetLoweringInfo().isTypeLegal(VT: Src.getValueType()))
4205 return SDValue();
4206 MVT SrcVT = Src.getSimpleValueType();
4207 if (EltTy != SrcVT.getVectorElementType())
4208 return SDValue();
4209 SDValue Idx = SplatVal.getOperand(i: 1);
4210 // The index must be a legal type.
4211 if (Idx.getValueType() != Subtarget.getXLenVT())
4212 return SDValue();
4213
4214 // Check that we know Idx lies within VT
4215 if (!TypeSize::isKnownLE(LHS: SrcVT.getSizeInBits(), RHS: VT.getSizeInBits())) {
4216 auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
4217 if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
4218 return SDValue();
4219 }
4220
4221 // Convert fixed length vectors to scalable
4222 MVT ContainerVT = VT;
4223 if (VT.isFixedLengthVector())
4224 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4225
4226 MVT SrcContainerVT = SrcVT;
4227 if (SrcVT.isFixedLengthVector()) {
4228 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
4229 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
4230 }
4231
4232 // Put Vec in a VT sized vector
4233 if (SrcContainerVT.getVectorMinNumElements() <
4234 ContainerVT.getVectorMinNumElements())
4235 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec: Src, Idx: 0);
4236 else
4237 Src = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec: Src, Idx: 0);
4238
4239 // We checked that Idx fits inside VT earlier
4240 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4241 SDValue Gather = DAG.getNode(Opcode: RISCVISD::VRGATHER_VX_VL, DL, VT: ContainerVT, N1: Src,
4242 N2: Idx, N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
4243 if (VT.isFixedLengthVector())
4244 Gather = convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
4245 return Gather;
4246}
4247
4248static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
4249 const RISCVSubtarget &Subtarget) {
4250 MVT VT = Op.getSimpleValueType();
4251 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4252
4253 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4254
4255 SDLoc DL(Op);
4256 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4257
4258 if (auto SimpleVID = isSimpleVIDSequence(Op, EltSizeInBits: Op.getScalarValueSizeInBits())) {
4259 int64_t StepNumerator = SimpleVID->StepNumerator;
4260 unsigned StepDenominator = SimpleVID->StepDenominator;
4261 int64_t Addend = SimpleVID->Addend;
4262
4263 assert(StepNumerator != 0 && "Invalid step");
4264 bool Negate = false;
4265 int64_t SplatStepVal = StepNumerator;
4266 unsigned StepOpcode = ISD::MUL;
4267 // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
4268 // anyway as the shift of 63 won't fit in uimm5.
4269 if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
4270 isPowerOf2_64(Value: std::abs(i: StepNumerator))) {
4271 Negate = StepNumerator < 0;
4272 StepOpcode = ISD::SHL;
4273 SplatStepVal = Log2_64(Value: std::abs(i: StepNumerator));
4274 }
4275
4276 // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
4277 // since it's the immediate value many RVV instructions accept. There is
4278 // no vmul.vi instruction so ensure multiply constant can fit in a
4279 // single addi instruction. For the addend, we allow up to 32 bits..
4280 if (((StepOpcode == ISD::MUL && isInt<12>(x: SplatStepVal)) ||
4281 (StepOpcode == ISD::SHL && isUInt<5>(x: SplatStepVal))) &&
4282 isPowerOf2_32(Value: StepDenominator) &&
4283 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(x: Addend)) {
4284 MVT VIDVT =
4285 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
4286 MVT VIDContainerVT = getContainerForFixedLengthVector(VT: VIDVT, Subtarget);
4287 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: VIDContainerVT, N1: Mask, N2: VL);
4288 // Convert right out of the scalable type so we can use standard ISD
4289 // nodes for the rest of the computation. If we used scalable types with
4290 // these, we'd lose the fixed-length vector info and generate worse
4291 // vsetvli code.
4292 VID = convertFromScalableVector(VT: VIDVT, V: VID, DAG, Subtarget);
4293 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
4294 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
4295 SDValue SplatStep = DAG.getSignedConstant(Val: SplatStepVal, DL, VT: VIDVT);
4296 VID = DAG.getNode(Opcode: StepOpcode, DL, VT: VIDVT, N1: VID, N2: SplatStep);
4297 }
4298 if (StepDenominator != 1) {
4299 SDValue SplatStep =
4300 DAG.getConstant(Val: Log2_64(Value: StepDenominator), DL, VT: VIDVT);
4301 VID = DAG.getNode(Opcode: ISD::SRL, DL, VT: VIDVT, N1: VID, N2: SplatStep);
4302 }
4303 if (Addend != 0 || Negate) {
4304 SDValue SplatAddend = DAG.getSignedConstant(Val: Addend, DL, VT: VIDVT);
4305 VID = DAG.getNode(Opcode: Negate ? ISD::SUB : ISD::ADD, DL, VT: VIDVT, N1: SplatAddend,
4306 N2: VID);
4307 }
4308 if (VT.isFloatingPoint()) {
4309 // TODO: Use vfwcvt to reduce register pressure.
4310 VID = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: VID);
4311 }
4312 return VID;
4313 }
4314 }
4315
4316 return SDValue();
4317}
4318
4319/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
4320/// which constitute a large proportion of the elements. In such cases we can
4321/// splat a vector with the dominant element and make up the shortfall with
4322/// INSERT_VECTOR_ELTs. Returns SDValue if not profitable.
4323/// Note that this includes vectors of 2 elements by association. The
4324/// upper-most element is the "dominant" one, allowing us to use a splat to
4325/// "insert" the upper element, and an insert of the lower element at position
4326/// 0, which improves codegen.
4327static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
4328 const RISCVSubtarget &Subtarget) {
4329 MVT VT = Op.getSimpleValueType();
4330 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4331
4332 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4333
4334 SDLoc DL(Op);
4335 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4336
4337 MVT XLenVT = Subtarget.getXLenVT();
4338 unsigned NumElts = Op.getNumOperands();
4339
4340 SDValue DominantValue;
4341 unsigned MostCommonCount = 0;
4342 DenseMap<SDValue, unsigned> ValueCounts;
4343 unsigned NumUndefElts =
4344 count_if(Range: Op->op_values(), P: [](const SDValue &V) { return V.isUndef(); });
4345
4346 // Track the number of scalar loads we know we'd be inserting, estimated as
4347 // any non-zero floating-point constant. Other kinds of element are either
4348 // already in registers or are materialized on demand. The threshold at which
4349 // a vector load is more desirable than several scalar materializion and
4350 // vector-insertion instructions is not known.
4351 unsigned NumScalarLoads = 0;
4352
4353 for (SDValue V : Op->op_values()) {
4354 if (V.isUndef())
4355 continue;
4356
4357 unsigned &Count = ValueCounts[V];
4358 if (0 == Count)
4359 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: V))
4360 NumScalarLoads += !CFP->isPosZero();
4361
4362 // Is this value dominant? In case of a tie, prefer the highest element as
4363 // it's cheaper to insert near the beginning of a vector than it is at the
4364 // end.
4365 if (++Count >= MostCommonCount) {
4366 DominantValue = V;
4367 MostCommonCount = Count;
4368 }
4369 }
4370
4371 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
4372 unsigned NumDefElts = NumElts - NumUndefElts;
4373 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
4374
4375 // Don't perform this optimization when optimizing for size, since
4376 // materializing elements and inserting them tends to cause code bloat.
4377 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
4378 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode())) &&
4379 ((MostCommonCount > DominantValueCountThreshold) ||
4380 (ValueCounts.size() <= Log2_32(Value: NumDefElts)))) {
4381 // Start by splatting the most common element.
4382 SDValue Vec = DAG.getSplatBuildVector(VT, DL, Op: DominantValue);
4383
4384 DenseSet<SDValue> Processed{DominantValue};
4385
4386 // We can handle an insert into the last element (of a splat) via
4387 // v(f)slide1down. This is slightly better than the vslideup insert
4388 // lowering as it avoids the need for a vector group temporary. It
4389 // is also better than using vmerge.vx as it avoids the need to
4390 // materialize the mask in a vector register.
4391 if (SDValue LastOp = Op->getOperand(Num: Op->getNumOperands() - 1);
4392 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
4393 LastOp != DominantValue) {
4394 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
4395 auto OpCode =
4396 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
4397 if (!VT.isFloatingPoint())
4398 LastOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: LastOp);
4399 Vec = DAG.getNode(Opcode: OpCode, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Vec,
4400 N3: LastOp, N4: Mask, N5: VL);
4401 Vec = convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
4402 Processed.insert(V: LastOp);
4403 }
4404
4405 MVT SelMaskTy = VT.changeVectorElementType(EltVT: MVT::i1);
4406 for (const auto &OpIdx : enumerate(First: Op->ops())) {
4407 const SDValue &V = OpIdx.value();
4408 if (V.isUndef() || !Processed.insert(V).second)
4409 continue;
4410 if (ValueCounts[V] == 1) {
4411 Vec = DAG.getInsertVectorElt(DL, Vec, Elt: V, Idx: OpIdx.index());
4412 } else {
4413 // Blend in all instances of this value using a VSELECT, using a
4414 // mask where each bit signals whether that element is the one
4415 // we're after.
4416 SmallVector<SDValue> Ops;
4417 transform(Range: Op->op_values(), d_first: std::back_inserter(x&: Ops), F: [&](SDValue V1) {
4418 return DAG.getConstant(Val: V == V1, DL, VT: XLenVT);
4419 });
4420 Vec = DAG.getNode(Opcode: ISD::VSELECT, DL, VT,
4421 N1: DAG.getBuildVector(VT: SelMaskTy, DL, Ops),
4422 N2: DAG.getSplatBuildVector(VT, DL, Op: V), N3: Vec);
4423 }
4424 }
4425
4426 return Vec;
4427 }
4428
4429 return SDValue();
4430}
4431
4432static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
4433 const RISCVSubtarget &Subtarget) {
4434 MVT VT = Op.getSimpleValueType();
4435 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4436
4437 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4438
4439 SDLoc DL(Op);
4440 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4441
4442 MVT XLenVT = Subtarget.getXLenVT();
4443 unsigned NumElts = Op.getNumOperands();
4444
4445 if (VT.getVectorElementType() == MVT::i1) {
4446 if (ISD::isBuildVectorAllZeros(N: Op.getNode())) {
4447 SDValue VMClr = DAG.getNode(Opcode: RISCVISD::VMCLR_VL, DL, VT: ContainerVT, Operand: VL);
4448 return convertFromScalableVector(VT, V: VMClr, DAG, Subtarget);
4449 }
4450
4451 if (ISD::isBuildVectorAllOnes(N: Op.getNode())) {
4452 SDValue VMSet = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
4453 return convertFromScalableVector(VT, V: VMSet, DAG, Subtarget);
4454 }
4455
4456 // Lower constant mask BUILD_VECTORs via an integer vector type, in
4457 // scalar integer chunks whose bit-width depends on the number of mask
4458 // bits and XLEN.
4459 // First, determine the most appropriate scalar integer type to use. This
4460 // is at most XLenVT, but may be shrunk to a smaller vector element type
4461 // according to the size of the final vector - use i8 chunks rather than
4462 // XLenVT if we're producing a v8i1. This results in more consistent
4463 // codegen across RV32 and RV64.
4464 unsigned NumViaIntegerBits = std::clamp(val: NumElts, lo: 8u, hi: Subtarget.getXLen());
4465 NumViaIntegerBits = std::min(a: NumViaIntegerBits, b: Subtarget.getELen());
4466 // If we have to use more than one INSERT_VECTOR_ELT then this
4467 // optimization is likely to increase code size; avoid performing it in
4468 // such a case. We can use a load from a constant pool in this case.
4469 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
4470 return SDValue();
4471 // Now we can create our integer vector type. Note that it may be larger
4472 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
4473 unsigned IntegerViaVecElts = divideCeil(Numerator: NumElts, Denominator: NumViaIntegerBits);
4474 MVT IntegerViaVecVT =
4475 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: NumViaIntegerBits),
4476 NumElements: IntegerViaVecElts);
4477
4478 uint64_t Bits = 0;
4479 unsigned BitPos = 0, IntegerEltIdx = 0;
4480 SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
4481
4482 for (unsigned I = 0; I < NumElts;) {
4483 SDValue V = Op.getOperand(i: I);
4484 bool BitValue = !V.isUndef() && V->getAsZExtVal();
4485 Bits |= ((uint64_t)BitValue << BitPos);
4486 ++BitPos;
4487 ++I;
4488
4489 // Once we accumulate enough bits to fill our scalar type or process the
4490 // last element, insert into our vector and clear our accumulated data.
4491 if (I % NumViaIntegerBits == 0 || I == NumElts) {
4492 if (NumViaIntegerBits <= 32)
4493 Bits = SignExtend64<32>(x: Bits);
4494 SDValue Elt = DAG.getSignedConstant(Val: Bits, DL, VT: XLenVT);
4495 Elts[IntegerEltIdx] = Elt;
4496 Bits = 0;
4497 BitPos = 0;
4498 IntegerEltIdx++;
4499 }
4500 }
4501
4502 SDValue Vec = DAG.getBuildVector(VT: IntegerViaVecVT, DL, Ops: Elts);
4503
4504 if (NumElts < NumViaIntegerBits) {
4505 // If we're producing a smaller vector than our minimum legal integer
4506 // type, bitcast to the equivalent (known-legal) mask type, and extract
4507 // our final mask.
4508 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
4509 Vec = DAG.getBitcast(VT: MVT::v8i1, V: Vec);
4510 Vec = DAG.getExtractSubvector(DL, VT, Vec, Idx: 0);
4511 } else {
4512 // Else we must have produced an integer type with the same size as the
4513 // mask type; bitcast for the final result.
4514 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
4515 Vec = DAG.getBitcast(VT, V: Vec);
4516 }
4517
4518 return Vec;
4519 }
4520
4521 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4522 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
4523 : RISCVISD::VMV_V_X_VL;
4524 if (!VT.isFloatingPoint())
4525 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Splat);
4526 Splat =
4527 DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Splat, N3: VL);
4528 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
4529 }
4530
4531 // Try and match index sequences, which we can lower to the vid instruction
4532 // with optional modifications. An all-undef vector is matched by
4533 // getSplatValue, above.
4534 if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
4535 return Res;
4536
4537 // For very small build_vectors, use a single scalar insert of a constant.
4538 // TODO: Base this on constant rematerialization cost, not size.
4539 const unsigned EltBitSize = VT.getScalarSizeInBits();
4540 if (VT.getSizeInBits() <= 32 &&
4541 ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode())) {
4542 MVT ViaIntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
4543 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
4544 "Unexpected sequence type");
4545 // If we can use the original VL with the modified element type, this
4546 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4547 // be moved into InsertVSETVLI?
4548 unsigned ViaVecLen =
4549 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
4550 MVT ViaVecVT = MVT::getVectorVT(VT: ViaIntVT, NumElements: ViaVecLen);
4551
4552 uint64_t EltMask = maskTrailingOnes<uint64_t>(N: EltBitSize);
4553 uint64_t SplatValue = 0;
4554 // Construct the amalgamated value at this larger vector type.
4555 for (const auto &OpIdx : enumerate(First: Op->op_values())) {
4556 const auto &SeqV = OpIdx.value();
4557 if (!SeqV.isUndef())
4558 SplatValue |=
4559 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
4560 }
4561
4562 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4563 // achieve better constant materializion.
4564 // On RV32, we need to sign-extend to use getSignedConstant.
4565 if (ViaIntVT == MVT::i32)
4566 SplatValue = SignExtend64<32>(x: SplatValue);
4567
4568 SDValue Vec = DAG.getInsertVectorElt(
4569 DL, Vec: DAG.getUNDEF(VT: ViaVecVT),
4570 Elt: DAG.getSignedConstant(Val: SplatValue, DL, VT: XLenVT), Idx: 0);
4571 if (ViaVecLen != 1)
4572 Vec = DAG.getExtractSubvector(DL, VT: MVT::getVectorVT(VT: ViaIntVT, NumElements: 1), Vec, Idx: 0);
4573 return DAG.getBitcast(VT, V: Vec);
4574 }
4575
4576
4577 // Attempt to detect "hidden" splats, which only reveal themselves as splats
4578 // when re-interpreted as a vector with a larger element type. For example,
4579 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
4580 // could be instead splat as
4581 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
4582 // TODO: This optimization could also work on non-constant splats, but it
4583 // would require bit-manipulation instructions to construct the splat value.
4584 SmallVector<SDValue> Sequence;
4585 const auto *BV = cast<BuildVectorSDNode>(Val&: Op);
4586 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
4587 ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode()) &&
4588 BV->getRepeatedSequence(Sequence) &&
4589 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
4590 unsigned SeqLen = Sequence.size();
4591 MVT ViaIntVT = MVT::getIntegerVT(BitWidth: EltBitSize * SeqLen);
4592 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
4593 ViaIntVT == MVT::i64) &&
4594 "Unexpected sequence type");
4595
4596 // If we can use the original VL with the modified element type, this
4597 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4598 // be moved into InsertVSETVLI?
4599 const unsigned RequiredVL = NumElts / SeqLen;
4600 const unsigned ViaVecLen =
4601 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
4602 NumElts : RequiredVL;
4603 MVT ViaVecVT = MVT::getVectorVT(VT: ViaIntVT, NumElements: ViaVecLen);
4604
4605 unsigned EltIdx = 0;
4606 uint64_t EltMask = maskTrailingOnes<uint64_t>(N: EltBitSize);
4607 uint64_t SplatValue = 0;
4608 // Construct the amalgamated value which can be splatted as this larger
4609 // vector type.
4610 for (const auto &SeqV : Sequence) {
4611 if (!SeqV.isUndef())
4612 SplatValue |=
4613 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
4614 EltIdx++;
4615 }
4616
4617 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4618 // achieve better constant materializion.
4619 // On RV32, we need to sign-extend to use getSignedConstant.
4620 if (ViaIntVT == MVT::i32)
4621 SplatValue = SignExtend64<32>(x: SplatValue);
4622
4623 // Since we can't introduce illegal i64 types at this stage, we can only
4624 // perform an i64 splat on RV32 if it is its own sign-extended value. That
4625 // way we can use RVV instructions to splat.
4626 assert((ViaIntVT.bitsLE(XLenVT) ||
4627 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
4628 "Unexpected bitcast sequence");
4629 if (ViaIntVT.bitsLE(VT: XLenVT) || isInt<32>(x: SplatValue)) {
4630 SDValue ViaVL =
4631 DAG.getConstant(Val: ViaVecVT.getVectorNumElements(), DL, VT: XLenVT);
4632 MVT ViaContainerVT =
4633 getContainerForFixedLengthVector(VT: ViaVecVT, Subtarget);
4634 SDValue Splat =
4635 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ViaContainerVT,
4636 N1: DAG.getUNDEF(VT: ViaContainerVT),
4637 N2: DAG.getSignedConstant(Val: SplatValue, DL, VT: XLenVT), N3: ViaVL);
4638 Splat = convertFromScalableVector(VT: ViaVecVT, V: Splat, DAG, Subtarget);
4639 if (ViaVecLen != RequiredVL)
4640 Splat = DAG.getExtractSubvector(
4641 DL, VT: MVT::getVectorVT(VT: ViaIntVT, NumElements: RequiredVL), Vec: Splat, Idx: 0);
4642 return DAG.getBitcast(VT, V: Splat);
4643 }
4644 }
4645
4646 // If the number of signbits allows, see if we can lower as a <N x i8>.
4647 // Our main goal here is to reduce LMUL (and thus work) required to
4648 // build the constant, but we will also narrow if the resulting
4649 // narrow vector is known to materialize cheaply.
4650 // TODO: We really should be costing the smaller vector. There are
4651 // profitable cases this misses.
4652 if (EltBitSize > 8 && VT.isInteger() &&
4653 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
4654 DAG.ComputeMaxSignificantBits(Op) <= 8) {
4655 SDValue Source = DAG.getBuildVector(VT: VT.changeVectorElementType(EltVT: MVT::i8),
4656 DL, Ops: Op->ops());
4657 Source = convertToScalableVector(VT: ContainerVT.changeVectorElementType(EltVT: MVT::i8),
4658 V: Source, DAG, Subtarget);
4659 SDValue Res = DAG.getNode(Opcode: RISCVISD::VSEXT_VL, DL, VT: ContainerVT, N1: Source, N2: Mask, N3: VL);
4660 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
4661 }
4662
4663 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4664 return Res;
4665
4666 // For constant vectors, use generic constant pool lowering. Otherwise,
4667 // we'd have to materialize constants in GPRs just to move them into the
4668 // vector.
4669 return SDValue();
4670}
4671
4672static unsigned getPACKOpcode(unsigned DestBW,
4673 const RISCVSubtarget &Subtarget) {
4674 switch (DestBW) {
4675 default:
4676 llvm_unreachable("Unsupported pack size");
4677 case 16:
4678 return RISCV::PACKH;
4679 case 32:
4680 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
4681 case 64:
4682 assert(Subtarget.is64Bit());
4683 return RISCV::PACK;
4684 }
4685}
4686
4687/// Double the element size of the build vector to reduce the number
4688/// of vslide1down in the build vector chain. In the worst case, this
4689/// trades three scalar operations for 1 vector operation. Scalar
4690/// operations are generally lower latency, and for out-of-order cores
4691/// we also benefit from additional parallelism.
4692static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
4693 const RISCVSubtarget &Subtarget) {
4694 SDLoc DL(Op);
4695 MVT VT = Op.getSimpleValueType();
4696 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4697 MVT ElemVT = VT.getVectorElementType();
4698 if (!ElemVT.isInteger())
4699 return SDValue();
4700
4701 // TODO: Relax these architectural restrictions, possibly with costing
4702 // of the actual instructions required.
4703 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4704 return SDValue();
4705
4706 unsigned NumElts = VT.getVectorNumElements();
4707 unsigned ElemSizeInBits = ElemVT.getSizeInBits();
4708 if (ElemSizeInBits >= std::min(a: Subtarget.getELen(), b: Subtarget.getXLen()) ||
4709 NumElts % 2 != 0)
4710 return SDValue();
4711
4712 // Produce [B,A] packed into a type twice as wide. Note that all
4713 // scalars are XLenVT, possibly masked (see below).
4714 MVT XLenVT = Subtarget.getXLenVT();
4715 SDValue Mask = DAG.getConstant(
4716 Val: APInt::getLowBitsSet(numBits: XLenVT.getSizeInBits(), loBitsSet: ElemSizeInBits), DL, VT: XLenVT);
4717 auto pack = [&](SDValue A, SDValue B) {
4718 // Bias the scheduling of the inserted operations to near the
4719 // definition of the element - this tends to reduce register
4720 // pressure overall.
4721 SDLoc ElemDL(B);
4722 if (Subtarget.hasStdExtZbkb())
4723 // Note that we're relying on the high bits of the result being
4724 // don't care. For PACKW, the result is *sign* extended.
4725 return SDValue(
4726 DAG.getMachineNode(Opcode: getPACKOpcode(DestBW: ElemSizeInBits * 2, Subtarget),
4727 dl: ElemDL, VT: XLenVT, Op1: A, Op2: B),
4728 0);
4729
4730 A = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(A), VT: XLenVT, N1: A, N2: Mask);
4731 B = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(B), VT: XLenVT, N1: B, N2: Mask);
4732 SDValue ShtAmt = DAG.getConstant(Val: ElemSizeInBits, DL: ElemDL, VT: XLenVT);
4733 return DAG.getNode(Opcode: ISD::OR, DL: ElemDL, VT: XLenVT, N1: A,
4734 N2: DAG.getNode(Opcode: ISD::SHL, DL: ElemDL, VT: XLenVT, N1: B, N2: ShtAmt),
4735 Flags: SDNodeFlags::Disjoint);
4736 };
4737
4738 SmallVector<SDValue> NewOperands;
4739 NewOperands.reserve(N: NumElts / 2);
4740 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
4741 NewOperands.push_back(Elt: pack(Op.getOperand(i), Op.getOperand(i: i + 1)));
4742 assert(NumElts == NewOperands.size() * 2);
4743 MVT WideVT = MVT::getIntegerVT(BitWidth: ElemSizeInBits * 2);
4744 MVT WideVecVT = MVT::getVectorVT(VT: WideVT, NumElements: NumElts / 2);
4745 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
4746 Operand: DAG.getBuildVector(VT: WideVecVT, DL, Ops: NewOperands));
4747}
4748
4749static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
4750 const RISCVSubtarget &Subtarget) {
4751 MVT VT = Op.getSimpleValueType();
4752 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4753
4754 MVT EltVT = VT.getVectorElementType();
4755 MVT XLenVT = Subtarget.getXLenVT();
4756
4757 SDLoc DL(Op);
4758
4759 if (Subtarget.isRV32() && Subtarget.hasStdExtP()) {
4760 if (VT == MVT::v2i16) {
4761 SDValue Lo = DAG.getBitcast(
4762 VT: MVT::v2i16,
4763 V: DAG.getAnyExtOrTrunc(Op: Op->getOperand(Num: 0), DL, VT: MVT::i32));
4764 SDValue Hi = DAG.getBitcast(
4765 VT: MVT::v2i16,
4766 V: DAG.getAnyExtOrTrunc(Op: Op->getOperand(Num: 1), DL, VT: MVT::i32));
4767 return DAG.getNode(Opcode: RISCVISD::PPAIRE, DL, VT: MVT::v2i16, N1: Lo, N2: Hi);
4768 }
4769
4770 if (VT == MVT::v4i8) {
4771 // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b,
4772 // d))
4773 SDValue Val0 =
4774 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 0));
4775 SDValue Val1 =
4776 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 1));
4777 SDValue Val2 =
4778 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 2));
4779 SDValue Val3 =
4780 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 3));
4781 SDValue Concat1 =
4782 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i8, N1: Val0, N2: Val2);
4783 SDValue Concat2 =
4784 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i8, N1: Val1, N2: Val3);
4785 SDValue PPairE =
4786 DAG.getNode(Opcode: RISCVISD::PPAIRE, DL, VT: MVT::v8i8, N1: Concat1, N2: Concat2);
4787
4788 SDValue Lo = DAG.getExtractSubvector(DL, VT: MVT::v4i8, Vec: PPairE, Idx: 0);
4789 SDValue Hi = DAG.getExtractSubvector(DL, VT: MVT::v4i8, Vec: PPairE, Idx: 4);
4790
4791 return DAG.getBitcast(VT: MVT::v4i8,
4792 V: DAG.getNode(Opcode: RISCVISD::PPAIRE, DL, VT: MVT::v2i16,
4793 N1: DAG.getBitcast(VT: MVT::v2i16, V: Lo),
4794 N2: DAG.getBitcast(VT: MVT::v2i16, V: Hi)));
4795 }
4796
4797 llvm_unreachable("Unexpected RV32 P BUILD_VECTOR type");
4798 }
4799
4800 // Proper support for f16 requires Zvfh. bf16 always requires special
4801 // handling. We need to cast the scalar to integer and create an integer
4802 // build_vector.
4803 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
4804 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
4805 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
4806 SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
4807 for (const auto &[I, U] : enumerate(First: Op->ops())) {
4808 SDValue Elem = U.get();
4809 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4810 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4811 // Called by LegalizeDAG, we need to use XLenVT operations since we
4812 // can't create illegal types.
4813 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: Elem)) {
4814 // Manually constant fold so the integer build_vector can be lowered
4815 // better. Waiting for DAGCombine will be too late.
4816 APInt V =
4817 C->getValueAPF().bitcastToAPInt().sext(width: XLenVT.getSizeInBits());
4818 NewOps[I] = DAG.getConstant(Val: V, DL, VT: XLenVT);
4819 } else {
4820 NewOps[I] = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Elem);
4821 }
4822 } else {
4823 // Called by scalar type legalizer, we can use i16.
4824 NewOps[I] = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: I));
4825 }
4826 }
4827 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: IVT, Ops: NewOps);
4828 return DAG.getBitcast(VT, V: Res);
4829 }
4830
4831 if (ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode()) ||
4832 ISD::isBuildVectorOfConstantFPSDNodes(N: Op.getNode()))
4833 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
4834
4835 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4836
4837 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4838
4839 if (VT.getVectorElementType() == MVT::i1) {
4840 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
4841 // vector type, we have a legal equivalently-sized i8 type, so we can use
4842 // that.
4843 MVT WideVecVT = VT.changeVectorElementType(EltVT: MVT::i8);
4844 SDValue VecZero = DAG.getConstant(Val: 0, DL, VT: WideVecVT);
4845
4846 SDValue WideVec;
4847 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4848 // For a splat, perform a scalar truncate before creating the wider
4849 // vector.
4850 Splat = DAG.getNode(Opcode: ISD::AND, DL, VT: Splat.getValueType(), N1: Splat,
4851 N2: DAG.getConstant(Val: 1, DL, VT: Splat.getValueType()));
4852 WideVec = DAG.getSplatBuildVector(VT: WideVecVT, DL, Op: Splat);
4853 } else {
4854 SmallVector<SDValue, 8> Ops(Op->op_values());
4855 WideVec = DAG.getBuildVector(VT: WideVecVT, DL, Ops);
4856 SDValue VecOne = DAG.getConstant(Val: 1, DL, VT: WideVecVT);
4857 WideVec = DAG.getNode(Opcode: ISD::AND, DL, VT: WideVecVT, N1: WideVec, N2: VecOne);
4858 }
4859
4860 return DAG.getSetCC(DL, VT, LHS: WideVec, RHS: VecZero, Cond: ISD::SETNE);
4861 }
4862
4863 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4864 if (auto Gather = matchSplatAsGather(SplatVal: Splat, VT, DL, DAG, Subtarget))
4865 return Gather;
4866
4867 if (!VT.isFloatingPoint())
4868 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Splat);
4869
4870 // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
4871 // pressure at high LMUL.
4872 bool IsScalar = all_of(Range: Op->ops().drop_front(),
4873 P: [](const SDUse &U) { return U.get().isUndef(); });
4874 unsigned Opc =
4875 VT.isFloatingPoint()
4876 ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
4877 : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
4878 Splat =
4879 DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Splat, N3: VL);
4880 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
4881 }
4882
4883 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4884 return Res;
4885
4886 // If we're compiling for an exact VLEN value, we can split our work per
4887 // register in the register group.
4888 if (const auto VLen = Subtarget.getRealVLen();
4889 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
4890 MVT ElemVT = VT.getVectorElementType();
4891 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
4892 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
4893 MVT OneRegVT = MVT::getVectorVT(VT: ElemVT, NumElements: ElemsPerVReg);
4894 MVT M1VT = getContainerForFixedLengthVector(VT: OneRegVT, Subtarget);
4895 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
4896
4897 // The following semantically builds up a fixed length concat_vector
4898 // of the component build_vectors. We eagerly lower to scalable and
4899 // insert_subvector here to avoid DAG combining it back to a large
4900 // build_vector.
4901 SmallVector<SDValue> BuildVectorOps(Op->ops());
4902 unsigned NumOpElts = M1VT.getVectorMinNumElements();
4903 SDValue Vec = DAG.getUNDEF(VT: ContainerVT);
4904 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4905 auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(N: i, M: ElemsPerVReg);
4906 SDValue SubBV =
4907 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: OneRegVT, Ops: OneVRegOfOps);
4908 SubBV = convertToScalableVector(VT: M1VT, V: SubBV, DAG, Subtarget);
4909 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4910 Vec = DAG.getInsertSubvector(DL, Vec, SubVec: SubBV, Idx: InsertIdx);
4911 }
4912 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
4913 }
4914
4915 // If we're about to resort to vslide1down (or stack usage), pack our
4916 // elements into the widest scalar type we can. This will force a VL/VTYPE
4917 // toggle, but reduces the critical path, the number of vslide1down ops
4918 // required, and possibly enables scalar folds of the values.
4919 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4920 return Res;
4921
4922 // For m1 vectors, if we have non-undef values in both halves of our vector,
4923 // split the vector into low and high halves, build them separately, then
4924 // use a vselect to combine them. For long vectors, this cuts the critical
4925 // path of the vslide1down sequence in half, and gives us an opportunity
4926 // to special case each half independently. Note that we don't change the
4927 // length of the sub-vectors here, so if both fallback to the generic
4928 // vslide1down path, we should be able to fold the vselect into the final
4929 // vslidedown (for the undef tail) for the first half w/ masking.
4930 unsigned NumElts = VT.getVectorNumElements();
4931 unsigned NumUndefElts =
4932 count_if(Range: Op->op_values(), P: [](const SDValue &V) { return V.isUndef(); });
4933 unsigned NumDefElts = NumElts - NumUndefElts;
4934 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4935 ContainerVT.bitsLE(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT))) {
4936 SmallVector<SDValue> SubVecAOps, SubVecBOps;
4937 SmallVector<SDValue> MaskVals;
4938 SDValue UndefElem = DAG.getUNDEF(VT: Op->getOperand(Num: 0)->getValueType(ResNo: 0));
4939 SubVecAOps.reserve(N: NumElts);
4940 SubVecBOps.reserve(N: NumElts);
4941 for (const auto &[Idx, U] : enumerate(First: Op->ops())) {
4942 SDValue Elem = U.get();
4943 if (Idx < NumElts / 2) {
4944 SubVecAOps.push_back(Elt: Elem);
4945 SubVecBOps.push_back(Elt: UndefElem);
4946 } else {
4947 SubVecAOps.push_back(Elt: UndefElem);
4948 SubVecBOps.push_back(Elt: Elem);
4949 }
4950 bool SelectMaskVal = (Idx < NumElts / 2);
4951 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
4952 }
4953 assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
4954 MaskVals.size() == NumElts);
4955
4956 SDValue SubVecA = DAG.getBuildVector(VT, DL, Ops: SubVecAOps);
4957 SDValue SubVecB = DAG.getBuildVector(VT, DL, Ops: SubVecBOps);
4958 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
4959 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
4960 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: SubVecA, N3: SubVecB);
4961 }
4962
4963 // Cap the cost at a value linear to the number of elements in the vector.
4964 // The default lowering is to use the stack. The vector store + scalar loads
4965 // is linear in VL. However, at high lmuls vslide1down and vslidedown end up
4966 // being (at least) linear in LMUL. As a result, using the vslidedown
4967 // lowering for every element ends up being VL*LMUL..
4968 // TODO: Should we be directly costing the stack alternative? Doing so might
4969 // give us a more accurate upper bound.
4970 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
4971
4972 // TODO: unify with TTI getSlideCost.
4973 InstructionCost PerSlideCost = 1;
4974 switch (RISCVTargetLowering::getLMUL(VT: ContainerVT)) {
4975 default: break;
4976 case RISCVVType::LMUL_2:
4977 PerSlideCost = 2;
4978 break;
4979 case RISCVVType::LMUL_4:
4980 PerSlideCost = 4;
4981 break;
4982 case RISCVVType::LMUL_8:
4983 PerSlideCost = 8;
4984 break;
4985 }
4986
4987 // TODO: Should we be using the build instseq then cost + evaluate scheme
4988 // we use for integer constants here?
4989 unsigned UndefCount = 0;
4990 for (const SDValue &V : Op->ops()) {
4991 if (V.isUndef()) {
4992 UndefCount++;
4993 continue;
4994 }
4995 if (UndefCount) {
4996 LinearBudget -= PerSlideCost;
4997 UndefCount = 0;
4998 }
4999 LinearBudget -= PerSlideCost;
5000 }
5001 if (UndefCount) {
5002 LinearBudget -= PerSlideCost;
5003 }
5004
5005 if (LinearBudget < 0)
5006 return SDValue();
5007
5008 assert((!VT.isFloatingPoint() ||
5009 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
5010 "Illegal type which will result in reserved encoding");
5011
5012 const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
5013
5014 // General case: splat the first operand and slide other operands down one
5015 // by one to form a vector. Alternatively, if every operand is an
5016 // extraction from element 0 of a vector, we use that vector from the last
5017 // extraction as the start value and slide up instead of slide down. Such that
5018 // (1) we can avoid the initial splat (2) we can turn those vslide1up into
5019 // vslideup of 1 later and eliminate the vector to scalar movement, which is
5020 // something we cannot do with vslide1down/vslidedown.
5021 // Of course, using vslide1up/vslideup might increase the register pressure,
5022 // and that's why we conservatively limit to cases where every operand is an
5023 // extraction from the first element.
5024 SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
5025 SDValue EVec;
5026 bool SlideUp = false;
5027 auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
5028 SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
5029 if (SlideUp)
5030 return getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: Vec, Offset,
5031 Mask, VL, Policy);
5032 return getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: Vec, Offset,
5033 Mask, VL, Policy);
5034 };
5035
5036 // The reason we don't use all_of here is because we're also capturing EVec
5037 // from the last non-undef operand. If the std::execution_policy of the
5038 // underlying std::all_of is anything but std::sequenced_policy we might
5039 // capture the wrong EVec.
5040 for (SDValue V : Operands) {
5041 using namespace SDPatternMatch;
5042 SlideUp = V.isUndef() || sd_match(N: V, P: m_ExtractElt(Vec: m_Value(N&: EVec), Idx: m_Zero()));
5043 if (!SlideUp)
5044 break;
5045 }
5046
5047 // Do not slideup if the element type of EVec is different.
5048 if (SlideUp) {
5049 MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
5050 MVT ContainerEltVT = ContainerVT.getVectorElementType();
5051 if (EVecEltVT != ContainerEltVT)
5052 SlideUp = false;
5053 }
5054
5055 if (SlideUp) {
5056 MVT EVecContainerVT = EVec.getSimpleValueType();
5057 // Make sure the original vector has scalable vector type.
5058 if (EVecContainerVT.isFixedLengthVector()) {
5059 EVecContainerVT =
5060 getContainerForFixedLengthVector(VT: EVecContainerVT, Subtarget);
5061 EVec = convertToScalableVector(VT: EVecContainerVT, V: EVec, DAG, Subtarget);
5062 }
5063
5064 // Adapt EVec's type into ContainerVT.
5065 if (EVecContainerVT.getVectorMinNumElements() <
5066 ContainerVT.getVectorMinNumElements())
5067 EVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec: EVec, Idx: 0);
5068 else
5069 EVec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec: EVec, Idx: 0);
5070
5071 // Reverse the elements as we're going to slide up from the last element.
5072 std::reverse(first: Operands.begin(), last: Operands.end());
5073 }
5074
5075 SDValue Vec;
5076 UndefCount = 0;
5077 for (SDValue V : Operands) {
5078 if (V.isUndef()) {
5079 UndefCount++;
5080 continue;
5081 }
5082
5083 // Start our sequence with either a TA splat or extract source in the
5084 // hopes that hardware is able to recognize there's no dependency on the
5085 // prior value of our temporary register.
5086 if (!Vec) {
5087 if (SlideUp) {
5088 Vec = EVec;
5089 } else {
5090 Vec = DAG.getSplatVector(VT, DL, Op: V);
5091 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
5092 }
5093
5094 UndefCount = 0;
5095 continue;
5096 }
5097
5098 if (UndefCount) {
5099 const SDValue Offset = DAG.getConstant(Val: UndefCount, DL, VT: Subtarget.getXLenVT());
5100 Vec = getVSlide(ContainerVT, DAG.getUNDEF(VT: ContainerVT), Vec, Offset, Mask,
5101 VL);
5102 UndefCount = 0;
5103 }
5104
5105 unsigned Opcode;
5106 if (VT.isFloatingPoint())
5107 Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
5108 else
5109 Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
5110
5111 if (!VT.isFloatingPoint())
5112 V = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: V);
5113 Vec = DAG.getNode(Opcode, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Vec,
5114 N3: V, N4: Mask, N5: VL);
5115 }
5116 if (UndefCount) {
5117 const SDValue Offset = DAG.getConstant(Val: UndefCount, DL, VT: Subtarget.getXLenVT());
5118 Vec = getVSlide(ContainerVT, DAG.getUNDEF(VT: ContainerVT), Vec, Offset, Mask,
5119 VL);
5120 }
5121 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5122}
5123
5124static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5125 SDValue Lo, SDValue Hi, SDValue VL,
5126 SelectionDAG &DAG) {
5127 if (!Passthru)
5128 Passthru = DAG.getUNDEF(VT);
5129 if (isa<ConstantSDNode>(Val: Lo) && isa<ConstantSDNode>(Val: Hi)) {
5130 int32_t LoC = cast<ConstantSDNode>(Val&: Lo)->getSExtValue();
5131 int32_t HiC = cast<ConstantSDNode>(Val&: Hi)->getSExtValue();
5132 // If Hi constant is all the same sign bit as Lo, lower this as a custom
5133 // node in order to try and match RVV vector/scalar instructions.
5134 if ((LoC >> 31) == HiC)
5135 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5136
5137 // Use vmv.v.x with EEW=32. Use either a vsetivli or vsetvli to change
5138 // VL. This can temporarily increase VL if VL less than VLMAX.
5139 if (LoC == HiC) {
5140 SDValue NewVL;
5141 if (isa<ConstantSDNode>(Val: VL) && isUInt<4>(x: VL->getAsZExtVal()))
5142 NewVL = DAG.getNode(Opcode: ISD::ADD, DL, VT: VL.getValueType(), N1: VL, N2: VL);
5143 else
5144 NewVL = DAG.getRegister(Reg: RISCV::X0, VT: MVT::i32);
5145 MVT InterVT =
5146 MVT::getVectorVT(VT: MVT::i32, EC: VT.getVectorElementCount() * 2);
5147 auto InterVec = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: InterVT,
5148 N1: DAG.getUNDEF(VT: InterVT), N2: Lo, N3: NewVL);
5149 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: InterVec);
5150 }
5151 }
5152
5153 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
5154 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(i: 0) == Lo &&
5155 isa<ConstantSDNode>(Val: Hi.getOperand(i: 1)) &&
5156 Hi.getConstantOperandVal(i: 1) == 31)
5157 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5158
5159 // If the hi bits of the splat are undefined, then it's fine to just splat Lo
5160 // even if it might be sign extended.
5161 if (Hi.isUndef())
5162 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5163
5164 // Fall back to a stack store and stride x0 vector load.
5165 return DAG.getNode(Opcode: RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, N1: Passthru, N2: Lo,
5166 N3: Hi, N4: VL);
5167}
5168
5169// Called by type legalization to handle splat of i64 on RV32.
5170// FIXME: We can optimize this when the type has sign or zero bits in one
5171// of the halves.
5172static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5173 SDValue Scalar, SDValue VL,
5174 SelectionDAG &DAG) {
5175 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
5176 SDValue Lo, Hi;
5177 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Scalar, DL, LoVT: MVT::i32, HiVT: MVT::i32);
5178 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
5179}
5180
5181// This function lowers a splat of a scalar operand Splat with the vector
5182// length VL. It ensures the final sequence is type legal, which is useful when
5183// lowering a splat after type legalization.
5184static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
5185 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
5186 const RISCVSubtarget &Subtarget) {
5187 bool HasPassthru = Passthru && !Passthru.isUndef();
5188 if (!HasPassthru && !Passthru)
5189 Passthru = DAG.getUNDEF(VT);
5190
5191 MVT EltVT = VT.getVectorElementType();
5192 MVT XLenVT = Subtarget.getXLenVT();
5193
5194 if (VT.isFloatingPoint()) {
5195 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
5196 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
5197 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
5198 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
5199 Scalar = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Scalar);
5200 else
5201 Scalar = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Scalar);
5202 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
5203 Passthru = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Passthru);
5204 SDValue Splat =
5205 lowerScalarSplat(Passthru, Scalar, VL, VT: IVT, DL, DAG, Subtarget);
5206 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Splat);
5207 }
5208 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
5209 }
5210
5211 // Simplest case is that the operand needs to be promoted to XLenVT.
5212 if (Scalar.getValueType().bitsLE(VT: XLenVT)) {
5213 // If the operand is a constant, sign extend to increase our chances
5214 // of being able to use a .vi instruction. ANY_EXTEND would become a
5215 // a zero extend and the simm5 check in isel would fail.
5216 // FIXME: Should we ignore the upper bits in isel instead?
5217 unsigned ExtOpc =
5218 isa<ConstantSDNode>(Val: Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5219 Scalar = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: Scalar);
5220 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
5221 }
5222
5223 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
5224 "Unexpected scalar for splat lowering!");
5225
5226 if (isOneConstant(V: VL) && isNullConstant(V: Scalar))
5227 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: Passthru,
5228 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: VL);
5229
5230 // Otherwise use the more complicated splatting algorithm.
5231 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
5232}
5233
5234// This function lowers an insert of a scalar operand Scalar into lane
5235// 0 of the vector regardless of the value of VL. The contents of the
5236// remaining lanes of the result vector are unspecified. VL is assumed
5237// to be non-zero.
5238static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
5239 const SDLoc &DL, SelectionDAG &DAG,
5240 const RISCVSubtarget &Subtarget) {
5241 assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
5242
5243 const MVT XLenVT = Subtarget.getXLenVT();
5244 SDValue Passthru = DAG.getUNDEF(VT);
5245
5246 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5247 isNullConstant(V: Scalar.getOperand(i: 1))) {
5248 SDValue ExtractedVal = Scalar.getOperand(i: 0);
5249 // The element types must be the same.
5250 if (ExtractedVal.getValueType().getVectorElementType() ==
5251 VT.getVectorElementType()) {
5252 MVT ExtractedVT = ExtractedVal.getSimpleValueType();
5253 MVT ExtractedContainerVT = ExtractedVT;
5254 if (ExtractedContainerVT.isFixedLengthVector()) {
5255 ExtractedContainerVT =
5256 getContainerForFixedLengthVector(VT: ExtractedContainerVT, Subtarget);
5257 ExtractedVal = convertToScalableVector(VT: ExtractedContainerVT,
5258 V: ExtractedVal, DAG, Subtarget);
5259 }
5260 if (ExtractedContainerVT.bitsLE(VT))
5261 return DAG.getInsertSubvector(DL, Vec: Passthru, SubVec: ExtractedVal, Idx: 0);
5262 return DAG.getExtractSubvector(DL, VT, Vec: ExtractedVal, Idx: 0);
5263 }
5264 }
5265
5266 if (VT.isFloatingPoint())
5267 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT, N1: DAG.getUNDEF(VT), N2: Scalar,
5268 N3: VL);
5269
5270 // Avoid the tricky legalization cases by falling back to using the
5271 // splat code which already handles it gracefully.
5272 if (!Scalar.getValueType().bitsLE(VT: XLenVT))
5273 return lowerScalarSplat(Passthru: DAG.getUNDEF(VT), Scalar,
5274 VL: DAG.getConstant(Val: 1, DL, VT: XLenVT),
5275 VT, DL, DAG, Subtarget);
5276
5277 // If the operand is a constant, sign extend to increase our chances
5278 // of being able to use a .vi instruction. ANY_EXTEND would become a
5279 // a zero extend and the simm5 check in isel would fail.
5280 // FIXME: Should we ignore the upper bits in isel instead?
5281 unsigned ExtOpc =
5282 isa<ConstantSDNode>(Val: Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5283 Scalar = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: Scalar);
5284 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: DAG.getUNDEF(VT), N2: Scalar,
5285 N3: VL);
5286}
5287
5288/// If concat_vector(V1,V2) could be folded away to some existing
5289/// vector source, return it. Note that the source may be larger
5290/// than the requested concat_vector (i.e. a extract_subvector
5291/// might be required.)
5292static SDValue foldConcatVector(SDValue V1, SDValue V2) {
5293 EVT VT = V1.getValueType();
5294 assert(VT == V2.getValueType() && "argument types must match");
5295 // Both input must be extracts.
5296 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5297 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
5298 return SDValue();
5299
5300 // Extracting from the same source.
5301 SDValue Src = V1.getOperand(i: 0);
5302 if (Src != V2.getOperand(i: 0) ||
5303 VT.isScalableVector() != Src.getValueType().isScalableVector())
5304 return SDValue();
5305
5306 // The extracts must extract the two halves of the source.
5307 if (V1.getConstantOperandVal(i: 1) != 0 ||
5308 V2.getConstantOperandVal(i: 1) != VT.getVectorMinNumElements())
5309 return SDValue();
5310
5311 return Src;
5312}
5313
5314// Can this shuffle be performed on exactly one (possibly larger) input?
5315static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
5316
5317 if (V2.isUndef())
5318 return V1;
5319
5320 unsigned NumElts = VT.getVectorNumElements();
5321 // Src needs to have twice the number of elements.
5322 // TODO: Update shuffle lowering to add the extract subvector
5323 if (SDValue Src = foldConcatVector(V1, V2);
5324 Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
5325 return Src;
5326
5327 return SDValue();
5328}
5329
5330static bool isLegalVTForZvzipOperand(MVT VT, const RISCVSubtarget &Subtarget) {
5331 MVT ContainerVT = VT;
5332 if (VT.isFixedLengthVector())
5333 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5334 // Determine LMUL of the container vector.
5335 return RISCVTargetLowering::getLMUL(VT: ContainerVT) != RISCVVType::LMUL_8;
5336}
5337
5338/// Is this shuffle interleaving contiguous elements from one vector into the
5339/// even elements and contiguous elements from another vector into the odd
5340/// elements. \p EvenSrc will contain the element that should be in the first
5341/// even element. \p OddSrc will contain the element that should be in the first
5342/// odd element. These can be the first element in a source or the element half
5343/// way through the source.
5344static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
5345 int &OddSrc, const RISCVSubtarget &Subtarget) {
5346 // We need to be able to widen elements to the next larger integer type or
5347 // use the vzip instruction at e64.
5348 if (VT.getScalarSizeInBits() >= Subtarget.getELen()) {
5349 if (!Subtarget.hasStdExtZvzip())
5350 return false;
5351 if (!isLegalVTForZvzipOperand(VT, Subtarget))
5352 return false;
5353 }
5354
5355 int Size = Mask.size();
5356 int NumElts = VT.getVectorNumElements();
5357 assert(Size == (int)NumElts && "Unexpected mask size");
5358
5359 SmallVector<unsigned, 2> StartIndexes;
5360 if (!ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Size * 2, StartIndexes))
5361 return false;
5362
5363 EvenSrc = StartIndexes[0];
5364 OddSrc = StartIndexes[1];
5365
5366 // One source should be low half of first vector.
5367 if (EvenSrc != 0 && OddSrc != 0)
5368 return false;
5369
5370 // Subvectors will be subtracted from either at the start of the two input
5371 // vectors, or at the start and middle of the first vector if it's an unary
5372 // interleave.
5373 // In both cases, HalfNumElts will be extracted.
5374 // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
5375 // we'll create an illegal extract_subvector.
5376 // FIXME: We could support other values using a slidedown first.
5377 int HalfNumElts = NumElts / 2;
5378 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
5379}
5380
5381/// Is this mask representing a masked combination of two slides?
5382static bool isMaskedSlidePair(ArrayRef<int> Mask,
5383 std::array<std::pair<int, int>, 2> &SrcInfo) {
5384 if (!llvm::isMaskedSlidePair(Mask, NumElts: Mask.size(), SrcInfo))
5385 return false;
5386
5387 // Avoid matching vselect idioms
5388 if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
5389 return false;
5390 // Prefer vslideup as the second instruction, and identity
5391 // only as the initial instruction.
5392 if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
5393 SrcInfo[1].second == 0)
5394 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
5395 assert(SrcInfo[0].first != -1 && "Must find one slide");
5396 return true;
5397}
5398
5399// Exactly matches the semantics of a previously existing custom matcher
5400// to allow migration to new matcher without changing output.
5401static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
5402 unsigned NumElts) {
5403 if (SrcInfo[1].first == -1)
5404 return true;
5405 return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
5406 SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
5407}
5408
5409static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
5410 ArrayRef<int> Mask, unsigned Factor,
5411 bool RequiredPolarity) {
5412 int NumElts = Mask.size();
5413 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
5414 if (M < 0)
5415 continue;
5416 int Src = M >= NumElts;
5417 int Diff = (int)Idx - (M % NumElts);
5418 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
5419 assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
5420 "Must match exactly one of the two slides");
5421 if (RequiredPolarity != (C == (Idx / Factor) % 2))
5422 return false;
5423 }
5424 return true;
5425}
5426
5427/// Given a shuffle which can be represented as a pair of two slides,
5428/// see if it is a pair-even idiom.
5429/// Pair-even is:
5430/// vs2: a0 a1 a2 a3
5431/// vs1: b0 b1 b2 b3
5432/// vd: a0 b0 a2 b2
5433static bool isPairEven(const std::array<std::pair<int, int>, 2> &SrcInfo,
5434 ArrayRef<int> Mask, unsigned &Factor) {
5435 Factor = SrcInfo[1].second;
5436 return SrcInfo[0].second == 0 && isPowerOf2_32(Value: Factor) &&
5437 Mask.size() % Factor == 0 &&
5438 isAlternating(SrcInfo, Mask, Factor, RequiredPolarity: true);
5439}
5440
5441/// Given a shuffle which can be represented as a pair of two slides,
5442/// see if it is a pair-odd idiom.
5443/// Pair-odd is:
5444/// vs2: a0 a1 a2 a3
5445/// vs1: b0 b1 b2 b3
5446/// vd: a1 b1 a3 b3
5447/// Note that the operand order is swapped due to the way we canonicalize
5448/// the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.
5449static bool isPairOdd(const std::array<std::pair<int, int>, 2> &SrcInfo,
5450 ArrayRef<int> Mask, unsigned &Factor) {
5451 Factor = -SrcInfo[1].second;
5452 return SrcInfo[0].second == 0 && isPowerOf2_32(Value: Factor) &&
5453 Mask.size() % Factor == 0 &&
5454 isAlternating(SrcInfo, Mask, Factor, RequiredPolarity: false);
5455}
5456
5457// Lower a deinterleave shuffle to SRL and TRUNC. Factor must be
5458// 2, 4, 8 and the integer type Factor-times larger than VT's
5459// element type must be a legal element type.
5460// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
5461// -> [p, q, r, s] (Factor=2, Index=1)
5462static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
5463 SDValue Src, unsigned Factor,
5464 unsigned Index, SelectionDAG &DAG) {
5465 unsigned EltBits = VT.getScalarSizeInBits();
5466 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
5467 MVT WideSrcVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits * Factor),
5468 EC: SrcEC.divideCoefficientBy(RHS: Factor));
5469 MVT ResVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits),
5470 EC: SrcEC.divideCoefficientBy(RHS: Factor));
5471 Src = DAG.getBitcast(VT: WideSrcVT, V: Src);
5472
5473 unsigned Shift = Index * EltBits;
5474 SDValue Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: WideSrcVT, N1: Src,
5475 N2: DAG.getConstant(Val: Shift, DL, VT: WideSrcVT));
5476 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResVT, Operand: Res);
5477 MVT CastVT = ResVT.changeVectorElementType(EltVT: VT.getVectorElementType());
5478 Res = DAG.getBitcast(VT: CastVT, V: Res);
5479 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: Res, Idx: 0);
5480}
5481
5482/// Match a single source shuffle which is an identity except that some
5483/// particular element is repeated. This can be lowered as a masked
5484/// vrgather.vi/vx. Note that the two source form of this is handled
5485/// by the recursive splitting logic and doesn't need special handling.
5486static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
5487 const RISCVSubtarget &Subtarget,
5488 SelectionDAG &DAG) {
5489
5490 SDLoc DL(SVN);
5491 MVT VT = SVN->getSimpleValueType(ResNo: 0);
5492 SDValue V1 = SVN->getOperand(Num: 0);
5493 assert(SVN->getOperand(1).isUndef());
5494 ArrayRef<int> Mask = SVN->getMask();
5495 const unsigned NumElts = VT.getVectorNumElements();
5496 MVT XLenVT = Subtarget.getXLenVT();
5497
5498 std::optional<int> SplatIdx;
5499 for (auto [I, M] : enumerate(First&: Mask)) {
5500 if (M == -1 || I == (unsigned)M)
5501 continue;
5502 if (SplatIdx && *SplatIdx != M)
5503 return SDValue();
5504 SplatIdx = M;
5505 }
5506
5507 if (!SplatIdx)
5508 return SDValue();
5509
5510 SmallVector<SDValue> MaskVals;
5511 for (int MaskIndex : Mask) {
5512 bool SelectMaskVal = MaskIndex == *SplatIdx;
5513 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
5514 }
5515 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
5516 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
5517 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
5518 SDValue Splat = DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT),
5519 Mask: SmallVector<int>(NumElts, *SplatIdx));
5520 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: Splat, N3: V1);
5521}
5522
5523// Lower the following shuffle to vslidedown.
5524// a)
5525// t49: v8i8 = extract_subvector t13, Constant:i64<0>
5526// t109: v8i8 = extract_subvector t13, Constant:i64<8>
5527// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
5528// b)
5529// t69: v16i16 = extract_subvector t68, Constant:i64<0>
5530// t23: v8i16 = extract_subvector t69, Constant:i64<0>
5531// t29: v4i16 = extract_subvector t23, Constant:i64<4>
5532// t26: v8i16 = extract_subvector t69, Constant:i64<8>
5533// t30: v4i16 = extract_subvector t26, Constant:i64<0>
5534// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
5535static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
5536 SDValue V1, SDValue V2,
5537 ArrayRef<int> Mask,
5538 const RISCVSubtarget &Subtarget,
5539 SelectionDAG &DAG) {
5540 auto findNonEXTRACT_SUBVECTORParent =
5541 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
5542 uint64_t Offset = 0;
5543 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5544 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
5545 // a scalable vector. But we don't want to match the case.
5546 Parent.getOperand(i: 0).getSimpleValueType().isFixedLengthVector()) {
5547 Offset += Parent.getConstantOperandVal(i: 1);
5548 Parent = Parent.getOperand(i: 0);
5549 }
5550 return std::make_pair(x&: Parent, y&: Offset);
5551 };
5552
5553 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
5554 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
5555
5556 // Extracting from the same source.
5557 SDValue Src = V1Src;
5558 if (Src != V2Src)
5559 return SDValue();
5560
5561 // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
5562 SmallVector<int, 16> NewMask(Mask);
5563 for (size_t i = 0; i != NewMask.size(); ++i) {
5564 if (NewMask[i] == -1)
5565 continue;
5566
5567 if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
5568 NewMask[i] = NewMask[i] + V1IndexOffset;
5569 } else {
5570 // Minus NewMask.size() is needed. Otherwise, the b case would be
5571 // <5,6,7,12> instead of <5,6,7,8>.
5572 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
5573 }
5574 }
5575
5576 // First index must be known and non-zero. It will be used as the slidedown
5577 // amount.
5578 if (NewMask[0] <= 0)
5579 return SDValue();
5580
5581 // NewMask is also continuous.
5582 for (unsigned i = 1; i != NewMask.size(); ++i)
5583 if (NewMask[i - 1] + 1 != NewMask[i])
5584 return SDValue();
5585
5586 MVT XLenVT = Subtarget.getXLenVT();
5587 MVT SrcVT = Src.getSimpleValueType();
5588 MVT ContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
5589 auto [TrueMask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
5590 SDValue Slidedown =
5591 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru: DAG.getUNDEF(VT: ContainerVT),
5592 Op: convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget),
5593 Offset: DAG.getConstant(Val: NewMask[0], DL, VT: XLenVT), Mask: TrueMask, VL);
5594 return DAG.getExtractSubvector(
5595 DL, VT, Vec: convertFromScalableVector(VT: SrcVT, V: Slidedown, DAG, Subtarget), Idx: 0);
5596}
5597
5598// Because vslideup leaves the destination elements at the start intact, we can
5599// use it to perform shuffles that insert subvectors:
5600//
5601// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
5602// ->
5603// vsetvli zero, 8, e8, mf2, ta, ma
5604// vslideup.vi v8, v9, 4
5605//
5606// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
5607// ->
5608// vsetvli zero, 5, e8, mf2, tu, ma
5609// vslideup.v1 v8, v9, 2
5610static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
5611 SDValue V1, SDValue V2,
5612 ArrayRef<int> Mask,
5613 const RISCVSubtarget &Subtarget,
5614 SelectionDAG &DAG) {
5615 unsigned NumElts = VT.getVectorNumElements();
5616 int NumSubElts, Index;
5617 if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumSrcElts: NumElts, NumSubElts,
5618 Index))
5619 return SDValue();
5620
5621 bool OpsSwapped = Mask[Index] < (int)NumElts;
5622 SDValue InPlace = OpsSwapped ? V2 : V1;
5623 SDValue ToInsert = OpsSwapped ? V1 : V2;
5624
5625 MVT XLenVT = Subtarget.getXLenVT();
5626 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5627 auto TrueMask = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).first;
5628 // We slide up by the index that the subvector is being inserted at, and set
5629 // VL to the index + the number of elements being inserted.
5630 unsigned Policy =
5631 RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
5632 // If the we're adding a suffix to the in place vector, i.e. inserting right
5633 // up to the very end of it, then we don't actually care about the tail.
5634 if (NumSubElts + Index >= (int)NumElts)
5635 Policy |= RISCVVType::TAIL_AGNOSTIC;
5636
5637 InPlace = convertToScalableVector(VT: ContainerVT, V: InPlace, DAG, Subtarget);
5638 ToInsert = convertToScalableVector(VT: ContainerVT, V: ToInsert, DAG, Subtarget);
5639 SDValue VL = DAG.getConstant(Val: NumSubElts + Index, DL, VT: XLenVT);
5640
5641 SDValue Res;
5642 // If we're inserting into the lowest elements, use a tail undisturbed
5643 // vmv.v.v.
5644 if (Index == 0)
5645 Res = DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: ContainerVT, N1: InPlace, N2: ToInsert,
5646 N3: VL);
5647 else
5648 Res = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: InPlace, Op: ToInsert,
5649 Offset: DAG.getConstant(Val: Index, DL, VT: XLenVT), Mask: TrueMask, VL, Policy);
5650 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
5651}
5652
5653// A shuffle of shuffles where the final data only is drawn from 2 input ops
5654// can be compressed into a single shuffle
5655static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
5656 const RISCVSubtarget &Subtarget,
5657 SelectionDAG &DAG) {
5658 SDValue V1 = SVN->getOperand(Num: 0);
5659 SDValue V2 = SVN->getOperand(Num: 1);
5660
5661 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
5662 V2.getOpcode() != ISD::VECTOR_SHUFFLE)
5663 return SDValue();
5664
5665 if (!V1.hasOneUse() || !V2.hasOneUse())
5666 return SDValue();
5667
5668 ArrayRef<int> Mask = SVN->getMask();
5669 ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(Val: V1.getNode())->getMask();
5670 ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(Val: V2.getNode())->getMask();
5671 unsigned NumElts = Mask.size();
5672 SmallVector<int> NewMask(NumElts, -1);
5673 for (unsigned Idx : seq<unsigned>(Size: NumElts)) {
5674 int Lane = Mask[Idx];
5675 // Don't assign if poison
5676 if (Lane == -1)
5677 continue;
5678 int OrigLane;
5679 bool SecondOp = false;
5680 if ((unsigned)Lane < NumElts) {
5681 OrigLane = V1Mask[Lane];
5682 } else {
5683 OrigLane = V2Mask[Lane - NumElts];
5684 SecondOp = true;
5685 }
5686 if (OrigLane == -1)
5687 continue;
5688 // Don't handle if shuffling from a second operand
5689 if ((unsigned)OrigLane >= NumElts)
5690 return SDValue();
5691 if (SecondOp)
5692 OrigLane += NumElts;
5693 NewMask[Idx] = OrigLane;
5694 }
5695
5696 EVT VT = SVN->getValueType(ResNo: 0);
5697 SDLoc DL(SVN);
5698
5699 return DAG.getVectorShuffle(VT, dl: DL, N1: V1->getOperand(Num: 0), N2: V2->getOperand(Num: 0),
5700 Mask: NewMask);
5701}
5702
5703/// Match v(f)slide1up/down idioms. These operations involve sliding
5704/// N-1 elements to make room for an inserted scalar at one end.
5705static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
5706 SDValue V1, SDValue V2,
5707 ArrayRef<int> Mask,
5708 const RISCVSubtarget &Subtarget,
5709 SelectionDAG &DAG) {
5710 bool OpsSwapped = false;
5711 if (!isa<BuildVectorSDNode>(Val: V1)) {
5712 if (!isa<BuildVectorSDNode>(Val: V2))
5713 return SDValue();
5714 std::swap(a&: V1, b&: V2);
5715 OpsSwapped = true;
5716 }
5717 SDValue Splat = cast<BuildVectorSDNode>(Val&: V1)->getSplatValue();
5718 if (!Splat)
5719 return SDValue();
5720
5721 // Return true if the mask could describe a slide of Mask.size() - 1
5722 // elements from concat_vector(V1, V2)[Base:] to [Offset:].
5723 auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
5724 const unsigned S = (Offset > 0) ? 0 : -Offset;
5725 const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
5726 for (unsigned i = S; i != E; ++i)
5727 if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
5728 return false;
5729 return true;
5730 };
5731
5732 const unsigned NumElts = VT.getVectorNumElements();
5733 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
5734 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
5735 return SDValue();
5736
5737 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
5738 // Inserted lane must come from splat, undef scalar is legal but not profitable.
5739 if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
5740 return SDValue();
5741
5742 MVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
5743 auto [TrueMask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
5744
5745 // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
5746 // vslide1{down,up}.vx instead.
5747 if ((VT.getVectorElementType() == MVT::bf16 &&
5748 !Subtarget.hasVInstructionsBF16()) ||
5749 (VT.getVectorElementType() == MVT::f16 &&
5750 !Subtarget.hasVInstructionsF16())) {
5751 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
5752 Splat =
5753 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: Subtarget.getXLenVT(), Operand: Splat);
5754 V2 = DAG.getBitcast(
5755 VT: IntVT, V: convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget));
5756 SDValue Vec = DAG.getNode(
5757 Opcode: IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
5758 VT: IntVT, N1: DAG.getUNDEF(VT: IntVT), N2: V2, N3: Splat, N4: TrueMask, N5: VL);
5759 Vec = DAG.getBitcast(VT: ContainerVT, V: Vec);
5760 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5761 }
5762
5763 auto OpCode = IsVSlidedown ?
5764 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
5765 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
5766 if (!VT.isFloatingPoint())
5767 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: Splat);
5768 auto Vec = DAG.getNode(Opcode: OpCode, DL, VT: ContainerVT,
5769 N1: DAG.getUNDEF(VT: ContainerVT),
5770 N2: convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget),
5771 N3: Splat, N4: TrueMask, N5: VL);
5772 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5773}
5774
5775/// Match a mask which "spreads" the leading elements of a vector evenly
5776/// across the result. Factor is the spread amount, and Index is the
5777/// offset applied. (on success, Index < Factor) This is the inverse
5778/// of a deinterleave with the same Factor and Index. This is analogous
5779/// to an interleave, except that all but one lane is undef.
5780bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
5781 unsigned &Index) {
5782 SmallVector<bool> LaneIsUndef(Factor, true);
5783 for (unsigned i = 0; i < Mask.size(); i++)
5784 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
5785
5786 bool Found = false;
5787 for (unsigned i = 0; i < Factor; i++) {
5788 if (LaneIsUndef[i])
5789 continue;
5790 if (Found)
5791 return false;
5792 Index = i;
5793 Found = true;
5794 }
5795 if (!Found)
5796 return false;
5797
5798 for (unsigned i = 0; i < Mask.size() / Factor; i++) {
5799 unsigned j = i * Factor + Index;
5800 if (Mask[j] != -1 && (unsigned)Mask[j] != i)
5801 return false;
5802 }
5803 return true;
5804}
5805
5806static SDValue lowerZvzipVPAIR(unsigned Opc, SDValue Op0, SDValue Op1,
5807 const SDLoc &DL, SelectionDAG &DAG,
5808 const RISCVSubtarget &Subtarget) {
5809 assert(RISCVISD::VPAIRE_VL == Opc || RISCVISD::VPAIRO_VL == Opc);
5810 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5811
5812 MVT VT = Op0.getSimpleValueType();
5813 MVT IntVT = VT.changeVectorElementTypeToInteger();
5814 Op0 = DAG.getBitcast(VT: IntVT, V: Op0);
5815 Op1 = DAG.getBitcast(VT: IntVT, V: Op1);
5816
5817 MVT ContainerVT = IntVT;
5818 if (VT.isFixedLengthVector()) {
5819 ContainerVT = getContainerForFixedLengthVector(VT: IntVT, Subtarget);
5820 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
5821 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
5822 }
5823
5824 MVT InnerVT = ContainerVT;
5825 auto [Mask, VL] = getDefaultVLOps(VecVT: IntVT, ContainerVT: InnerVT, DL, DAG, Subtarget);
5826
5827 SDValue Passthru = DAG.getUNDEF(VT: InnerVT);
5828 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: InnerVT, N1: Op0, N2: Op1, N3: Passthru, N4: Mask, N5: VL);
5829 if (IntVT.isFixedLengthVector())
5830 Res = convertFromScalableVector(VT: IntVT, V: Res, DAG, Subtarget);
5831 Res = DAG.getBitcast(VT, V: Res);
5832 return Res;
5833}
5834
5835static SDValue lowerZvzipVZIP(SDValue Op0, SDValue Op1, const SDLoc &DL,
5836 SelectionDAG &DAG,
5837 const RISCVSubtarget &Subtarget) {
5838 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5839 MVT VT = Op0.getSimpleValueType();
5840 MVT IntVT = VT.changeVectorElementTypeToInteger();
5841 Op0 = DAG.getBitcast(VT: IntVT, V: Op0);
5842 Op1 = DAG.getBitcast(VT: IntVT, V: Op1);
5843 MVT ContainerVT = IntVT;
5844 if (VT.isFixedLengthVector()) {
5845 ContainerVT = getContainerForFixedLengthVector(VT: IntVT, Subtarget);
5846 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
5847 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
5848 }
5849 MVT ResVT = ContainerVT.getDoubleNumVectorElementsVT();
5850 auto [Mask, VL] = getDefaultVLOps(VecVT: IntVT, ContainerVT, DL, DAG, Subtarget);
5851 SDValue Passthru = DAG.getUNDEF(VT: ResVT);
5852 SDValue Res =
5853 DAG.getNode(Opcode: RISCVISD::VZIP_VL, DL, VT: ResVT, N1: Op0, N2: Op1, N3: Passthru, N4: Mask, N5: VL);
5854 if (IntVT.isFixedLengthVector())
5855 Res = convertFromScalableVector(VT: IntVT.getDoubleNumVectorElementsVT(), V: Res,
5856 DAG, Subtarget);
5857 Res = DAG.getBitcast(VT: VT.getDoubleNumVectorElementsVT(), V: Res);
5858 return Res;
5859}
5860
5861static SDValue lowerZvzipVUNZIP(unsigned Opc, SDValue Op, const SDLoc &DL,
5862 SelectionDAG &DAG,
5863 const RISCVSubtarget &Subtarget) {
5864 assert(Opc == RISCVISD::VUNZIPE_VL || Opc == RISCVISD::VUNZIPO_VL);
5865 MVT VT = Op.getSimpleValueType();
5866 assert(VT.getVectorMinNumElements() >= 2);
5867
5868 MVT IntVT = VT.changeVectorElementTypeToInteger();
5869 Op = DAG.getBitcast(VT: IntVT, V: Op);
5870 MVT ContainerVT = IntVT;
5871 if (VT.isFixedLengthVector()) {
5872 ContainerVT = getContainerForFixedLengthVector(VT: IntVT, Subtarget);
5873 // For E64 with LMUL <= 1, we can't represent a smaller fractional LMUL for
5874 // the result (LMUL <= 1/2 is not valid for E64). We must widen the input
5875 // container to at least LMUL=2 so the result can be LMUL=1.
5876 if (ContainerVT.getVectorElementType() == MVT::i64 &&
5877 RISCVTargetLowering::getLMUL(VT: ContainerVT) == RISCVVType::LMUL_1) {
5878 ContainerVT = MVT::getScalableVectorVT(VT: MVT::i64, NumElements: 2);
5879 }
5880 Op = convertToScalableVector(VT: ContainerVT, V: Op, DAG, Subtarget);
5881 }
5882
5883 MVT ResVT = ContainerVT.getHalfNumVectorElementsVT();
5884 MVT HalfVT = VT.getHalfNumVectorElementsVT();
5885 MVT HalfIntVT = IntVT.getHalfNumVectorElementsVT();
5886 auto [Mask, VL] = getDefaultVLOps(VecVT: ResVT, ContainerVT: ResVT, DL, DAG, Subtarget);
5887 if (VT.isFixedLengthVector())
5888 VL = DAG.getConstant(Val: VT.getVectorNumElements() / 2, DL,
5889 VT: Subtarget.getXLenVT());
5890 SDValue Passthru = DAG.getUNDEF(VT: ResVT);
5891 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ResVT, N1: Op, N2: Passthru, N3: Mask, N4: VL);
5892 if (HalfIntVT.isFixedLengthVector())
5893 Res = convertFromScalableVector(VT: HalfIntVT, V: Res, DAG, Subtarget);
5894 Res = DAG.getBitcast(VT: HalfVT, V: Res);
5895 return Res;
5896}
5897
5898// Given a vector a, b, c, d return a vector Factor times longer
5899// with Factor-1 undef's between elements. Ex:
5900// a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
5901// undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
5902static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
5903 const SDLoc &DL, SelectionDAG &DAG) {
5904
5905 MVT VT = V.getSimpleValueType();
5906 unsigned EltBits = VT.getScalarSizeInBits();
5907 ElementCount EC = VT.getVectorElementCount();
5908 V = DAG.getBitcast(VT: VT.changeTypeToInteger(), V);
5909
5910 MVT WideVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits * Factor), EC);
5911
5912 SDValue Result = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: V);
5913 // TODO: On rv32, the constant becomes a splat_vector_parts which does not
5914 // allow the SHL to fold away if Index is 0.
5915 if (Index != 0)
5916 Result = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Result,
5917 N2: DAG.getConstant(Val: EltBits * Index, DL, VT: WideVT));
5918 // Make sure to use original element type
5919 MVT ResultVT = MVT::getVectorVT(VT: VT.getVectorElementType(),
5920 EC: EC.multiplyCoefficientBy(RHS: Factor));
5921 return DAG.getBitcast(VT: ResultVT, V: Result);
5922}
5923
5924// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
5925// to create an interleaved vector of <[vscale x] n*2 x ty>.
5926// This requires that the size of ty is less than the subtarget's maximum ELEN.
5927static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
5928 const SDLoc &DL, SelectionDAG &DAG,
5929 const RISCVSubtarget &Subtarget) {
5930
5931 // FIXME: Not only does this optimize the code, it fixes some correctness
5932 // issues because MIR does not have freeze.
5933 if (EvenV.isUndef())
5934 return getWideningSpread(V: OddV, Factor: 2, Index: 1, DL, DAG);
5935 if (OddV.isUndef())
5936 return getWideningSpread(V: EvenV, Factor: 2, Index: 0, DL, DAG);
5937
5938 MVT VecVT = EvenV.getSimpleValueType();
5939 MVT VecContainerVT = VecVT; // <vscale x n x ty>
5940 // Convert fixed vectors to scalable if needed
5941 if (VecContainerVT.isFixedLengthVector()) {
5942 VecContainerVT = getContainerForFixedLengthVector(VT: VecVT, Subtarget);
5943 EvenV = convertToScalableVector(VT: VecContainerVT, V: EvenV, DAG, Subtarget);
5944 OddV = convertToScalableVector(VT: VecContainerVT, V: OddV, DAG, Subtarget);
5945 }
5946
5947 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
5948
5949 // We're working with a vector of the same size as the resulting
5950 // interleaved vector, but with half the number of elements and
5951 // twice the SEW (Hence the restriction on not using the maximum
5952 // ELEN)
5953 MVT WideVT =
5954 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VecVT.getScalarSizeInBits() * 2),
5955 EC: VecVT.getVectorElementCount());
5956 MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
5957 if (WideContainerVT.isFixedLengthVector())
5958 WideContainerVT = getContainerForFixedLengthVector(VT: WideVT, Subtarget);
5959
5960 // Bitcast the input vectors to integers in case they are FP
5961 VecContainerVT = VecContainerVT.changeTypeToInteger();
5962 EvenV = DAG.getBitcast(VT: VecContainerVT, V: EvenV);
5963 OddV = DAG.getBitcast(VT: VecContainerVT, V: OddV);
5964
5965 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT: VecContainerVT, DL, DAG, Subtarget);
5966 SDValue Passthru = DAG.getUNDEF(VT: WideContainerVT);
5967
5968 SDValue Interleaved;
5969 if (Subtarget.hasStdExtZvbb()) {
5970 // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
5971 SDValue OffsetVec =
5972 DAG.getConstant(Val: VecVT.getScalarSizeInBits(), DL, VT: VecContainerVT);
5973 Interleaved = DAG.getNode(Opcode: RISCVISD::VWSLL_VL, DL, VT: WideContainerVT, N1: OddV,
5974 N2: OffsetVec, N3: Passthru, N4: Mask, N5: VL);
5975 Interleaved = DAG.getNode(Opcode: RISCVISD::VWADDU_W_VL, DL, VT: WideContainerVT,
5976 N1: Interleaved, N2: EvenV, N3: Passthru, N4: Mask, N5: VL);
5977 } else {
5978 // FIXME: We should freeze the odd vector here. We already handled the case
5979 // of provably undef/poison above.
5980
5981 // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
5982 // vwaddu.vv
5983 Interleaved = DAG.getNode(Opcode: RISCVISD::VWADDU_VL, DL, VT: WideContainerVT, N1: EvenV,
5984 N2: OddV, N3: Passthru, N4: Mask, N5: VL);
5985
5986 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
5987 SDValue AllOnesVec = DAG.getSplatVector(
5988 VT: VecContainerVT, DL, Op: DAG.getAllOnesConstant(DL, VT: Subtarget.getXLenVT()));
5989 SDValue OddsMul = DAG.getNode(Opcode: RISCVISD::VWMULU_VL, DL, VT: WideContainerVT,
5990 N1: OddV, N2: AllOnesVec, N3: Passthru, N4: Mask, N5: VL);
5991
5992 // Add the two together so we get
5993 // (OddV * 0xff...ff) + (OddV + EvenV)
5994 // = (OddV * 0x100...00) + EvenV
5995 // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
5996 // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
5997 Interleaved = DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT: WideContainerVT,
5998 N1: Interleaved, N2: OddsMul, N3: Passthru, N4: Mask, N5: VL);
5999 }
6000
6001 // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
6002 MVT ResultContainerVT = MVT::getVectorVT(
6003 VT: VecVT.getVectorElementType(), // Make sure to use original type
6004 EC: VecContainerVT.getVectorElementCount().multiplyCoefficientBy(RHS: 2));
6005 Interleaved = DAG.getBitcast(VT: ResultContainerVT, V: Interleaved);
6006
6007 // Convert back to a fixed vector if needed
6008 MVT ResultVT =
6009 MVT::getVectorVT(VT: VecVT.getVectorElementType(),
6010 EC: VecVT.getVectorElementCount().multiplyCoefficientBy(RHS: 2));
6011 if (ResultVT.isFixedLengthVector())
6012 Interleaved =
6013 convertFromScalableVector(VT: ResultVT, V: Interleaved, DAG, Subtarget);
6014
6015 return Interleaved;
6016}
6017
6018// If we have a vector of bits that we want to reverse, we can use a vbrev on a
6019// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
6020static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
6021 SelectionDAG &DAG,
6022 const RISCVSubtarget &Subtarget) {
6023 SDLoc DL(SVN);
6024 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6025 SDValue V = SVN->getOperand(Num: 0);
6026 unsigned NumElts = VT.getVectorNumElements();
6027
6028 assert(VT.getVectorElementType() == MVT::i1);
6029
6030 if (!ShuffleVectorInst::isReverseMask(Mask: SVN->getMask(),
6031 NumSrcElts: SVN->getMask().size()) ||
6032 !SVN->getOperand(Num: 1).isUndef())
6033 return SDValue();
6034
6035 unsigned ViaEltSize = std::max(a: (uint64_t)8, b: PowerOf2Ceil(A: NumElts));
6036 EVT ViaVT = EVT::getVectorVT(
6037 Context&: *DAG.getContext(), VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ViaEltSize), NumElements: 1);
6038 EVT ViaBitVT =
6039 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: ViaVT.getScalarSizeInBits());
6040
6041 // If we don't have zvbb or the larger element type > ELEN, the operation will
6042 // be illegal.
6043 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(Op: ISD::BITREVERSE,
6044 VT: ViaVT) ||
6045 !Subtarget.getTargetLowering()->isTypeLegal(VT: ViaBitVT))
6046 return SDValue();
6047
6048 // If the bit vector doesn't fit exactly into the larger element type, we need
6049 // to insert it into the larger vector and then shift up the reversed bits
6050 // afterwards to get rid of the gap introduced.
6051 if (ViaEltSize > NumElts)
6052 V = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ViaBitVT), SubVec: V, Idx: 0);
6053
6054 SDValue Res =
6055 DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ViaVT, Operand: DAG.getBitcast(VT: ViaVT, V));
6056
6057 // Shift up the reversed bits if the vector didn't exactly fit into the larger
6058 // element type.
6059 if (ViaEltSize > NumElts)
6060 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: ViaVT, N1: Res,
6061 N2: DAG.getConstant(Val: ViaEltSize - NumElts, DL, VT: ViaVT));
6062
6063 Res = DAG.getBitcast(VT: ViaBitVT, V: Res);
6064
6065 if (ViaEltSize > NumElts)
6066 Res = DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0);
6067 return Res;
6068}
6069
6070static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
6071 const RISCVSubtarget &Subtarget,
6072 MVT &RotateVT, unsigned &RotateAmt) {
6073 unsigned NumElts = VT.getVectorNumElements();
6074 unsigned EltSizeInBits = VT.getScalarSizeInBits();
6075 unsigned NumSubElts;
6076 if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, MinSubElts: 2,
6077 MaxSubElts: NumElts, NumSubElts, RotateAmt))
6078 return false;
6079 RotateVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSizeInBits * NumSubElts),
6080 NumElements: NumElts / NumSubElts);
6081
6082 // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
6083 return Subtarget.getTargetLowering()->isTypeLegal(VT: RotateVT);
6084}
6085
6086// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
6087// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
6088// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
6089static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
6090 SelectionDAG &DAG,
6091 const RISCVSubtarget &Subtarget) {
6092 SDLoc DL(SVN);
6093
6094 EVT VT = SVN->getValueType(ResNo: 0);
6095 unsigned RotateAmt;
6096 MVT RotateVT;
6097 if (!isLegalBitRotate(Mask: SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
6098 return SDValue();
6099
6100 SDValue Op = DAG.getBitcast(VT: RotateVT, V: SVN->getOperand(Num: 0));
6101
6102 SDValue Rotate;
6103 // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
6104 // so canonicalize to vrev8.
6105 if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
6106 Rotate = DAG.getNode(Opcode: ISD::BSWAP, DL, VT: RotateVT, Operand: Op);
6107 else
6108 Rotate = DAG.getNode(Opcode: ISD::ROTL, DL, VT: RotateVT, N1: Op,
6109 N2: DAG.getConstant(Val: RotateAmt, DL, VT: RotateVT));
6110
6111 return DAG.getBitcast(VT, V: Rotate);
6112}
6113
6114// If compiling with an exactly known VLEN, see if we can split a
6115// shuffle on m2 or larger into a small number of m1 sized shuffles
6116// which write each destination registers exactly once.
6117static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
6118 SelectionDAG &DAG,
6119 const RISCVSubtarget &Subtarget) {
6120 SDLoc DL(SVN);
6121 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6122 SDValue V1 = SVN->getOperand(Num: 0);
6123 SDValue V2 = SVN->getOperand(Num: 1);
6124 ArrayRef<int> Mask = SVN->getMask();
6125
6126 // If we don't know exact data layout, not much we can do. If this
6127 // is already m1 or smaller, no point in splitting further.
6128 const auto VLen = Subtarget.getRealVLen();
6129 if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
6130 return SDValue();
6131
6132 // Avoid picking up bitrotate patterns which we have a linear-in-lmul
6133 // expansion for.
6134 unsigned RotateAmt;
6135 MVT RotateVT;
6136 if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
6137 return SDValue();
6138
6139 MVT ElemVT = VT.getVectorElementType();
6140 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
6141
6142 EVT ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
6143 MVT OneRegVT = MVT::getVectorVT(VT: ElemVT, NumElements: ElemsPerVReg);
6144 MVT M1VT = getContainerForFixedLengthVector(VT: OneRegVT, Subtarget);
6145 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
6146 unsigned NumOpElts = M1VT.getVectorMinNumElements();
6147 unsigned NumElts = ContainerVT.getVectorMinNumElements();
6148 unsigned NumOfSrcRegs = NumElts / NumOpElts;
6149 unsigned NumOfDestRegs = NumElts / NumOpElts;
6150 // The following semantically builds up a fixed length concat_vector
6151 // of the component shuffle_vectors. We eagerly lower to scalable here
6152 // to avoid DAG combining it back to a large shuffle_vector again.
6153 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
6154 V2 = convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget);
6155 SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
6156 Operands;
6157 processShuffleMasks(
6158 Mask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs,
6159 NoInputAction: [&]() { Operands.emplace_back(); },
6160 SingleInputAction: [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
6161 Operands.emplace_back().emplace_back(Args&: SrcVecIdx, UINT_MAX,
6162 Args: SmallVector<int>(SrcSubMask));
6163 },
6164 ManyInputsAction: [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
6165 if (NewReg)
6166 Operands.emplace_back();
6167 Operands.back().emplace_back(Args&: Idx1, Args&: Idx2, Args: SmallVector<int>(SrcSubMask));
6168 });
6169 assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
6170 // Note: check that we do not emit too many shuffles here to prevent code
6171 // size explosion.
6172 // TODO: investigate, if it can be improved by extra analysis of the masks to
6173 // check if the code is more profitable.
6174 unsigned NumShuffles = std::accumulate(
6175 first: Operands.begin(), last: Operands.end(), init: 0u,
6176 binary_op: [&](unsigned N,
6177 ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
6178 if (Data.empty())
6179 return N;
6180 N += Data.size();
6181 for (const auto &P : Data) {
6182 unsigned Idx2 = std::get<1>(t: P);
6183 ArrayRef<int> Mask = std::get<2>(t: P);
6184 if (Idx2 != UINT_MAX)
6185 ++N;
6186 else if (ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts: Mask.size()))
6187 --N;
6188 }
6189 return N;
6190 });
6191 if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
6192 (NumOfDestRegs <= 2 && NumShuffles >= 4))
6193 return SDValue();
6194 auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
6195 SDValue SubVec = DAG.getExtractSubvector(DL, VT: M1VT, Vec: SrcVec, Idx: ExtractIdx);
6196 SubVec = convertFromScalableVector(VT: OneRegVT, V: SubVec, DAG, Subtarget);
6197 return SubVec;
6198 };
6199 auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
6200 ArrayRef<int> Mask) {
6201 SDValue SubVec = DAG.getVectorShuffle(VT: OneRegVT, dl: DL, N1: SubVec1, N2: SubVec2, Mask);
6202 return SubVec;
6203 };
6204 SDValue Vec = DAG.getUNDEF(VT: ContainerVT);
6205 for (auto [I, Data] : enumerate(First&: Operands)) {
6206 if (Data.empty())
6207 continue;
6208 SmallDenseMap<unsigned, SDValue, 4> Values;
6209 for (unsigned I : seq<unsigned>(Size: Data.size())) {
6210 const auto &[Idx1, Idx2, _] = Data[I];
6211 // If the shuffle contains permutation of odd number of elements,
6212 // Idx1 might be used already in the first iteration.
6213 //
6214 // Idx1 = shuffle Idx1, Idx2
6215 // Idx1 = shuffle Idx1, Idx3
6216 SDValue &V = Values.try_emplace(Key: Idx1).first->getSecond();
6217 if (!V)
6218 V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
6219 (Idx1 % NumOfSrcRegs) * NumOpElts);
6220 if (Idx2 != UINT_MAX) {
6221 SDValue &V = Values.try_emplace(Key: Idx2).first->getSecond();
6222 if (!V)
6223 V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
6224 (Idx2 % NumOfSrcRegs) * NumOpElts);
6225 }
6226 }
6227 SDValue V;
6228 for (const auto &[Idx1, Idx2, Mask] : Data) {
6229 SDValue V1 = Values.at(Val: Idx1);
6230 SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Val: Idx2);
6231 V = PerformShuffle(V1, V2, Mask);
6232 Values[Idx1] = V;
6233 }
6234
6235 unsigned InsertIdx = I * NumOpElts;
6236 V = convertToScalableVector(VT: M1VT, V, DAG, Subtarget);
6237 Vec = DAG.getInsertSubvector(DL, Vec, SubVec: V, Idx: InsertIdx);
6238 }
6239 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
6240}
6241
6242// Matches a subset of compress masks with a contiguous prefix of output
6243// elements. This could be extended to allow gaps by deciding which
6244// source elements to spuriously demand.
6245static bool isCompressMask(ArrayRef<int> Mask) {
6246 int Last = -1;
6247 bool SawUndef = false;
6248 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
6249 if (M == -1) {
6250 SawUndef = true;
6251 continue;
6252 }
6253 if (SawUndef)
6254 return false;
6255 if (Idx > (unsigned)M)
6256 return false;
6257 if (M <= Last)
6258 return false;
6259 Last = M;
6260 }
6261 return true;
6262}
6263
6264/// Given a shuffle where the indices are disjoint between the two sources,
6265/// e.g.:
6266///
6267/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
6268///
6269/// Merge the two sources into one and do a single source shuffle:
6270///
6271/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
6272/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
6273///
6274/// A vselect will either be merged into a masked instruction or be lowered as a
6275/// vmerge.vvm, which is cheaper than a vrgather.vv.
6276static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
6277 SelectionDAG &DAG,
6278 const RISCVSubtarget &Subtarget) {
6279 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6280 MVT XLenVT = Subtarget.getXLenVT();
6281 SDLoc DL(SVN);
6282
6283 const ArrayRef<int> Mask = SVN->getMask();
6284
6285 // Work out which source each lane will come from.
6286 SmallVector<int, 16> Srcs(Mask.size(), -1);
6287
6288 for (int Idx : Mask) {
6289 if (Idx == -1)
6290 continue;
6291 unsigned SrcIdx = Idx % Mask.size();
6292 int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
6293 if (Srcs[SrcIdx] == -1)
6294 // Mark this source as using this lane.
6295 Srcs[SrcIdx] = Src;
6296 else if (Srcs[SrcIdx] != Src)
6297 // The other source is using this lane: not disjoint.
6298 return SDValue();
6299 }
6300
6301 SmallVector<SDValue> SelectMaskVals;
6302 for (int Lane : Srcs) {
6303 if (Lane == -1)
6304 SelectMaskVals.push_back(Elt: DAG.getUNDEF(VT: XLenVT));
6305 else
6306 SelectMaskVals.push_back(Elt: DAG.getConstant(Val: Lane ? 0 : 1, DL, VT: XLenVT));
6307 }
6308 MVT MaskVT = VT.changeVectorElementType(EltVT: MVT::i1);
6309 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: SelectMaskVals);
6310 SDValue Select = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask,
6311 N2: SVN->getOperand(Num: 0), N3: SVN->getOperand(Num: 1));
6312
6313 // Move all indices relative to the first source.
6314 SmallVector<int> NewMask(Mask.size());
6315 for (unsigned I = 0; I < Mask.size(); I++) {
6316 if (Mask[I] == -1)
6317 NewMask[I] = -1;
6318 else
6319 NewMask[I] = Mask[I] % Mask.size();
6320 }
6321
6322 return DAG.getVectorShuffle(VT, dl: DL, N1: Select, N2: DAG.getUNDEF(VT), Mask: NewMask);
6323}
6324
6325/// Is this mask local (i.e. elements only move within their local span), and
6326/// repeating (that is, the same rearrangement is being done within each span)?
6327static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
6328 // Require a prefix from the original mask until the consumer code
6329 // is adjusted to rewrite the mask instead of just taking a prefix.
6330 for (auto [I, M] : enumerate(First&: Mask)) {
6331 if (M == -1)
6332 continue;
6333 if ((M / Span) != (int)(I / Span))
6334 return false;
6335 int SpanIdx = I % Span;
6336 int Expected = M % Span;
6337 if (Mask[SpanIdx] != Expected)
6338 return false;
6339 }
6340 return true;
6341}
6342
6343/// Is this mask only using elements from the first span of the input?
6344static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
6345 return all_of(Range&: Mask, P: [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
6346}
6347
6348/// Return true for a mask which performs an arbitrary shuffle within the first
6349/// span, and then repeats that same result across all remaining spans. Note
6350/// that this doesn't check if all the inputs come from a single span!
6351static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
6352 // Require a prefix from the original mask until the consumer code
6353 // is adjusted to rewrite the mask instead of just taking a prefix.
6354 for (auto [I, M] : enumerate(First&: Mask)) {
6355 if (M == -1)
6356 continue;
6357 int SpanIdx = I % Span;
6358 if (Mask[SpanIdx] != M)
6359 return false;
6360 }
6361 return true;
6362}
6363
6364/// Try to widen element type to get a new mask value for a better permutation
6365/// sequence. This doesn't try to inspect the widened mask for profitability;
6366/// we speculate the widened form is equal or better. This has the effect of
6367/// reducing mask constant sizes - allowing cheaper materialization sequences
6368/// - and index sequence sizes - reducing register pressure and materialization
6369/// cost, at the cost of (possibly) an extra VTYPE toggle.
6370static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
6371 SDLoc DL(Op);
6372 MVT VT = Op.getSimpleValueType();
6373 MVT ScalarVT = VT.getVectorElementType();
6374 unsigned ElementSize = ScalarVT.getFixedSizeInBits();
6375 SDValue V0 = Op.getOperand(i: 0);
6376 SDValue V1 = Op.getOperand(i: 1);
6377 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
6378
6379 // Avoid wasted work leading to isTypeLegal check failing below
6380 if (ElementSize > 32)
6381 return SDValue();
6382
6383 SmallVector<int, 8> NewMask;
6384 if (!widenShuffleMaskElts(M: Mask, NewMask))
6385 return SDValue();
6386
6387 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(BitWidth: ElementSize * 2)
6388 : MVT::getIntegerVT(BitWidth: ElementSize * 2);
6389 MVT NewVT = MVT::getVectorVT(VT: NewEltVT, NumElements: VT.getVectorNumElements() / 2);
6390 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT: NewVT))
6391 return SDValue();
6392 V0 = DAG.getBitcast(VT: NewVT, V: V0);
6393 V1 = DAG.getBitcast(VT: NewVT, V: V1);
6394 return DAG.getBitcast(VT, V: DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: V0, N2: V1, Mask: NewMask));
6395}
6396
6397// Match an interleave shuffle that forms a P-extension packed zip:
6398// <a0, b0, a1, b1, ...> -> zip*p/wzip*p
6399static SDValue lowerVECTOR_SHUFFLEAsPZip(ShuffleVectorSDNode *SVN,
6400 const RISCVSubtarget &Subtarget,
6401 SelectionDAG &DAG) {
6402 SDValue V1 = SVN->getOperand(Num: 0);
6403 SDValue V2 = SVN->getOperand(Num: 1);
6404 SDLoc DL(SVN);
6405 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6406 unsigned NumElts = VT.getVectorNumElements();
6407 ArrayRef<int> Mask = SVN->getMask();
6408
6409 if (VT != MVT::v8i8 && VT != MVT::v4i16)
6410 return SDValue();
6411
6412 SmallVector<unsigned, 2> StartIndexes;
6413 if (!V2.isUndef() &&
6414 ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: NumElts * 2, StartIndexes)) {
6415 unsigned EvenSrc = StartIndexes[0];
6416 unsigned OddSrc = StartIndexes[1];
6417 if (EvenSrc == 0 && OddSrc == NumElts) {
6418 if (Subtarget.is64Bit())
6419 return DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT, N1: V1, N2: V2);
6420 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6421 V1 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V1, Idx: 0);
6422 V2 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V2, Idx: 0);
6423 return DAG.getNode(Opcode: RISCVISD::PWZIP, DL, VT, N1: V1, N2: V2);
6424 }
6425 if (EvenSrc == NumElts && OddSrc == 0) {
6426 if (Subtarget.is64Bit())
6427 return DAG.getNode(Opcode: RISCVISD::PZIP, DL, VT, N1: V2, N2: V1);
6428 EVT HalfVT = VT.getHalfNumVectorElementsVT();
6429 V1 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V1, Idx: 0);
6430 V2 = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: V2, Idx: 0);
6431 return DAG.getNode(Opcode: RISCVISD::PWZIP, DL, VT, N1: V2, N2: V1);
6432 }
6433 }
6434
6435 return SDValue();
6436}
6437
6438// Match a deinterleave shuffle that forms a P-extension packed unzip:
6439// <a0, a2, ..., b0, b2, ...> -> unzip*p
6440// <a1, a3, ..., b1, b3, ...> -> unzip*hp
6441static SDValue lowerVECTOR_SHUFFLEAsPUnzip(ShuffleVectorSDNode *SVN,
6442 SelectionDAG &DAG, bool IsRV64) {
6443 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6444 if (!IsRV64 || (VT != MVT::v8i8 && VT != MVT::v4i16))
6445 return SDValue();
6446
6447 SDValue V1 = SVN->getOperand(Num: 0);
6448 SDValue V2 = SVN->getOperand(Num: 1);
6449 SDLoc DL(SVN);
6450 ArrayRef<int> Mask = SVN->getMask();
6451
6452 unsigned Index = 0;
6453 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2, Index))
6454 return SDValue();
6455
6456 unsigned Opc = Index == 0 ? RISCVISD::PUNZIPE : RISCVISD::PUNZIPO;
6457 return DAG.getNode(Opcode: Opc, DL, VT, N1: V1, N2: V2);
6458}
6459
6460// Match a legalized deinterleave shuffle on two RV32 vector halves and lower
6461// it to an RV32 P narrowing shift on the concatenated source.
6462static SDValue
6463lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(ShuffleVectorSDNode *SVN,
6464 const RISCVSubtarget &Subtarget,
6465 SelectionDAG &DAG) {
6466 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6467 if (Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
6468 return SDValue();
6469
6470 SDValue V1 = SVN->getOperand(Num: 0);
6471 SDValue V2 = SVN->getOperand(Num: 1);
6472 SDLoc DL(SVN);
6473 unsigned NumElts = VT.getVectorNumElements();
6474
6475 SDValue Src = foldConcatVector(V1, V2);
6476 if (!Src) {
6477 MVT SrcVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
6478 Src = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: SrcVT, N1: V1, N2: V2);
6479 }
6480
6481 // The source vector should be twice the size.
6482 if (Src.getValueType().getVectorNumElements() != 2 * NumElts)
6483 return SDValue();
6484
6485 unsigned Index = 0;
6486 if (!ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask: SVN->getMask(), Factor: 2, Index))
6487 return SDValue();
6488
6489 unsigned EltBits = VT.getVectorElementType().getSizeInBits();
6490 return DAG.getNode(Opcode: RISCVISD::PNSRL, DL, VT, N1: Src,
6491 N2: DAG.getConstant(Val: Index * EltBits, DL, VT: MVT::i32));
6492}
6493
6494// Match a strided-interleave shuffle that forms a P-extension packed pair:
6495// <a0, b0, a2, b2, ...> -> ppaire.*
6496// <a1, b1, a3, b3, ...> -> ppairo.*
6497static SDValue lowerVECTOR_SHUFFLEAsPPair(ShuffleVectorSDNode *SVN,
6498 SelectionDAG &DAG) {
6499 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6500 if (VT != MVT::v4i8 && VT != MVT::v8i8 && VT != MVT::v4i16)
6501 return SDValue();
6502
6503 SDValue V1 = SVN->getOperand(Num: 0);
6504 SDValue V2 = SVN->getOperand(Num: 1);
6505 SDLoc DL(SVN);
6506 unsigned NumElts = VT.getVectorNumElements();
6507 ArrayRef<int> Mask = SVN->getMask();
6508 if (V2.isUndef())
6509 return SDValue();
6510
6511 // Match <start, N+start, start+2, N+start+2, ...>: each widened element of
6512 // V1/V2 contributes its low (start=0, ppaire) or high (start=1, ppairo) byte.
6513 // Trailing lanes may be undef when a 4-byte source was widened to v8i8.
6514 auto IsStrided = [&](unsigned Start) {
6515 for (unsigned I = 0; I != NumElts / 2; ++I) {
6516 int M0 = Mask[2 * I];
6517 int M1 = Mask[2 * I + 1];
6518 if (M0 < 0 && M1 < 0)
6519 continue;
6520 if (M0 != (int)(Start + 2 * I) || M1 != (int)(NumElts + Start + 2 * I))
6521 return false;
6522 }
6523 return true;
6524 };
6525
6526 bool IsOdd;
6527 if (IsStrided(0))
6528 IsOdd = false;
6529 else if (IsStrided(1))
6530 IsOdd = true;
6531 else
6532 return SDValue();
6533
6534 unsigned Opc = IsOdd ? RISCVISD::PPAIRO : RISCVISD::PPAIRE;
6535 return DAG.getNode(Opcode: Opc, DL, VT, N1: V1, N2: V2);
6536}
6537
6538SDValue RISCVTargetLowering::lowerVECTOR_SHUFFLE(SDValue Op,
6539 SelectionDAG &DAG) const {
6540 SDValue V1 = Op.getOperand(i: 0);
6541 SDValue V2 = Op.getOperand(i: 1);
6542 SDLoc DL(Op);
6543 MVT XLenVT = Subtarget.getXLenVT();
6544 MVT VT = Op.getSimpleValueType();
6545 unsigned NumElts = VT.getVectorNumElements();
6546 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6547
6548 // Select RVP-specific packed shuffles before falling back to the generic
6549 // fixed/scalable-vector lowering below.
6550 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
6551 ArrayRef<int> Mask = SVN->getMask();
6552
6553 // Select an element reverse shuffle to VECTOR_REVERSE. The tablegen
6554 // patterns select rev8/rev16/ppairoe.* from VECTOR_REVERSE.
6555 // Reverse of the low L lanes, higher lanes poison. L == NumElts is a plain
6556 // reverse; L == NumElts/2 is a widened RV64 v4i8/v2i16 reverse.
6557 auto IsLowReverse = [&](unsigned L) {
6558 return V2.isUndef() &&
6559 ShuffleVectorInst::isReverseMask(Mask: Mask.take_front(N: L), NumSrcElts: L) &&
6560 all_of(Range: Mask.drop_front(N: L), P: [](int M) { return M < 0; });
6561 };
6562 if (IsLowReverse(NumElts))
6563 return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V1);
6564 if (Subtarget.is64Bit() && VT == MVT::v4i16 && IsLowReverse(/*L=*/2))
6565 return DAG.getNode(Opcode: RISCVISD::PPAIROE_H, DL, VT, N1: V1, N2: V1);
6566 // Widened: reversing sends the low-half lanes to the top half, so shift
6567 // them back down by half the register. Only the 64-bit packed types are
6568 // legal here, so the register is XLen (i64).
6569 if (Subtarget.is64Bit() && VT.getSizeInBits() == 64 &&
6570 IsLowReverse(NumElts / 2)) {
6571 SDValue Rev = DAG.getBitcast(
6572 VT: MVT::i64, V: DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V1));
6573 SDValue Srl =
6574 DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Rev,
6575 N2: DAG.getConstant(Val: VT.getSizeInBits() / 2, DL, VT: MVT::i64));
6576 return DAG.getBitcast(VT, V: Srl);
6577 }
6578
6579 if (SDValue V = lowerVECTOR_SHUFFLEAsPUnzip(SVN, DAG, IsRV64: Subtarget.is64Bit()))
6580 return V;
6581 if (SDValue V = lowerVECTOR_SHUFFLEAsPZip(SVN, Subtarget, DAG))
6582 return V;
6583 if (SDValue V =
6584 lowerVECTOR_SHUFFLEAsRV32PNarrowingShift(SVN, Subtarget, DAG))
6585 return V;
6586 if (SDValue V = lowerVECTOR_SHUFFLEAsPPair(SVN, DAG))
6587 return V;
6588 return SDValue();
6589 }
6590
6591 if (VT.getVectorElementType() == MVT::i1) {
6592 // Lower to a vror.vi of a larger element type if possible before we promote
6593 // i1s to i8s.
6594 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6595 return V;
6596 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
6597 return V;
6598
6599 // Promote i1 shuffle to i8 shuffle.
6600 MVT WidenVT = MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount());
6601 V1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: V1);
6602 V2 = V2.isUndef() ? DAG.getUNDEF(VT: WidenVT)
6603 : DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: V2);
6604 SDValue Shuffled = DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: V1, N2: V2, Mask: SVN->getMask());
6605 return DAG.getSetCC(DL, VT, LHS: Shuffled, RHS: DAG.getConstant(Val: 0, DL, VT: WidenVT),
6606 Cond: ISD::SETNE);
6607 }
6608
6609 MVT ContainerVT = getContainerForFixedLengthVector(VT);
6610
6611 // Store the return value in a single variable instead of structured bindings
6612 // so that we can pass it to GetSlide below, which cannot capture structured
6613 // bindings until C++20.
6614 auto TrueMaskVL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
6615 auto [TrueMask, VL] = TrueMaskVL;
6616
6617 if (SVN->isSplat()) {
6618 const int Lane = SVN->getSplatIndex();
6619 if (Lane >= 0) {
6620 MVT SVT = VT.getVectorElementType();
6621
6622 // Turn splatted vector load into a strided load with an X0 stride.
6623 SDValue V = V1;
6624 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
6625 // with undef.
6626 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
6627 int Offset = Lane;
6628 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
6629 int OpElements =
6630 V.getOperand(i: 0).getSimpleValueType().getVectorNumElements();
6631 V = V.getOperand(i: Offset / OpElements);
6632 Offset %= OpElements;
6633 }
6634
6635 // We need to ensure the load isn't atomic or volatile.
6636 if (ISD::isNormalLoad(N: V.getNode()) && cast<LoadSDNode>(Val&: V)->isSimple()) {
6637 auto *Ld = cast<LoadSDNode>(Val&: V);
6638 Offset *= SVT.getStoreSize();
6639 SDValue NewAddr = DAG.getMemBasePlusOffset(
6640 Base: Ld->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: Offset), DL);
6641
6642 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
6643 if (SVT.isInteger() && SVT.bitsGT(VT: XLenVT)) {
6644 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
6645 SDValue IntID =
6646 DAG.getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: XLenVT);
6647 SDValue Ops[] = {Ld->getChain(),
6648 IntID,
6649 DAG.getUNDEF(VT: ContainerVT),
6650 NewAddr,
6651 DAG.getRegister(Reg: RISCV::X0, VT: XLenVT),
6652 VL};
6653 SDValue NewLoad = DAG.getMemIntrinsicNode(
6654 Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT: SVT,
6655 MMO: DAG.getMachineFunction().getMachineMemOperand(
6656 MMO: Ld->getMemOperand(), Offset, Size: SVT.getStoreSize()));
6657 DAG.makeEquivalentMemoryOrdering(OldLoad: Ld, NewMemOp: NewLoad);
6658 return convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
6659 }
6660
6661 MVT SplatVT = ContainerVT;
6662
6663 // f16 with zvfhmin and bf16 need to use an integer scalar load.
6664 if (SVT == MVT::bf16 ||
6665 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
6666 SVT = MVT::i16;
6667 SplatVT = ContainerVT.changeVectorElementType(EltVT: SVT);
6668 }
6669
6670 // Otherwise use a scalar load and splat. This will give the best
6671 // opportunity to fold a splat into the operation. ISel can turn it into
6672 // the x0 strided load if we aren't able to fold away the select.
6673 if (SVT.isFloatingPoint())
6674 V = DAG.getLoad(VT: SVT, dl: DL, Chain: Ld->getChain(), Ptr: NewAddr,
6675 PtrInfo: Ld->getPointerInfo().getWithOffset(O: Offset),
6676 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
6677 else
6678 V = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: XLenVT, Chain: Ld->getChain(), Ptr: NewAddr,
6679 PtrInfo: Ld->getPointerInfo().getWithOffset(O: Offset), MemVT: SVT,
6680 Alignment: Ld->getBaseAlign(),
6681 MMOFlags: Ld->getMemOperand()->getFlags());
6682 DAG.makeEquivalentMemoryOrdering(OldLoad: Ld, NewMemOp: V);
6683
6684 unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
6685 : RISCVISD::VMV_V_X_VL;
6686 SDValue Splat =
6687 DAG.getNode(Opcode: Opc, DL, VT: SplatVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: V, N3: VL);
6688 Splat = DAG.getBitcast(VT: ContainerVT, V: Splat);
6689 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
6690 }
6691
6692 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
6693 assert(Lane < (int)NumElts && "Unexpected lane!");
6694 SDValue Gather = DAG.getNode(Opcode: RISCVISD::VRGATHER_VX_VL, DL, VT: ContainerVT,
6695 N1: V1, N2: DAG.getConstant(Val: Lane, DL, VT: XLenVT),
6696 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
6697 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6698 }
6699 }
6700
6701 // For exact VLEN m2 or greater, try to split to m1 operations if we
6702 // can split cleanly.
6703 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
6704 return V;
6705
6706 ArrayRef<int> Mask = SVN->getMask();
6707
6708 if (SDValue V =
6709 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
6710 return V;
6711
6712 if (SDValue V =
6713 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
6714 return V;
6715
6716 // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
6717 // available.
6718 if (Subtarget.hasStdExtZvkb())
6719 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6720 return V;
6721
6722 if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: NumElts) && V2.isUndef() &&
6723 NumElts != 2)
6724 return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V1);
6725
6726 // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
6727 // use shift and truncate to perform the shuffle.
6728 // TODO: For Factor=6, we can perform the first step of the deinterleave via
6729 // shift-and-trunc reducing total cost for everything except an mf8 result.
6730 // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
6731 // to do the entire operation.
6732 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6733 const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6734 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6735 for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
6736 unsigned Index = 0;
6737 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
6738 1 < count_if(Range&: Mask, P: [](int Idx) { return Idx != -1; })) {
6739 if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
6740 return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
6741 if (1 < count_if(Range&: Mask,
6742 P: [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6743 1 < count_if(Range&: Mask, P: [&Mask](int Idx) {
6744 return Idx >= (int)Mask.size();
6745 })) {
6746 // Narrow each source and concatenate them.
6747 // FIXME: For small LMUL it is better to concatenate first.
6748 MVT EltVT = VT.getVectorElementType();
6749 auto EltCnt = VT.getVectorElementCount();
6750 MVT SubVT =
6751 MVT::getVectorVT(VT: EltVT, EC: EltCnt.divideCoefficientBy(RHS: Factor));
6752
6753 SDValue Lo =
6754 getDeinterleaveShiftAndTrunc(DL, VT: SubVT, Src: V1, Factor, Index, DAG);
6755 SDValue Hi =
6756 getDeinterleaveShiftAndTrunc(DL, VT: SubVT, Src: V2, Factor, Index, DAG);
6757
6758 SDValue Concat =
6759 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL,
6760 VT: SubVT.getDoubleNumVectorElementsVT(), N1: Lo, N2: Hi);
6761 if (Factor == 2)
6762 return Concat;
6763
6764 SDValue Vec = DAG.getUNDEF(VT);
6765 return DAG.getInsertSubvector(DL, Vec, SubVec: Concat, Idx: 0);
6766 }
6767 }
6768 }
6769 }
6770
6771 // If this is a deinterleave(2), try using vunzip{e,o}. This mostly catches
6772 // e64 which can't match above.
6773 unsigned Index = 0;
6774 if (Subtarget.hasStdExtZvzip() &&
6775 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2, Index) &&
6776 1 < count_if(Range&: Mask, P: [](int Idx) { return Idx != -1; })) {
6777 bool UsesBothSources =
6778 1 < count_if(Range&: Mask,
6779 P: [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6780 1 < count_if(Range&: Mask,
6781 P: [&Mask](int Idx) { return Idx >= (int)Mask.size(); });
6782
6783 if (isLegalVTForZvzipOperand(VT, Subtarget)) {
6784 unsigned Opc = Index == 0 ? RISCVISD::VUNZIPE_VL : RISCVISD::VUNZIPO_VL;
6785 MVT NewVT = VT.getDoubleNumVectorElementsVT();
6786 if (isTypeLegal(VT: NewVT)) {
6787 SDValue Op;
6788 if (V2.isUndef()) {
6789 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, N1: V1, N2: V2);
6790 } else if (auto VLEN = Subtarget.getRealVLen();
6791 VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0) {
6792 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, N1: V1, N2: V2);
6793 } else if (SDValue Src = foldConcatVector(V1, V2)) {
6794 Op = DAG.getExtractSubvector(DL, VT: NewVT, Vec: Src, Idx: 0);
6795 }
6796 if (Op)
6797 return lowerZvzipVUNZIP(Opc, Op, DL, DAG, Subtarget);
6798 }
6799
6800 if (UsesBothSources &&
6801 V1.getSimpleValueType().getVectorMinNumElements() >= 2 &&
6802 V2.getSimpleValueType().getVectorMinNumElements() >= 2) {
6803 SDValue Lo = lowerZvzipVUNZIP(Opc, Op: V1, DL, DAG, Subtarget);
6804 SDValue Hi = lowerZvzipVUNZIP(Opc, Op: V2, DL, DAG, Subtarget);
6805 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
6806 }
6807 }
6808 }
6809
6810 if (SDValue V =
6811 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
6812 return V;
6813
6814 // Detect an interleave shuffle and lower to
6815 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
6816 int EvenSrc, OddSrc;
6817 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
6818 !(NumElts == 2 &&
6819 ShuffleVectorInst::isSingleSourceMask(Mask, NumSrcElts: Mask.size()))) {
6820 // Extract the halves of the vectors.
6821 MVT HalfVT = VT.getHalfNumVectorElementsVT();
6822
6823 // Recognize if one half is actually undef; the matching above will
6824 // otherwise reuse the even stream for the undef one. This improves
6825 // spread(2) shuffles.
6826 bool LaneIsUndef[2] = { true, true};
6827 for (const auto &[Idx, M] : enumerate(First&: Mask))
6828 LaneIsUndef[Idx % 2] &= (M == -1);
6829
6830 int Size = Mask.size();
6831 SDValue EvenV, OddV;
6832 if (LaneIsUndef[0]) {
6833 EvenV = DAG.getUNDEF(VT: HalfVT);
6834 } else {
6835 assert(EvenSrc >= 0 && "Undef source?");
6836 EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
6837 EvenV = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: EvenV, Idx: EvenSrc % Size);
6838 }
6839
6840 if (LaneIsUndef[1]) {
6841 OddV = DAG.getUNDEF(VT: HalfVT);
6842 } else {
6843 assert(OddSrc >= 0 && "Undef source?");
6844 OddV = (OddSrc / Size) == 0 ? V1 : V2;
6845 OddV = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: OddV, Idx: OddSrc % Size);
6846 }
6847
6848 // Prefer vzip if available.
6849 // TODO: Extend to matching vzip if EvenSrc and OddSrc allow.
6850 if (Subtarget.hasStdExtZvzip() && isLegalVTForZvzipOperand(VT, Subtarget))
6851 return lowerZvzipVZIP(Op0: EvenV, Op1: OddV, DL, DAG, Subtarget);
6852 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
6853 }
6854
6855 // Recognize a pattern which can handled via a pair of vslideup/vslidedown
6856 // instructions (in any combination) with masking on the second instruction.
6857 // Also handles masked slides into an identity source, and single slides
6858 // without masking. Avoid matching bit rotates (which are not also element
6859 // rotates) as slide pairs. This is a performance heuristic, not a
6860 // functional check.
6861 std::array<std::pair<int, int>, 2> SrcInfo;
6862 unsigned RotateAmt;
6863 MVT RotateVT;
6864 if (::isMaskedSlidePair(Mask, SrcInfo) &&
6865 (isElementRotate(SrcInfo, NumElts) ||
6866 !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
6867 SDValue Sources[2];
6868 auto GetSourceFor = [&](const std::pair<int, int> &Info) {
6869 int SrcIdx = Info.first;
6870 assert(SrcIdx == 0 || SrcIdx == 1);
6871 SDValue &Src = Sources[SrcIdx];
6872 if (!Src) {
6873 SDValue SrcV = SrcIdx == 0 ? V1 : V2;
6874 Src = convertToScalableVector(VT: ContainerVT, V: SrcV, DAG, Subtarget);
6875 }
6876 return Src;
6877 };
6878 auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
6879 SDValue Passthru) {
6880 auto [TrueMask, VL] = TrueMaskVL;
6881 SDValue SrcV = GetSourceFor(Src);
6882 int SlideAmt = Src.second;
6883 if (SlideAmt == 0) {
6884 // Should never be second operation
6885 assert(Mask == TrueMask);
6886 return SrcV;
6887 }
6888 if (SlideAmt < 0)
6889 return getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: SrcV,
6890 Offset: DAG.getConstant(Val: -SlideAmt, DL, VT: XLenVT), Mask, VL,
6891 Policy: RISCVVType::TAIL_AGNOSTIC);
6892 return getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: SrcV,
6893 Offset: DAG.getConstant(Val: SlideAmt, DL, VT: XLenVT), Mask, VL,
6894 Policy: RISCVVType::TAIL_AGNOSTIC);
6895 };
6896
6897 if (SrcInfo[1].first == -1) {
6898 SDValue Res = DAG.getUNDEF(VT: ContainerVT);
6899 Res = GetSlide(SrcInfo[0], TrueMask, Res);
6900 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
6901 }
6902
6903 if (Subtarget.hasStdExtZvzip()) {
6904 bool TryWiden = false;
6905 unsigned Factor;
6906 if (isPairEven(SrcInfo, Mask, Factor)) {
6907 if (Factor == 1) {
6908 SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
6909 SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
6910 return lowerZvzipVPAIR(Opc: RISCVISD::VPAIRE_VL, Op0: Src1, Op1: Src2, DL, DAG,
6911 Subtarget);
6912 }
6913 TryWiden = true;
6914 }
6915 if (isPairOdd(SrcInfo, Mask, Factor)) {
6916 if (Factor == 1) {
6917 SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
6918 SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
6919 return lowerZvzipVPAIR(Opc: RISCVISD::VPAIRO_VL, Op0: Src1, Op1: Src2, DL, DAG,
6920 Subtarget);
6921 }
6922 TryWiden = true;
6923 }
6924 // If we found a widening oppurtunity which would let us form a
6925 // pair-even or pair-odd, use the generic code to widen the shuffle
6926 // and recurse through this logic.
6927 if (TryWiden)
6928 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
6929 return V;
6930 }
6931
6932 // Build the mask. Note that vslideup unconditionally preserves elements
6933 // below the slide amount in the destination, and thus those elements are
6934 // undefined in the mask. If the mask ends up all true (or undef), it
6935 // will be folded away by general logic.
6936 SmallVector<SDValue> MaskVals;
6937 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
6938 if (M < 0 ||
6939 (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
6940 MaskVals.push_back(Elt: DAG.getUNDEF(VT: XLenVT));
6941 continue;
6942 }
6943 int Src = M >= (int)NumElts;
6944 int Diff = (int)Idx - (M % NumElts);
6945 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
6946 assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
6947 "Must match exactly one of the two slides");
6948 MaskVals.push_back(Elt: DAG.getConstant(Val: C, DL, VT: XLenVT));
6949 }
6950 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
6951 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
6952 SDValue SelectMask = convertToScalableVector(
6953 VT: ContainerVT.changeVectorElementType(EltVT: MVT::i1),
6954 V: DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals), DAG, Subtarget);
6955
6956 SDValue Res = DAG.getUNDEF(VT: ContainerVT);
6957 Res = GetSlide(SrcInfo[0], TrueMask, Res);
6958 Res = GetSlide(SrcInfo[1], SelectMask, Res);
6959 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
6960 }
6961
6962 // Handle any remaining single source shuffles
6963 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
6964 if (V2.isUndef()) {
6965 // We might be able to express the shuffle as a bitrotate. But even if we
6966 // don't have Zvkb and have to expand, the expanded sequence of approx. 2
6967 // shifts and a vor will have a higher throughput than a vrgather.
6968 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6969 return V;
6970
6971 if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
6972 return V;
6973
6974 // Match a spread(4,8) which can be done via extend and shift. Spread(2)
6975 // is fully covered in interleave(2) above, so it is ignored here.
6976 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6977 unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6978 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6979 for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
6980 unsigned Index;
6981 if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
6982 MVT NarrowVT =
6983 MVT::getVectorVT(VT: VT.getVectorElementType(), NumElements: NumElts / Factor);
6984 SDValue Src = DAG.getExtractSubvector(DL, VT: NarrowVT, Vec: V1, Idx: 0);
6985 return getWideningSpread(V: Src, Factor, Index, DL, DAG);
6986 }
6987 }
6988 }
6989
6990 // If only a prefix of the source elements influence a prefix of the
6991 // destination elements, try to see if we can reduce the required LMUL
6992 unsigned MinVLen = Subtarget.getRealMinVLen();
6993 unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
6994 if (NumElts > MinVLMAX) {
6995 unsigned MaxIdx = 0;
6996 for (auto [I, M] : enumerate(First&: Mask)) {
6997 if (M == -1)
6998 continue;
6999 MaxIdx = std::max(l: {(unsigned)I, (unsigned)M, MaxIdx});
7000 }
7001 unsigned NewNumElts =
7002 std::max(a: (uint64_t)MinVLMAX, b: PowerOf2Ceil(A: MaxIdx + 1));
7003 if (NewNumElts != NumElts) {
7004 MVT NewVT = MVT::getVectorVT(VT: VT.getVectorElementType(), NumElements: NewNumElts);
7005 V1 = DAG.getExtractSubvector(DL, VT: NewVT, Vec: V1, Idx: 0);
7006 SDValue Res = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT: NewVT),
7007 Mask: Mask.take_front(N: NewNumElts));
7008 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: Res, Idx: 0);
7009 }
7010 }
7011
7012 // Before hitting generic lowering fallbacks, try to widen the mask
7013 // to a wider SEW.
7014 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7015 return V;
7016
7017 // Can we generate a vcompress instead of a vrgather? These scale better
7018 // at high LMUL, at the cost of not being able to fold a following select
7019 // into them. The mask constants are also smaller than the index vector
7020 // constants, and thus easier to materialize.
7021 if (isCompressMask(Mask)) {
7022 SmallVector<SDValue> MaskVals(NumElts,
7023 DAG.getConstant(Val: false, DL, VT: XLenVT));
7024 for (auto Idx : Mask) {
7025 if (Idx == -1)
7026 break;
7027 assert(Idx >= 0 && (unsigned)Idx < NumElts);
7028 MaskVals[Idx] = DAG.getConstant(Val: true, DL, VT: XLenVT);
7029 }
7030 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
7031 SDValue CompressMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
7032 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT, N1: V1, N2: CompressMask,
7033 N3: DAG.getUNDEF(VT));
7034 }
7035
7036 if (VT.getScalarSizeInBits() == 8 &&
7037 any_of(Range&: Mask, P: [&](const auto &Idx) { return Idx > 255; })) {
7038 // On such a vector we're unable to use i8 as the index type.
7039 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
7040 // may involve vector splitting if we're already at LMUL=8, or our
7041 // user-supplied maximum fixed-length LMUL.
7042 return SDValue();
7043 }
7044
7045 // Base case for the two operand recursion below - handle the worst case
7046 // single source shuffle.
7047 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
7048 MVT IndexVT = VT.changeTypeToInteger();
7049 // Since we can't introduce illegal index types at this stage, use i16 and
7050 // vrgatherei16 if the corresponding index type for plain vrgather is greater
7051 // than XLenVT.
7052 if (IndexVT.getScalarType().bitsGT(VT: XLenVT)) {
7053 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7054 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
7055 }
7056
7057 // If the mask allows, we can do all the index computation in 16 bits. This
7058 // requires less work and less register pressure at high LMUL, and creates
7059 // smaller constants which may be cheaper to materialize.
7060 if (IndexVT.getScalarType().bitsGT(VT: MVT::i16) && isUInt<16>(x: NumElts - 1) &&
7061 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
7062 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
7063 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
7064 }
7065
7066 MVT IndexContainerVT =
7067 ContainerVT.changeVectorElementType(EltVT: IndexVT.getScalarType());
7068
7069 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
7070 SmallVector<SDValue> GatherIndicesLHS;
7071 for (int MaskIndex : Mask) {
7072 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
7073 GatherIndicesLHS.push_back(Elt: IsLHSIndex
7074 ? DAG.getConstant(Val: MaskIndex, DL, VT: XLenVT)
7075 : DAG.getUNDEF(VT: XLenVT));
7076 }
7077 SDValue LHSIndices = DAG.getBuildVector(VT: IndexVT, DL, Ops: GatherIndicesLHS);
7078 LHSIndices =
7079 convertToScalableVector(VT: IndexContainerVT, V: LHSIndices, DAG, Subtarget);
7080 // At m1 and less, there's no point trying any of the high LMUL splitting
7081 // techniques. TODO: Should we reconsider this for DLEN < VLEN?
7082 if (NumElts <= MinVLMAX) {
7083 SDValue Gather = DAG.getNode(Opcode: GatherVVOpc, DL, VT: ContainerVT, N1: V1, N2: LHSIndices,
7084 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
7085 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7086 }
7087
7088 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
7089 EVT SubIndexVT = M1VT.changeVectorElementType(EltVT: IndexVT.getScalarType());
7090 auto [InnerTrueMask, InnerVL] =
7091 getDefaultScalableVLOps(VecVT: M1VT, DL, DAG, Subtarget);
7092 int N =
7093 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
7094 assert(isPowerOf2_32(N) && N <= 8);
7095
7096 // If we have a locally repeating mask, then we can reuse the first
7097 // register in the index register group for all registers within the
7098 // source register group. TODO: This generalizes to m2, and m4.
7099 if (isLocalRepeatingShuffle(Mask, Span: MinVLMAX)) {
7100 SDValue SubIndex = DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
7101 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
7102 for (int i = 0; i < N; i++) {
7103 unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
7104 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: SubIdx);
7105 SDValue SubVec =
7106 DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
7107 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
7108 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec, Idx: SubIdx);
7109 }
7110 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7111 }
7112
7113 // If we have a shuffle which only uses the first register in our source
7114 // register group, and repeats the same index across all spans, we can
7115 // use a single vrgather (and possibly some register moves).
7116 // TODO: This can be generalized for m2 or m4, or for any shuffle for
7117 // which we can do a linear number of shuffles to form an m1 which
7118 // contains all the output elements.
7119 if (isLowSourceShuffle(Mask, Span: MinVLMAX) &&
7120 isSpanSplatShuffle(Mask, Span: MinVLMAX)) {
7121 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: 0);
7122 SDValue SubIndex = DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
7123 SDValue SubVec = DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
7124 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
7125 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
7126 for (int i = 0; i < N; i++)
7127 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec,
7128 Idx: M1VT.getVectorMinNumElements() * i);
7129 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7130 }
7131
7132 // If we have a shuffle which only uses the first register in our
7133 // source register group, we can do a linear number of m1 vrgathers
7134 // reusing the same source register (but with different indices)
7135 // TODO: This can be generalized for m2 or m4, or for any shuffle
7136 // for which we can do a vslidedown followed by this expansion.
7137 if (isLowSourceShuffle(Mask, Span: MinVLMAX)) {
7138 SDValue SlideAmt =
7139 DAG.getElementCount(DL, VT: XLenVT, EC: M1VT.getVectorElementCount());
7140 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: 0);
7141 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
7142 for (int i = 0; i < N; i++) {
7143 if (i != 0)
7144 LHSIndices = getVSlidedown(DAG, Subtarget, DL, VT: IndexContainerVT,
7145 Passthru: DAG.getUNDEF(VT: IndexContainerVT), Op: LHSIndices,
7146 Offset: SlideAmt, Mask: TrueMask, VL);
7147 SDValue SubIndex =
7148 DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
7149 SDValue SubVec =
7150 DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
7151 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
7152 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec,
7153 Idx: M1VT.getVectorMinNumElements() * i);
7154 }
7155 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7156 }
7157
7158 // Fallback to generic vrgather if we can't find anything better.
7159 // On many machines, this will be O(LMUL^2)
7160 SDValue Gather = DAG.getNode(Opcode: GatherVVOpc, DL, VT: ContainerVT, N1: V1, N2: LHSIndices,
7161 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
7162 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
7163 }
7164
7165 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
7166 // merged with a second vrgather.
7167 SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
7168
7169 // Now construct the mask that will be used by the blended vrgather operation.
7170 // Construct the appropriate indices into each vector.
7171 for (int MaskIndex : Mask) {
7172 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
7173 ShuffleMaskLHS.push_back(Elt: IsLHSOrUndefIndex && MaskIndex >= 0
7174 ? MaskIndex : -1);
7175 ShuffleMaskRHS.push_back(Elt: IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
7176 }
7177
7178 // If the mask indices are disjoint between the two sources, we can lower it
7179 // as a vselect + a single source vrgather.vv. Don't do this if we think the
7180 // operands may end up being lowered to something cheaper than a vrgather.vv.
7181 if (!DAG.isSplatValue(V: V2) && !DAG.isSplatValue(V: V1) &&
7182 !ShuffleVectorSDNode::isSplatMask(Mask: ShuffleMaskLHS) &&
7183 !ShuffleVectorSDNode::isSplatMask(Mask: ShuffleMaskRHS) &&
7184 !ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskLHS, NumSrcElts: NumElts) &&
7185 !ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskRHS, NumSrcElts: NumElts))
7186 if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
7187 return V;
7188
7189 // Before hitting generic lowering fallbacks, try to widen the mask
7190 // to a wider SEW.
7191 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
7192 return V;
7193
7194 // Try to pick a profitable operand order.
7195 bool SwapOps = DAG.isSplatValue(V: V2) && !DAG.isSplatValue(V: V1);
7196 SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskRHS, NumSrcElts: NumElts);
7197
7198 // Recursively invoke lowering for each operand if we had two
7199 // independent single source shuffles, and then combine the result via a
7200 // vselect. Note that the vselect will likely be folded back into the
7201 // second permute (vrgather, or other) by the post-isel combine.
7202 V1 = DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT), Mask: ShuffleMaskLHS);
7203 V2 = DAG.getVectorShuffle(VT, dl: DL, N1: V2, N2: DAG.getUNDEF(VT), Mask: ShuffleMaskRHS);
7204
7205 SmallVector<SDValue> MaskVals;
7206 for (int MaskIndex : Mask) {
7207 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
7208 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
7209 }
7210
7211 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
7212 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
7213 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
7214
7215 if (SwapOps)
7216 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: V1, N3: V2);
7217 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: V2, N3: V1);
7218}
7219
7220bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
7221 // Only support legal VTs for other shuffles for now.
7222 if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
7223 return false;
7224
7225 // Support splats for any type. These should type legalize well.
7226 if (ShuffleVectorSDNode::isSplatMask(Mask: M))
7227 return true;
7228
7229 const unsigned NumElts = M.size();
7230 MVT SVT = VT.getSimpleVT();
7231
7232 // Not for i1 vectors.
7233 if (SVT.getScalarType() == MVT::i1)
7234 return false;
7235
7236 std::array<std::pair<int, int>, 2> SrcInfo;
7237 int Dummy1, Dummy2;
7238 return ShuffleVectorInst::isReverseMask(Mask: M, NumSrcElts: NumElts) ||
7239 (::isMaskedSlidePair(Mask: M, SrcInfo) &&
7240 isElementRotate(SrcInfo, NumElts)) ||
7241 isInterleaveShuffle(Mask: M, VT: SVT, EvenSrc&: Dummy1, OddSrc&: Dummy2, Subtarget);
7242}
7243
7244// Lower CTLZ_ZERO_POISON or CTTZ_ZERO_POISON by converting to FP and extracting
7245// the exponent.
7246SDValue
7247RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_POISON(SDValue Op,
7248 SelectionDAG &DAG) const {
7249 MVT VT = Op.getSimpleValueType();
7250 unsigned EltSize = VT.getScalarSizeInBits();
7251 SDValue Src = Op.getOperand(i: 0);
7252 SDLoc DL(Op);
7253 MVT ContainerVT = VT;
7254
7255 // We choose FP type that can represent the value if possible. Otherwise, we
7256 // use rounding to zero conversion for correct exponent of the result.
7257 // TODO: Use f16 for i8 when possible?
7258 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
7259 if (!isTypeLegal(VT: MVT::getVectorVT(VT: FloatEltVT, EC: VT.getVectorElementCount())))
7260 FloatEltVT = MVT::f32;
7261 MVT FloatVT = MVT::getVectorVT(VT: FloatEltVT, EC: VT.getVectorElementCount());
7262
7263 // Legal types should have been checked in the RISCVTargetLowering
7264 // constructor.
7265 // TODO: Splitting may make sense in some cases.
7266 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
7267 "Expected legal float type!");
7268
7269 // For CTTZ_ZERO_POISON, we need to extract the lowest set bit using X & -X.
7270 // The trailing zero count is equal to log2 of this single bit value.
7271 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON) {
7272 SDValue Neg = DAG.getNegative(Val: Src, DL, VT);
7273 Src = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Src, N2: Neg);
7274 }
7275
7276 // We have a legal FP type, convert to it.
7277 SDValue FloatVal;
7278 if (FloatVT.bitsGT(VT)) {
7279 FloatVal = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVT, Operand: Src);
7280 } else {
7281 // Use RTZ to avoid rounding influencing exponent of FloatVal.
7282 if (VT.isFixedLengthVector()) {
7283 ContainerVT = getContainerForFixedLengthVector(VT);
7284 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
7285 }
7286 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
7287 SDValue RTZRM =
7288 DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: Subtarget.getXLenVT());
7289 MVT ContainerFloatVT =
7290 MVT::getVectorVT(VT: FloatEltVT, EC: ContainerVT.getVectorElementCount());
7291 FloatVal = DAG.getNode(Opcode: RISCVISD::VFCVT_RM_F_XU_VL, DL, VT: ContainerFloatVT,
7292 N1: Src, N2: Mask, N3: RTZRM, N4: VL);
7293 if (VT.isFixedLengthVector())
7294 FloatVal = convertFromScalableVector(VT: FloatVT, V: FloatVal, DAG, Subtarget);
7295 }
7296 // Bitcast to integer and shift the exponent to the LSB.
7297 EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
7298 SDValue Bitcast = DAG.getBitcast(VT: IntVT, V: FloatVal);
7299 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
7300
7301 // Restore back to original type. Truncation after SRL is to generate vnsrl.
7302 SDValue Exp = DAG.getNode(Opcode: ISD::SRL, DL, VT: IntVT, N1: Bitcast,
7303 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: IntVT));
7304 if (IntVT.bitsLT(VT))
7305 Exp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Exp);
7306 else if (IntVT.bitsGT(VT))
7307 Exp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Exp);
7308
7309 // The exponent contains log2 of the value in biased form.
7310 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
7311 // For trailing zeros, we just need to subtract the bias.
7312 if (Op.getOpcode() == ISD::CTTZ_ZERO_POISON)
7313 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Exp,
7314 N2: DAG.getConstant(Val: ExponentBias, DL, VT));
7315
7316 // For leading zeros, we need to remove the bias and convert from log2 to
7317 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
7318 unsigned Adjust = ExponentBias + (EltSize - 1);
7319 SDValue Res =
7320 DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: Adjust, DL, VT), N2: Exp);
7321
7322 // The above result with zero input equals to Adjust which is greater than
7323 // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
7324 if (Op.getOpcode() == ISD::CTLZ)
7325 Res = DAG.getNode(Opcode: ISD::UMIN, DL, VT, N1: Res, N2: DAG.getConstant(Val: EltSize, DL, VT));
7326
7327 return Res;
7328}
7329
7330SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
7331 SelectionDAG &DAG) const {
7332 SDLoc DL(Op);
7333 MVT XLenVT = Subtarget.getXLenVT();
7334 SDValue Source = Op->getOperand(Num: 0);
7335 MVT SrcVT = Source.getSimpleValueType();
7336 SDValue Mask = Op->getOperand(Num: 1);
7337 SDValue EVL = Op->getOperand(Num: 2);
7338
7339 if (SrcVT.isFixedLengthVector()) {
7340 MVT ContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
7341 Source = convertToScalableVector(VT: ContainerVT, V: Source, DAG, Subtarget);
7342 Mask = convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Mask, DAG,
7343 Subtarget);
7344 SrcVT = ContainerVT;
7345 }
7346
7347 // Convert to boolean vector.
7348 if (SrcVT.getScalarType() != MVT::i1) {
7349 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
7350 SrcVT = MVT::getVectorVT(VT: MVT::i1, EC: SrcVT.getVectorElementCount());
7351 Source = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: SrcVT,
7352 Ops: {Source, AllZero, DAG.getCondCode(Cond: ISD::SETNE),
7353 DAG.getUNDEF(VT: SrcVT), Mask, EVL});
7354 }
7355
7356 SDValue Res = DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Source, N2: Mask, N3: EVL);
7357 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_POISON)
7358 // In this case, we can interpret poison as -1, so nothing to do further.
7359 return Res;
7360
7361 // Convert -1 to VL.
7362 SDValue SetCC =
7363 DAG.getSetCC(DL, VT: XLenVT, LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETLT);
7364 Res = DAG.getSelect(DL, VT: XLenVT, Cond: SetCC, LHS: EVL, RHS: Res);
7365 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: Res);
7366}
7367
7368// While RVV has alignment restrictions, we should always be able to load as a
7369// legal equivalently-sized byte-typed vector instead. This method is
7370// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
7371// the load is already correctly-aligned, it returns SDValue().
7372SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
7373 SelectionDAG &DAG) const {
7374 auto *Load = cast<LoadSDNode>(Val&: Op);
7375 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7376
7377 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7378 VT: Load->getMemoryVT(),
7379 MMO: *Load->getMemOperand()))
7380 return SDValue();
7381
7382 SDLoc DL(Op);
7383 MVT VT = Op.getSimpleValueType();
7384 unsigned EltSizeBits = VT.getScalarSizeInBits();
7385 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7386 "Unexpected unaligned RVV load type");
7387 MVT NewVT =
7388 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7389 assert(NewVT.isValid() &&
7390 "Expecting equally-sized RVV vector types to be legal");
7391 SDValue L = DAG.getLoad(VT: NewVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
7392 PtrInfo: Load->getPointerInfo(), Alignment: Load->getBaseAlign(),
7393 MMOFlags: Load->getMemOperand()->getFlags());
7394 return DAG.getMergeValues(Ops: {DAG.getBitcast(VT, V: L), L.getValue(R: 1)}, dl: DL);
7395}
7396
7397// While RVV has alignment restrictions, we should always be able to store as a
7398// legal equivalently-sized byte-typed vector instead. This method is
7399// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
7400// returns SDValue() if the store is already correctly aligned.
7401SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
7402 SelectionDAG &DAG) const {
7403 auto *Store = cast<StoreSDNode>(Val&: Op);
7404 assert(Store && Store->getValue().getValueType().isVector() &&
7405 "Expected vector store");
7406
7407 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7408 VT: Store->getMemoryVT(),
7409 MMO: *Store->getMemOperand()))
7410 return SDValue();
7411
7412 SDLoc DL(Op);
7413 SDValue StoredVal = Store->getValue();
7414 MVT VT = StoredVal.getSimpleValueType();
7415 unsigned EltSizeBits = VT.getScalarSizeInBits();
7416 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7417 "Unexpected unaligned RVV store type");
7418 MVT NewVT =
7419 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7420 assert(NewVT.isValid() &&
7421 "Expecting equally-sized RVV vector types to be legal");
7422 StoredVal = DAG.getBitcast(VT: NewVT, V: StoredVal);
7423 return DAG.getStore(Chain: Store->getChain(), dl: DL, Val: StoredVal, Ptr: Store->getBasePtr(),
7424 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
7425 MMOFlags: Store->getMemOperand()->getFlags());
7426}
7427
7428// While RVV has alignment restrictions, we should always be able to load as a
7429// legal equivalently-sized byte-typed vector instead. This method is
7430// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
7431// the load is already correctly-aligned, it returns SDValue().
7432SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
7433 SelectionDAG &DAG) const {
7434 auto *Load = cast<VPLoadSDNode>(Val&: Op);
7435 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7436
7437 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7438 VT: Load->getMemoryVT(),
7439 MMO: *Load->getMemOperand()))
7440 return SDValue();
7441
7442 SDValue Mask = Load->getMask();
7443
7444 // FIXME: Handled masked loads somehow.
7445 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
7446 return SDValue();
7447
7448 SDLoc DL(Op);
7449 MVT VT = Op.getSimpleValueType();
7450 unsigned EltSizeBits = VT.getScalarSizeInBits();
7451 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7452 "Unexpected unaligned RVV load type");
7453 MVT NewVT =
7454 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7455 assert(NewVT.isValid() &&
7456 "Expecting equally-sized RVV vector types to be legal");
7457
7458 SDValue VL = Load->getVectorLength();
7459 VL = DAG.getNode(Opcode: ISD::MUL, DL, VT: VL.getValueType(), N1: VL,
7460 N2: DAG.getConstant(Val: (EltSizeBits / 8), DL, VT: VL.getValueType()));
7461
7462 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, EC: NewVT.getVectorElementCount());
7463 SDValue L = DAG.getLoadVP(VT: NewVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
7464 Mask: DAG.getAllOnesConstant(DL, VT: MaskVT), EVL: VL,
7465 PtrInfo: Load->getPointerInfo(), Alignment: Load->getBaseAlign(),
7466 MMOFlags: Load->getMemOperand()->getFlags(), AAInfo: AAMDNodes());
7467 return DAG.getMergeValues(Ops: {DAG.getBitcast(VT, V: L), L.getValue(R: 1)}, dl: DL);
7468}
7469
7470// While RVV has alignment restrictions, we should always be able to store as a
7471// legal equivalently-sized byte-typed vector instead. This method is
7472// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
7473// It returns SDValue() if the store is already correctly aligned.
7474SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
7475 SelectionDAG &DAG) const {
7476 auto *Store = cast<VPStoreSDNode>(Val&: Op);
7477 assert(Store && Store->getValue().getValueType().isVector() &&
7478 "Expected vector store");
7479
7480 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7481 VT: Store->getMemoryVT(),
7482 MMO: *Store->getMemOperand()))
7483 return SDValue();
7484
7485 SDValue Mask = Store->getMask();
7486
7487 // FIXME: Handled masked stores somehow.
7488 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
7489 return SDValue();
7490
7491 SDLoc DL(Op);
7492 SDValue StoredVal = Store->getValue();
7493 MVT VT = StoredVal.getSimpleValueType();
7494 unsigned EltSizeBits = VT.getScalarSizeInBits();
7495 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7496 "Unexpected unaligned RVV store type");
7497 MVT NewVT =
7498 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7499 assert(NewVT.isValid() &&
7500 "Expecting equally-sized RVV vector types to be legal");
7501
7502 SDValue VL = Store->getVectorLength();
7503 VL = DAG.getNode(Opcode: ISD::MUL, DL, VT: VL.getValueType(), N1: VL,
7504 N2: DAG.getConstant(Val: (EltSizeBits / 8), DL, VT: VL.getValueType()));
7505
7506 StoredVal = DAG.getBitcast(VT: NewVT, V: StoredVal);
7507
7508 LocationSize Size = LocationSize::precise(Value: NewVT.getStoreSize());
7509 MachineFunction &MF = DAG.getMachineFunction();
7510 MachineMemOperand *MMO = MF.getMachineMemOperand(
7511 PtrInfo: Store->getPointerInfo(), F: Store->getMemOperand()->getFlags(), Size,
7512 BaseAlignment: Store->getBaseAlign());
7513
7514 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, EC: NewVT.getVectorElementCount());
7515 return DAG.getStoreVP(Chain: Store->getChain(), dl: DL, Val: StoredVal, Ptr: Store->getBasePtr(),
7516 Offset: DAG.getUNDEF(VT: Store->getBasePtr().getValueType()),
7517 Mask: DAG.getAllOnesConstant(DL, VT: MaskVT), EVL: VL, MemVT: NewVT, MMO,
7518 AM: ISD::UNINDEXED);
7519}
7520
7521static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
7522 const RISCVSubtarget &Subtarget) {
7523 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
7524
7525 int64_t Imm = cast<ConstantSDNode>(Val&: Op)->getSExtValue();
7526
7527 // All simm32 constants should be handled by isel.
7528 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
7529 // this check redundant, but small immediates are common so this check
7530 // should have better compile time.
7531 if (isInt<32>(x: Imm))
7532 return Op;
7533
7534 // We only need to cost the immediate, if constant pool lowering is enabled.
7535 if (!Subtarget.useConstantPoolForLargeInts())
7536 return Op;
7537
7538 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
7539 if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
7540 return Op;
7541
7542 // Optimizations below are disabled for opt size. If we're optimizing for
7543 // size, use a constant pool.
7544 if (DAG.shouldOptForSize())
7545 return SDValue();
7546
7547 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
7548 // that if it will avoid a constant pool.
7549 // It will require an extra temporary register though.
7550 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
7551 // low and high 32 bits are the same and bit 31 and 63 are set.
7552 unsigned ShiftAmt, AddOpc;
7553 RISCVMatInt::InstSeq SeqLo =
7554 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
7555 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
7556 return Op;
7557
7558 return SDValue();
7559}
7560
7561SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
7562 SelectionDAG &DAG) const {
7563 MVT VT = Op.getSimpleValueType();
7564 const APFloat &Imm = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
7565
7566 // Can this constant be selected by a Zfa FLI instruction?
7567 bool Negate = false;
7568 int Index = getLegalZfaFPImm(Imm, VT);
7569
7570 // If the constant is negative, try negating.
7571 if (Index < 0 && Imm.isNegative()) {
7572 Index = getLegalZfaFPImm(Imm: -Imm, VT);
7573 Negate = true;
7574 }
7575
7576 // If we couldn't find a FLI lowering, fall back to generic code.
7577 if (Index < 0)
7578 return SDValue();
7579
7580 // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
7581 SDLoc DL(Op);
7582 SDValue Const =
7583 DAG.getNode(Opcode: RISCVISD::FLI, DL, VT,
7584 Operand: DAG.getTargetConstant(Val: Index, DL, VT: Subtarget.getXLenVT()));
7585 if (!Negate)
7586 return Const;
7587
7588 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Const);
7589}
7590
7591static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
7592 SelectionDAG &DAG) {
7593
7594 unsigned IsData = Op.getConstantOperandVal(i: 4);
7595
7596 // mips-p8700 we support data prefetch for now.
7597 if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
7598 return Op.getOperand(i: 0);
7599 return Op;
7600}
7601
7602static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
7603 const RISCVSubtarget &Subtarget) {
7604 SDLoc dl(Op);
7605 AtomicOrdering FenceOrdering =
7606 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
7607 SyncScope::ID FenceSSID =
7608 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
7609
7610 if (Subtarget.hasStdExtZtso()) {
7611 // The only fence that needs an instruction is a sequentially-consistent
7612 // cross-thread fence.
7613 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
7614 FenceSSID == SyncScope::System)
7615 return Op;
7616
7617 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7618 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
7619 }
7620
7621 // singlethread fences only synchronize with signal handlers on the same
7622 // thread and thus only need to preserve instruction order, not actually
7623 // enforce memory ordering.
7624 if (FenceSSID == SyncScope::SingleThread)
7625 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7626 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
7627
7628 return Op;
7629}
7630
7631SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
7632 SelectionDAG &DAG) const {
7633 SDLoc DL(Op);
7634 MVT VT = Op.getSimpleValueType();
7635 MVT XLenVT = Subtarget.getXLenVT();
7636 unsigned Check = Op.getConstantOperandVal(i: 1);
7637 unsigned TDCMask = 0;
7638 if (Check & fcSNan)
7639 TDCMask |= RISCV::FPMASK_Signaling_NaN;
7640 if (Check & fcQNan)
7641 TDCMask |= RISCV::FPMASK_Quiet_NaN;
7642 if (Check & fcPosInf)
7643 TDCMask |= RISCV::FPMASK_Positive_Infinity;
7644 if (Check & fcNegInf)
7645 TDCMask |= RISCV::FPMASK_Negative_Infinity;
7646 if (Check & fcPosNormal)
7647 TDCMask |= RISCV::FPMASK_Positive_Normal;
7648 if (Check & fcNegNormal)
7649 TDCMask |= RISCV::FPMASK_Negative_Normal;
7650 if (Check & fcPosSubnormal)
7651 TDCMask |= RISCV::FPMASK_Positive_Subnormal;
7652 if (Check & fcNegSubnormal)
7653 TDCMask |= RISCV::FPMASK_Negative_Subnormal;
7654 if (Check & fcPosZero)
7655 TDCMask |= RISCV::FPMASK_Positive_Zero;
7656 if (Check & fcNegZero)
7657 TDCMask |= RISCV::FPMASK_Negative_Zero;
7658
7659 bool IsOneBitMask = isPowerOf2_32(Value: TDCMask);
7660
7661 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: XLenVT);
7662
7663 if (VT.isVector()) {
7664 SDValue Op0 = Op.getOperand(i: 0);
7665 MVT VT0 = Op.getOperand(i: 0).getSimpleValueType();
7666
7667 if (VT.isScalableVector()) {
7668 MVT DstVT = VT0.changeVectorElementTypeToInteger();
7669 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: VT0, DL, DAG, Subtarget);
7670 SDValue FPCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS_VL, DL, VT: DstVT, N1: Op0, N2: Mask,
7671 N3: VL, Flags: Op->getFlags());
7672 if (IsOneBitMask)
7673 return DAG.getSetCC(DL, VT, LHS: FPCLASS,
7674 RHS: DAG.getConstant(Val: TDCMask, DL, VT: DstVT),
7675 Cond: ISD::CondCode::SETEQ);
7676 SDValue AND = DAG.getNode(Opcode: ISD::AND, DL, VT: DstVT, N1: FPCLASS,
7677 N2: DAG.getConstant(Val: TDCMask, DL, VT: DstVT));
7678 return DAG.getSetCC(DL, VT, LHS: AND, RHS: DAG.getConstant(Val: 0, DL, VT: DstVT),
7679 Cond: ISD::SETNE);
7680 }
7681
7682 MVT ContainerVT0 = getContainerForFixedLengthVector(VT: VT0);
7683 MVT ContainerVT = getContainerForFixedLengthVector(VT);
7684 MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
7685 auto [Mask, VL] = getDefaultVLOps(VecVT: VT0, ContainerVT: ContainerVT0, DL, DAG, Subtarget);
7686 Op0 = convertToScalableVector(VT: ContainerVT0, V: Op0, DAG, Subtarget);
7687
7688 SDValue FPCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS_VL, DL, VT: ContainerDstVT, N1: Op0,
7689 N2: Mask, N3: VL, Flags: Op->getFlags());
7690
7691 TDCMaskV = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerDstVT,
7692 N1: DAG.getUNDEF(VT: ContainerDstVT), N2: TDCMaskV, N3: VL);
7693 if (IsOneBitMask) {
7694 SDValue VMSEQ =
7695 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
7696 Ops: {FPCLASS, TDCMaskV, DAG.getCondCode(Cond: ISD::SETEQ),
7697 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7698 return convertFromScalableVector(VT, V: VMSEQ, DAG, Subtarget);
7699 }
7700 SDValue AND = DAG.getNode(Opcode: RISCVISD::AND_VL, DL, VT: ContainerDstVT, N1: FPCLASS,
7701 N2: TDCMaskV, N3: DAG.getUNDEF(VT: ContainerDstVT), N4: Mask, N5: VL);
7702
7703 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
7704 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerDstVT,
7705 N1: DAG.getUNDEF(VT: ContainerDstVT), N2: SplatZero, N3: VL);
7706
7707 SDValue VMSNE = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
7708 Ops: {AND, SplatZero, DAG.getCondCode(Cond: ISD::SETNE),
7709 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7710 return convertFromScalableVector(VT, V: VMSNE, DAG, Subtarget);
7711 }
7712
7713 SDValue FCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS, DL, VT: XLenVT, Operand: Op.getOperand(i: 0));
7714 SDValue AND = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: FCLASS, N2: TDCMaskV);
7715 SDValue Res = DAG.getSetCC(DL, VT: XLenVT, LHS: AND, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT),
7716 Cond: ISD::CondCode::SETNE);
7717 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
7718}
7719
7720// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
7721// operations propagate nans.
7722static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
7723 const RISCVSubtarget &Subtarget) {
7724 SDLoc DL(Op);
7725 MVT VT = Op.getSimpleValueType();
7726
7727 SDValue X = Op.getOperand(i: 0);
7728 SDValue Y = Op.getOperand(i: 1);
7729
7730 if (!VT.isVector()) {
7731 MVT XLenVT = Subtarget.getXLenVT();
7732
7733 // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
7734 // ensures that when one input is a nan, the other will also be a nan
7735 // allowing the nan to propagate. If both inputs are nan, this will swap the
7736 // inputs which is harmless.
7737
7738 SDValue NewY = Y;
7739 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Op: X)) {
7740 SDValue XIsNonNan = DAG.getSetCC(DL, VT: XLenVT, LHS: X, RHS: X, Cond: ISD::SETOEQ);
7741 NewY = DAG.getSelect(DL, VT, Cond: XIsNonNan, LHS: Y, RHS: X);
7742 }
7743
7744 SDValue NewX = X;
7745 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Op: Y)) {
7746 SDValue YIsNonNan = DAG.getSetCC(DL, VT: XLenVT, LHS: Y, RHS: Y, Cond: ISD::SETOEQ);
7747 NewX = DAG.getSelect(DL, VT, Cond: YIsNonNan, LHS: X, RHS: Y);
7748 }
7749
7750 unsigned Opc =
7751 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
7752 return DAG.getNode(Opcode: Opc, DL, VT, N1: NewX, N2: NewY);
7753 }
7754
7755 // Check no NaNs before converting to fixed vector scalable.
7756 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Op: X);
7757 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Op: Y);
7758
7759 MVT ContainerVT = VT;
7760 if (VT.isFixedLengthVector()) {
7761 ContainerVT = getContainerForFixedLengthVector(VT, Subtarget);
7762 X = convertToScalableVector(VT: ContainerVT, V: X, DAG, Subtarget);
7763 Y = convertToScalableVector(VT: ContainerVT, V: Y, DAG, Subtarget);
7764 }
7765
7766 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
7767
7768 SDValue NewY = Y;
7769 if (!XIsNeverNan) {
7770 SDValue XIsNonNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
7771 Ops: {X, X, DAG.getCondCode(Cond: ISD::SETOEQ),
7772 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7773 NewY = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: XIsNonNan, N2: Y, N3: X,
7774 N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
7775 }
7776
7777 SDValue NewX = X;
7778 if (!YIsNeverNan) {
7779 SDValue YIsNonNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
7780 Ops: {Y, Y, DAG.getCondCode(Cond: ISD::SETOEQ),
7781 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7782 NewX = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: YIsNonNan, N2: X, N3: Y,
7783 N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
7784 }
7785
7786 unsigned Opc =
7787 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::VFMAX_VL : RISCVISD::VFMIN_VL;
7788 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: NewX, N2: NewY,
7789 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
7790 if (VT.isFixedLengthVector())
7791 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
7792 return Res;
7793}
7794
7795static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
7796 const RISCVSubtarget &Subtarget) {
7797 bool IsFABS = Op.getOpcode() == ISD::FABS;
7798 assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
7799 "Wrong opcode for lowering FABS or FNEG.");
7800
7801 MVT XLenVT = Subtarget.getXLenVT();
7802 MVT VT = Op.getSimpleValueType();
7803 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7804
7805 SDLoc DL(Op);
7806 SDValue Fmv =
7807 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Op.getOperand(i: 0));
7808
7809 APInt Mask = IsFABS ? APInt::getSignedMaxValue(numBits: 16) : APInt::getSignMask(BitWidth: 16);
7810 Mask = Mask.sext(width: Subtarget.getXLen());
7811
7812 unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
7813 SDValue Logic =
7814 DAG.getNode(Opcode: LogicOpc, DL, VT: XLenVT, N1: Fmv, N2: DAG.getConstant(Val: Mask, DL, VT: XLenVT));
7815 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: Logic);
7816}
7817
7818static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
7819 const RISCVSubtarget &Subtarget) {
7820 assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
7821
7822 MVT XLenVT = Subtarget.getXLenVT();
7823 MVT VT = Op.getSimpleValueType();
7824 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7825
7826 SDValue Mag = Op.getOperand(i: 0);
7827 SDValue Sign = Op.getOperand(i: 1);
7828
7829 SDLoc DL(Op);
7830
7831 // Get sign bit into an integer value.
7832 unsigned SignSize = Sign.getValueSizeInBits();
7833 SDValue SignAsInt = [&]() {
7834 if (SignSize == Subtarget.getXLen())
7835 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: XLenVT, Operand: Sign);
7836 switch (SignSize) {
7837 case 16:
7838 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Sign);
7839 case 32:
7840 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: XLenVT, Operand: Sign);
7841 case 64: {
7842 assert(XLenVT == MVT::i32 && "Unexpected type");
7843 // Copy the upper word to integer.
7844 SignSize = 32;
7845 return DAG.getNode(Opcode: RISCVISD::SplitF64, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: Sign)
7846 .getValue(R: 1);
7847 }
7848 default:
7849 llvm_unreachable("Unexpected sign size");
7850 }
7851 }();
7852
7853 // Get the signbit at the right position for MagAsInt.
7854 if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
7855 SignAsInt = DAG.getNode(Opcode: ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, VT: XLenVT,
7856 N1: SignAsInt,
7857 N2: DAG.getConstant(Val: std::abs(x: ShiftAmount), DL, VT: XLenVT));
7858
7859 // Mask the sign bit and any bits above it. The extra bits will be dropped
7860 // when we convert back to FP.
7861 SDValue SignMask = DAG.getConstant(
7862 Val: APInt::getSignMask(BitWidth: 16).sext(width: Subtarget.getXLen()), DL, VT: XLenVT);
7863 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: SignAsInt, N2: SignMask);
7864
7865 // Transform Mag value to integer, and clear the sign bit.
7866 SDValue MagAsInt = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Mag);
7867 SDValue ClearSignMask = DAG.getConstant(
7868 Val: APInt::getSignedMaxValue(numBits: 16).sext(width: Subtarget.getXLen()), DL, VT: XLenVT);
7869 SDValue ClearedSign =
7870 DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: MagAsInt, N2: ClearSignMask);
7871
7872 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: XLenVT, N1: ClearedSign, N2: SignBit,
7873 Flags: SDNodeFlags::Disjoint);
7874
7875 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: CopiedSign);
7876}
7877
7878/// Get a RISC-V target specified VL op for a given SDNode.
7879static unsigned getRISCVVLOp(SDValue Op) {
7880#define OP_CASE(NODE) \
7881 case ISD::NODE: \
7882 return RISCVISD::NODE##_VL;
7883#define VP_CASE(NODE) \
7884 case ISD::VP_##NODE: \
7885 return RISCVISD::NODE##_VL;
7886 // clang-format off
7887 switch (Op.getOpcode()) {
7888 default:
7889 llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
7890 OP_CASE(ADD)
7891 OP_CASE(SUB)
7892 OP_CASE(MUL)
7893 OP_CASE(MULHS)
7894 OP_CASE(MULHU)
7895 OP_CASE(SDIV)
7896 OP_CASE(SREM)
7897 OP_CASE(UDIV)
7898 OP_CASE(UREM)
7899 OP_CASE(SHL)
7900 OP_CASE(SRA)
7901 OP_CASE(SRL)
7902 OP_CASE(ROTL)
7903 OP_CASE(ROTR)
7904 OP_CASE(BSWAP)
7905 OP_CASE(CTTZ)
7906 OP_CASE(CTLZ)
7907 OP_CASE(CTPOP)
7908 OP_CASE(BITREVERSE)
7909 OP_CASE(SADDSAT)
7910 OP_CASE(UADDSAT)
7911 OP_CASE(SSUBSAT)
7912 OP_CASE(USUBSAT)
7913 OP_CASE(AVGFLOORS)
7914 OP_CASE(AVGFLOORU)
7915 OP_CASE(AVGCEILS)
7916 OP_CASE(AVGCEILU)
7917 OP_CASE(FADD)
7918 OP_CASE(FSUB)
7919 OP_CASE(FMUL)
7920 OP_CASE(FDIV)
7921 OP_CASE(FNEG)
7922 OP_CASE(FABS)
7923 OP_CASE(FCOPYSIGN)
7924 OP_CASE(FSQRT)
7925 OP_CASE(SMIN)
7926 OP_CASE(SMAX)
7927 OP_CASE(UMIN)
7928 OP_CASE(UMAX)
7929 OP_CASE(ABDS)
7930 OP_CASE(ABDU)
7931 OP_CASE(STRICT_FADD)
7932 OP_CASE(STRICT_FSUB)
7933 OP_CASE(STRICT_FMUL)
7934 OP_CASE(STRICT_FDIV)
7935 OP_CASE(STRICT_FSQRT)
7936 VP_CASE(SDIV) // VP_SDIV
7937 VP_CASE(SREM) // VP_SREM
7938 VP_CASE(UDIV) // VP_UDIV
7939 VP_CASE(UREM) // VP_UREM
7940 case ISD::CTLZ_ZERO_POISON:
7941 return RISCVISD::CTLZ_VL;
7942 case ISD::CTTZ_ZERO_POISON:
7943 return RISCVISD::CTTZ_VL;
7944 case ISD::FMA:
7945 return RISCVISD::VFMADD_VL;
7946 case ISD::STRICT_FMA:
7947 return RISCVISD::STRICT_VFMADD_VL;
7948 case ISD::AND:
7949 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7950 return RISCVISD::VMAND_VL;
7951 return RISCVISD::AND_VL;
7952 case ISD::OR:
7953 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7954 return RISCVISD::VMOR_VL;
7955 return RISCVISD::OR_VL;
7956 case ISD::XOR:
7957 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7958 return RISCVISD::VMXOR_VL;
7959 return RISCVISD::XOR_VL;
7960 case ISD::ANY_EXTEND:
7961 case ISD::ZERO_EXTEND:
7962 return RISCVISD::VZEXT_VL;
7963 case ISD::SIGN_EXTEND:
7964 return RISCVISD::VSEXT_VL;
7965 case ISD::SETCC:
7966 return RISCVISD::SETCC_VL;
7967 case ISD::VSELECT:
7968 return RISCVISD::VMERGE_VL;
7969 case ISD::VP_MERGE:
7970 return RISCVISD::VMERGE_VL;
7971 case ISD::FMINNUM:
7972 case ISD::FMINIMUMNUM:
7973 return RISCVISD::VFMIN_VL;
7974 case ISD::FMAXNUM:
7975 case ISD::FMAXIMUMNUM:
7976 return RISCVISD::VFMAX_VL;
7977 case ISD::LRINT:
7978 case ISD::LLRINT:
7979 return RISCVISD::VFCVT_RM_X_F_VL;
7980 case ISD::MASKED_UDIV:
7981 return RISCVISD::UDIV_VL;
7982 case ISD::MASKED_UREM:
7983 return RISCVISD::UREM_VL;
7984 case ISD::MASKED_SDIV:
7985 return RISCVISD::SDIV_VL;
7986 case ISD::MASKED_SREM:
7987 return RISCVISD::SREM_VL;
7988 }
7989 // clang-format on
7990#undef OP_CASE
7991#undef VP_CASE
7992}
7993
7994static bool isPromotedOpNeedingSplit(SDValue Op,
7995 const RISCVSubtarget &Subtarget,
7996 const TargetLowering &TLI) {
7997 MVT OpVT = Op.getSimpleValueType();
7998 if (!OpVT.isVector())
7999 return false;
8000 MVT EltVT = OpVT.getVectorElementType();
8001 if (!(EltVT == MVT::f16 && Subtarget.hasVInstructionsF16Minimal() &&
8002 !Subtarget.hasVInstructionsF16()) &&
8003 !(EltVT == MVT::bf16 && Subtarget.hasVInstructionsBF16Minimal() &&
8004 (!Subtarget.hasVInstructionsBF16() ||
8005 !llvm::is_contained(Range: ZvfbfaOps, Element: Op.getOpcode()))))
8006 return false;
8007 // Need to split when the same width f32 vector type isn't legal.
8008 return !TLI.isTypeLegal(
8009 VT: MVT::getVectorVT(VT: MVT::f32, EC: OpVT.getVectorElementCount()));
8010}
8011
8012static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
8013 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: Op.getValueType());
8014 SDLoc DL(Op);
8015
8016 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8017 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8018
8019 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8020 if (!Op.getOperand(i: j).getValueType().isVector()) {
8021 LoOperands[j] = Op.getOperand(i: j);
8022 HiOperands[j] = Op.getOperand(i: j);
8023 continue;
8024 }
8025 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
8026 DAG.SplitVector(N: Op.getOperand(i: j), DL);
8027 }
8028
8029 SDValue LoRes =
8030 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: LoVT, Ops: LoOperands, Flags: Op->getFlags());
8031 SDValue HiRes =
8032 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: HiVT, Ops: HiOperands, Flags: Op->getFlags());
8033
8034 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Op.getValueType(), N1: LoRes, N2: HiRes);
8035}
8036
8037static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG,
8038 bool IsVP) {
8039 SDLoc DL(Op);
8040
8041 if (IsVP) {
8042 auto [Lo, Hi] = DAG.SplitVector(N: Op.getOperand(i: 1), DL);
8043 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Op.getOperand(i: 2), DL);
8044 auto [EVLLo, EVLHi] =
8045 DAG.SplitEVL(N: Op.getOperand(i: 3), VecVT: Op.getOperand(i: 1).getValueType(), DL);
8046
8047 SDValue ResLo =
8048 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
8049 Ops: {Op.getOperand(i: 0), Lo, MaskLo, EVLLo}, Flags: Op->getFlags());
8050 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
8051 Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags: Op->getFlags());
8052 }
8053
8054 unsigned Opcode = Op.getOpcode();
8055 unsigned OpNo = Opcode == ISD::VECREDUCE_SEQ_FADD ? 1 : 0;
8056
8057 auto [Lo, Hi] = DAG.SplitVector(N: Op.getOperand(i: OpNo), DL);
8058 if (Opcode == ISD::VECREDUCE_SEQ_FADD) {
8059 SDValue ResLo = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
8060 N1: Op.getOperand(i: 0), N2: Lo, Flags: Op->getFlags());
8061 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), N1: ResLo, N2: Hi,
8062 Flags: Op->getFlags());
8063 }
8064
8065 SDValue ResLo =
8066 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: Lo, Flags: Op->getFlags());
8067 SDValue ResHi =
8068 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: Hi, Flags: Op->getFlags());
8069 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
8070 return DAG.getNode(Opcode: BaseOpc, DL, VT: Op.getValueType(), N1: ResLo, N2: ResHi,
8071 Flags: Op->getFlags());
8072}
8073
8074static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
8075
8076 assert(Op->isStrictFPOpcode());
8077
8078 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: Op->getValueType(ResNo: 0));
8079
8080 SDVTList LoVTs = DAG.getVTList(VT1: LoVT, VT2: Op->getValueType(ResNo: 1));
8081 SDVTList HiVTs = DAG.getVTList(VT1: HiVT, VT2: Op->getValueType(ResNo: 1));
8082
8083 SDLoc DL(Op);
8084
8085 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
8086 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
8087
8088 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
8089 if (!Op.getOperand(i: j).getValueType().isVector()) {
8090 LoOperands[j] = Op.getOperand(i: j);
8091 HiOperands[j] = Op.getOperand(i: j);
8092 continue;
8093 }
8094 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
8095 DAG.SplitVector(N: Op.getOperand(i: j), DL);
8096 }
8097
8098 SDValue LoRes =
8099 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: LoVTs, Ops: LoOperands, Flags: Op->getFlags());
8100 HiOperands[0] = LoRes.getValue(R: 1);
8101 SDValue HiRes =
8102 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: HiVTs, Ops: HiOperands, Flags: Op->getFlags());
8103
8104 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Op->getValueType(ResNo: 0),
8105 N1: LoRes.getValue(R: 0), N2: HiRes.getValue(R: 0));
8106 return DAG.getMergeValues(Ops: {V, HiRes.getValue(R: 1)}, dl: DL);
8107}
8108
8109SDValue
8110RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
8111 SelectionDAG &DAG) const {
8112 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8113 "Unexpected bfloat16 load lowering");
8114
8115 SDLoc DL(Op);
8116 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
8117 EVT MemVT = LD->getMemoryVT();
8118 SDValue Load = DAG.getExtLoad(
8119 ExtType: ISD::ZEXTLOAD, dl: DL, VT: Subtarget.getXLenVT(), Chain: LD->getChain(),
8120 Ptr: LD->getBasePtr(),
8121 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
8122 MMO: LD->getMemOperand());
8123 // Using mask to make bf16 nan-boxing valid when we don't have flh
8124 // instruction. -65536 would be treat as a small number and thus it can be
8125 // directly used lui to get the constant.
8126 SDValue mask = DAG.getSignedConstant(Val: -65536, DL, VT: Subtarget.getXLenVT());
8127 SDValue OrSixteenOne =
8128 DAG.getNode(Opcode: ISD::OR, DL, VT: Load.getValueType(), Ops: {Load, mask});
8129 SDValue ConvertedResult =
8130 DAG.getNode(Opcode: RISCVISD::NDS_FMV_BF16_X, DL, VT: MVT::bf16, Operand: OrSixteenOne);
8131 return DAG.getMergeValues(Ops: {ConvertedResult, Load.getValue(R: 1)}, dl: DL);
8132}
8133
8134SDValue
8135RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
8136 SelectionDAG &DAG) const {
8137 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
8138 "Unexpected bfloat16 store lowering");
8139
8140 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
8141 SDLoc DL(Op);
8142 SDValue FMV = DAG.getNode(Opcode: RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
8143 VT: Subtarget.getXLenVT(), Operand: ST->getValue());
8144 return DAG.getTruncStore(
8145 Chain: ST->getChain(), dl: DL, Val: FMV, Ptr: ST->getBasePtr(),
8146 SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ST->getMemoryVT().getSizeInBits()),
8147 MMO: ST->getMemOperand());
8148}
8149
8150static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
8151 const RISCVSubtarget &Subtarget);
8152
8153SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
8154 SelectionDAG &DAG) const {
8155 switch (Op.getOpcode()) {
8156 default:
8157 reportFatalInternalError(
8158 reason: "Unimplemented RISCVTargetLowering::LowerOperation Case");
8159 case ISD::PREFETCH:
8160 return LowerPREFETCH(Op, Subtarget, DAG);
8161 case ISD::ATOMIC_FENCE:
8162 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
8163 case ISD::GlobalAddress:
8164 return lowerGlobalAddress(Op, DAG);
8165 case ISD::BlockAddress:
8166 return lowerBlockAddress(Op, DAG);
8167 case ISD::ConstantPool:
8168 return lowerConstantPool(Op, DAG);
8169 case ISD::JumpTable:
8170 return lowerJumpTable(Op, DAG);
8171 case ISD::GlobalTLSAddress:
8172 return lowerGlobalTLSAddress(Op, DAG);
8173 case ISD::Constant:
8174 return lowerConstant(Op, DAG, Subtarget);
8175 case ISD::ConstantFP:
8176 return lowerConstantFP(Op, DAG);
8177 case ISD::SELECT:
8178 return lowerSELECT(Op, DAG);
8179 case ISD::BRCOND:
8180 return lowerBRCOND(Op, DAG);
8181 case ISD::VASTART:
8182 return lowerVASTART(Op, DAG);
8183 case ISD::FRAMEADDR:
8184 return lowerFRAMEADDR(Op, DAG);
8185 case ISD::RETURNADDR:
8186 return lowerRETURNADDR(Op, DAG);
8187 case ISD::SHL_PARTS:
8188 return lowerShiftLeftParts(Op, DAG);
8189 case ISD::SRA_PARTS:
8190 return lowerShiftRightParts(Op, DAG, IsSRA: true);
8191 case ISD::SRL_PARTS:
8192 return lowerShiftRightParts(Op, DAG, IsSRA: false);
8193 case ISD::ROTL:
8194 case ISD::ROTR:
8195 if (Op.getValueType().isFixedLengthVector()) {
8196 assert(Subtarget.hasStdExtZvkb());
8197 return lowerToScalableOp(Op, DAG);
8198 }
8199 assert(Subtarget.hasVendorXTHeadBb() &&
8200 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
8201 "Unexpected custom legalization");
8202 // XTHeadBb only supports rotate by constant.
8203 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
8204 return SDValue();
8205 return Op;
8206 case ISD::BITCAST: {
8207 SDLoc DL(Op);
8208 EVT VT = Op.getValueType();
8209 SDValue Op0 = Op.getOperand(i: 0);
8210 EVT Op0VT = Op0.getValueType();
8211 MVT XLenVT = Subtarget.getXLenVT();
8212 if (Op0VT == MVT::i16 &&
8213 ((VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
8214 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
8215 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Op0);
8216 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: NewOp0);
8217 }
8218 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
8219 Subtarget.hasStdExtFOrZfinx()) {
8220 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
8221 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: NewOp0);
8222 }
8223 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit() &&
8224 Subtarget.hasStdExtDOrZdinx()) {
8225 SDValue Lo, Hi;
8226 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op0, DL, LoVT: MVT::i32, HiVT: MVT::i32);
8227 return DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
8228 }
8229
8230 if (Subtarget.hasStdExtP() && VT.isSimple() && Op0VT.isSimple()) {
8231 if (VT.getSimpleVT() == Subtarget.getXLenVT() &&
8232 Subtarget.isPExtPackedType(VT: Op0VT.getSimpleVT()))
8233 return Op;
8234 if (Op0VT.getSimpleVT() == Subtarget.getXLenVT() &&
8235 Subtarget.isPExtPackedType(VT: VT.getSimpleVT()))
8236 return Op;
8237 }
8238
8239 // Consider other scalar<->scalar casts as legal if the types are legal.
8240 // Otherwise expand them.
8241 if (!VT.isVector() && !Op0VT.isVector()) {
8242 if (isTypeLegal(VT) && isTypeLegal(VT: Op0VT))
8243 return Op;
8244 return SDValue();
8245 }
8246
8247 assert(!VT.isScalableVector() && !Op0VT.isScalableVector() &&
8248 "Unexpected types");
8249
8250 if (VT.isFixedLengthVector()) {
8251 // We can handle fixed length vector bitcasts with a simple replacement
8252 // in isel.
8253 if (Op0VT.isFixedLengthVector())
8254 return Op;
8255 // When bitcasting from scalar to fixed-length vector, insert the scalar
8256 // into a one-element vector of the result type, and perform a vector
8257 // bitcast.
8258 if (!Op0VT.isVector()) {
8259 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: Op0VT, NumElements: 1);
8260 if (!isTypeLegal(VT: BVT))
8261 return SDValue();
8262 return DAG.getBitcast(
8263 VT, V: DAG.getInsertVectorElt(DL, Vec: DAG.getUNDEF(VT: BVT), Elt: Op0, Idx: 0));
8264 }
8265 return SDValue();
8266 }
8267 // Custom-legalize bitcasts from fixed-length vector types to scalar types
8268 // thus: bitcast the vector to a one-element vector type whose element type
8269 // is the same as the result type, and extract the first element.
8270 if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
8271 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 1);
8272 if (!isTypeLegal(VT: BVT))
8273 return SDValue();
8274 SDValue BVec = DAG.getBitcast(VT: BVT, V: Op0);
8275 return DAG.getExtractVectorElt(DL, VT, Vec: BVec, Idx: 0);
8276 }
8277 return SDValue();
8278 }
8279 case ISD::INTRINSIC_WO_CHAIN:
8280 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
8281 case ISD::INTRINSIC_W_CHAIN:
8282 return LowerINTRINSIC_W_CHAIN(Op, DAG);
8283 case ISD::INTRINSIC_VOID:
8284 return LowerINTRINSIC_VOID(Op, DAG);
8285 case ISD::IS_FPCLASS:
8286 return LowerIS_FPCLASS(Op, DAG);
8287 case ISD::BITREVERSE: {
8288 MVT VT = Op.getSimpleValueType();
8289 if (VT.isFixedLengthVector()) {
8290 assert(Subtarget.hasStdExtZvbb());
8291 return lowerToScalableOp(Op, DAG);
8292 }
8293 SDLoc DL(Op);
8294 assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
8295 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
8296 // Expand bitreverse to a bswap(rev8) followed by brev8.
8297 SDValue BSwap = DAG.getNode(Opcode: ISD::BSWAP, DL, VT, Operand: Op.getOperand(i: 0));
8298 return DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT, Operand: BSwap);
8299 }
8300 case ISD::TRUNCATE:
8301 case ISD::TRUNCATE_SSAT_S:
8302 case ISD::TRUNCATE_USAT_U:
8303 // Only custom-lower vector truncates
8304 if (!Op.getSimpleValueType().isVector())
8305 return Op;
8306 return lowerVectorTrunc(Op, DAG);
8307 case ISD::ANY_EXTEND:
8308 case ISD::ZERO_EXTEND:
8309 if (Op.getOperand(i: 0).getValueType().isVector() &&
8310 Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8311 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ ExtTrueVal: 1);
8312 if (Op.getValueType().isScalableVector())
8313 return Op;
8314 return lowerToScalableOp(Op, DAG);
8315 case ISD::SIGN_EXTEND:
8316 if (Op.getOperand(i: 0).getValueType().isVector() &&
8317 Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8318 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ ExtTrueVal: -1);
8319 if (Op.getValueType().isScalableVector())
8320 return Op;
8321 return lowerToScalableOp(Op, DAG);
8322 case ISD::SPLAT_VECTOR_PARTS:
8323 return lowerSPLAT_VECTOR_PARTS(Op, DAG);
8324 case ISD::INSERT_VECTOR_ELT:
8325 return lowerINSERT_VECTOR_ELT(Op, DAG);
8326 case ISD::EXTRACT_VECTOR_ELT:
8327 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
8328 case ISD::SCALAR_TO_VECTOR: {
8329 MVT VT = Op.getSimpleValueType();
8330 SDLoc DL(Op);
8331 SDValue Scalar = Op.getOperand(i: 0);
8332 if (VT.getVectorElementType() == MVT::i1) {
8333 MVT WideVT = VT.changeVectorElementType(EltVT: MVT::i8);
8334 SDValue V = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: WideVT, Operand: Scalar);
8335 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: V);
8336 }
8337 MVT ContainerVT = VT;
8338 if (VT.isFixedLengthVector())
8339 ContainerVT = getContainerForFixedLengthVector(VT);
8340 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
8341
8342 SDValue V;
8343 if (VT.isFloatingPoint()) {
8344 V = DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT: ContainerVT,
8345 N1: DAG.getUNDEF(VT: ContainerVT), N2: Scalar, N3: VL);
8346 } else {
8347 Scalar = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: Scalar);
8348 V = DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT: ContainerVT,
8349 N1: DAG.getUNDEF(VT: ContainerVT), N2: Scalar, N3: VL);
8350 }
8351 if (VT.isFixedLengthVector())
8352 V = convertFromScalableVector(VT, V, DAG, Subtarget);
8353 return V;
8354 }
8355 case ISD::VSCALE: {
8356 MVT XLenVT = Subtarget.getXLenVT();
8357 MVT VT = Op.getSimpleValueType();
8358 SDLoc DL(Op);
8359 SDValue Res = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
8360 // We define our scalable vector types for lmul=1 to use a 64 bit known
8361 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
8362 // vscale as VLENB / 8.
8363 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
8364 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
8365 reportFatalInternalError(reason: "Support for VLEN==32 is incomplete.");
8366 // We assume VLENB is a multiple of 8. We manually choose the best shift
8367 // here because SimplifyDemandedBits isn't always able to simplify it.
8368 uint64_t Val = Op.getConstantOperandVal(i: 0);
8369 if (isPowerOf2_64(Value: Val)) {
8370 uint64_t Log2 = Log2_64(Value: Val);
8371 if (Log2 < 3) {
8372 SDNodeFlags Flags;
8373 Flags.setExact(true);
8374 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Res,
8375 N2: DAG.getConstant(Val: 3 - Log2, DL, VT: XLenVT), Flags);
8376 } else if (Log2 > 3) {
8377 Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: Res,
8378 N2: DAG.getConstant(Val: Log2 - 3, DL, VT: XLenVT));
8379 }
8380 } else if ((Val % 8) == 0) {
8381 // If the multiplier is a multiple of 8, scale it down to avoid needing
8382 // to shift the VLENB value.
8383 Res = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Res,
8384 N2: DAG.getConstant(Val: Val / 8, DL, VT: XLenVT));
8385 } else {
8386 SDNodeFlags Flags;
8387 Flags.setExact(true);
8388 SDValue VScale = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Res,
8389 N2: DAG.getConstant(Val: 3, DL, VT: XLenVT), Flags);
8390 Res = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: VScale,
8391 N2: DAG.getConstant(Val, DL, VT: XLenVT));
8392 }
8393 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
8394 }
8395 case ISD::FPOWI: {
8396 // Custom promote f16 powi with illegal i32 integer type on RV64. Once
8397 // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
8398 if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
8399 Op.getOperand(i: 1).getValueType() == MVT::i32) {
8400 SDLoc DL(Op);
8401 SDValue Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op.getOperand(i: 0));
8402 SDValue Powi =
8403 DAG.getNode(Opcode: ISD::FPOWI, DL, VT: MVT::f32, N1: Op0, N2: Op.getOperand(i: 1));
8404 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Powi,
8405 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
8406 }
8407 return SDValue();
8408 }
8409 case ISD::FMAXIMUM:
8410 case ISD::FMINIMUM:
8411 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
8412 return SplitVectorOp(Op, DAG);
8413 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
8414 case ISD::FP_EXTEND:
8415 case ISD::FP_ROUND:
8416 return lowerVectorFPExtendOrRound(Op, DAG);
8417 case ISD::STRICT_FP_ROUND:
8418 case ISD::STRICT_FP_EXTEND:
8419 return lowerStrictFPExtendOrRoundLike(Op, DAG);
8420 case ISD::SINT_TO_FP:
8421 case ISD::UINT_TO_FP:
8422 // Fall back to zvfbfmin for bf16 case if source type is wider than 8 bits.
8423 if (SDValue Op1 = Op.getOperand(i: 0);
8424 Op.getValueType().isVector() &&
8425 ((Op.getValueType().getScalarType() == MVT::f16 &&
8426 (Subtarget.hasVInstructionsF16Minimal() &&
8427 !Subtarget.hasVInstructionsF16())) ||
8428 (Op.getValueType().getScalarType() == MVT::bf16 &&
8429 (Subtarget.hasVInstructionsBF16Minimal() &&
8430 (!Subtarget.hasVInstructionsBF16() ||
8431 Op1.getValueType().getScalarSizeInBits() > 8))))) {
8432 MVT NVT =
8433 MVT::getVectorVT(VT: MVT::f32, EC: Op.getValueType().getVectorElementCount());
8434 if (!isTypeLegal(VT: NVT))
8435 return SplitVectorOp(Op, DAG);
8436 // int -> f32
8437 SDLoc DL(Op);
8438 SDValue NC = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: NVT, Ops: Op->ops());
8439 // f32 -> [b]f16
8440 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Op.getValueType(), N1: NC,
8441 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
8442 }
8443 [[fallthrough]];
8444 case ISD::FP_TO_SINT:
8445 case ISD::FP_TO_UINT:
8446 // Fall back to zvfbfmin for bf16 case if destination type is wider than 8
8447 // bits.
8448 if (SDValue Op1 = Op.getOperand(i: 0);
8449 Op1.getValueType().isVector() &&
8450 ((Op1.getValueType().getScalarType() == MVT::f16 &&
8451 (Subtarget.hasVInstructionsF16Minimal() &&
8452 !Subtarget.hasVInstructionsF16())) ||
8453 (Op1.getValueType().getScalarType() == MVT::bf16 &&
8454 (Subtarget.hasVInstructionsBF16Minimal() &&
8455 (!Subtarget.hasVInstructionsBF16() ||
8456 Op.getValueType().getScalarSizeInBits() > 8))))) {
8457 MVT NVT = MVT::getVectorVT(VT: MVT::f32,
8458 EC: Op1.getValueType().getVectorElementCount());
8459 if (!isTypeLegal(VT: NVT))
8460 return SplitVectorOp(Op, DAG);
8461 // [b]f16 -> f32
8462 SDLoc DL(Op);
8463 SDValue WidenVec = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NVT, Operand: Op1);
8464 // f32 -> int
8465 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: WidenVec);
8466 }
8467 [[fallthrough]];
8468 case ISD::STRICT_FP_TO_SINT:
8469 case ISD::STRICT_FP_TO_UINT:
8470 case ISD::STRICT_SINT_TO_FP:
8471 case ISD::STRICT_UINT_TO_FP: {
8472 // RVV can only do fp<->int conversions to types half/double the size as
8473 // the source. We custom-lower any conversions that do two hops into
8474 // sequences.
8475 MVT VT = Op.getSimpleValueType();
8476 if (VT.isScalarInteger())
8477 return lowerFP_TO_INT(Op, DAG, Subtarget);
8478 bool IsStrict = Op->isStrictFPOpcode();
8479 SDValue Src = Op.getOperand(i: 0 + IsStrict);
8480 MVT SrcVT = Src.getSimpleValueType();
8481 if (SrcVT.isScalarInteger())
8482 return lowerINT_TO_FP(Op, DAG, Subtarget);
8483 if (!VT.isVector())
8484 return Op;
8485 SDLoc DL(Op);
8486 MVT EltVT = VT.getVectorElementType();
8487 MVT SrcEltVT = SrcVT.getVectorElementType();
8488 unsigned EltSize = EltVT.getSizeInBits();
8489 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
8490 assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
8491 "Unexpected vector element types");
8492
8493 bool IsInt2FP = SrcEltVT.isInteger();
8494 // Widening conversions
8495 if (EltSize > (2 * SrcEltSize)) {
8496 if (IsInt2FP) {
8497 // Do a regular integer sign/zero extension then convert to float.
8498 MVT IVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSize / 2),
8499 EC: VT.getVectorElementCount());
8500 unsigned ExtOpcode = (Op.getOpcode() == ISD::UINT_TO_FP ||
8501 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
8502 ? ISD::ZERO_EXTEND
8503 : ISD::SIGN_EXTEND;
8504 SDValue Ext = DAG.getNode(Opcode: ExtOpcode, DL, VT: IVecVT, Operand: Src);
8505 if (IsStrict)
8506 return DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: Op->getVTList(),
8507 N1: Op.getOperand(i: 0), N2: Ext);
8508 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT, Operand: Ext);
8509 }
8510 // FP2Int
8511 assert((SrcEltVT == MVT::f16 || SrcEltVT == MVT::bf16) &&
8512 "Unexpected FP_TO_[US]INT lowering");
8513 // Do one doubling fp_extend then complete the operation by converting
8514 // to int.
8515 MVT InterimFVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
8516 if (IsStrict) {
8517 auto [FExt, Chain] =
8518 DAG.getStrictFPExtendOrRound(Op: Src, Chain: Op.getOperand(i: 0), DL, VT: InterimFVT);
8519 return DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: Op->getVTList(), N1: Chain, N2: FExt);
8520 }
8521 SDValue FExt = DAG.getFPExtendOrRound(Op: Src, DL, VT: InterimFVT);
8522 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT, Operand: FExt);
8523 }
8524
8525 // Narrowing conversions
8526 if (SrcEltSize > (2 * EltSize)) {
8527 if (IsInt2FP) {
8528 // One narrowing int_to_fp, then an fp_round.
8529 assert((EltVT == MVT::f16 || EltVT == MVT::bf16) &&
8530 "Unexpected [US]_TO_FP lowering");
8531 MVT InterimFVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
8532 if (IsStrict) {
8533 SDValue Int2FP = DAG.getNode(Opcode: Op.getOpcode(), DL,
8534 VTList: DAG.getVTList(VT1: InterimFVT, VT2: MVT::Other),
8535 N1: Op.getOperand(i: 0), N2: Src);
8536 SDValue Chain = Int2FP.getValue(R: 1);
8537 return DAG.getStrictFPExtendOrRound(Op: Int2FP, Chain, DL, VT).first;
8538 }
8539 SDValue Int2FP = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: InterimFVT, Operand: Src);
8540 return DAG.getFPExtendOrRound(Op: Int2FP, DL, VT);
8541 }
8542 // FP2Int
8543 // One narrowing fp_to_int, then truncate the integer. If the float isn't
8544 // representable by the integer, the result is poison.
8545 MVT IVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SrcEltSize / 2),
8546 EC: VT.getVectorElementCount());
8547 if (IsStrict) {
8548 SDValue FP2Int =
8549 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: DAG.getVTList(VT1: IVecVT, VT2: MVT::Other),
8550 N1: Op.getOperand(i: 0), N2: Src);
8551 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FP2Int);
8552 return DAG.getMergeValues(Ops: {Res, FP2Int.getValue(R: 1)}, dl: DL);
8553 }
8554 SDValue FP2Int = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: IVecVT, Operand: Src);
8555 if (EltSize == 1)
8556 // The integer should be 0 or 1/-1, so compare the integer result to 0.
8557 return DAG.getSetCC(DL, VT, LHS: DAG.getConstant(Val: 0, DL, VT: IVecVT), RHS: FP2Int,
8558 Cond: ISD::SETNE);
8559 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FP2Int);
8560 }
8561
8562 // Scalable vectors can exit here. Patterns will handle equally-sized
8563 // conversions halving/doubling ones.
8564 if (!VT.isFixedLengthVector())
8565 return Op;
8566
8567 // For fixed-length vectors we lower to a custom "VL" node.
8568 unsigned RVVOpc = 0;
8569 switch (Op.getOpcode()) {
8570 default:
8571 llvm_unreachable("Impossible opcode");
8572 case ISD::FP_TO_SINT:
8573 RVVOpc = RISCVISD::VFCVT_RTZ_X_F_VL;
8574 break;
8575 case ISD::FP_TO_UINT:
8576 RVVOpc = RISCVISD::VFCVT_RTZ_XU_F_VL;
8577 break;
8578 case ISD::SINT_TO_FP:
8579 RVVOpc = RISCVISD::SINT_TO_FP_VL;
8580 break;
8581 case ISD::UINT_TO_FP:
8582 RVVOpc = RISCVISD::UINT_TO_FP_VL;
8583 break;
8584 case ISD::STRICT_FP_TO_SINT:
8585 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_X_F_VL;
8586 break;
8587 case ISD::STRICT_FP_TO_UINT:
8588 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_XU_F_VL;
8589 break;
8590 case ISD::STRICT_SINT_TO_FP:
8591 RVVOpc = RISCVISD::STRICT_SINT_TO_FP_VL;
8592 break;
8593 case ISD::STRICT_UINT_TO_FP:
8594 RVVOpc = RISCVISD::STRICT_UINT_TO_FP_VL;
8595 break;
8596 }
8597
8598 MVT ContainerVT = getContainerForFixedLengthVector(VT);
8599 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
8600 assert(ContainerVT.getVectorElementCount() == SrcContainerVT.getVectorElementCount() &&
8601 "Expected same element count");
8602
8603 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
8604
8605 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
8606 if (IsStrict) {
8607 Src = DAG.getNode(Opcode: RVVOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
8608 N1: Op.getOperand(i: 0), N2: Src, N3: Mask, N4: VL);
8609 SDValue SubVec = convertFromScalableVector(VT, V: Src, DAG, Subtarget);
8610 return DAG.getMergeValues(Ops: {SubVec, Src.getValue(R: 1)}, dl: DL);
8611 }
8612 Src = DAG.getNode(Opcode: RVVOpc, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
8613 return convertFromScalableVector(VT, V: Src, DAG, Subtarget);
8614 }
8615 case ISD::FP_TO_SINT_SAT:
8616 case ISD::FP_TO_UINT_SAT:
8617 return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
8618 case ISD::FP_TO_BF16: {
8619 // Custom lower to ensure the libcall return is passed in an FPR on hard
8620 // float ABIs.
8621 assert(!Subtarget.isSoftFPABI() && "Unexpected custom legalization");
8622 SDLoc DL(Op);
8623 MakeLibCallOptions CallOptions;
8624 RTLIB::Libcall LC =
8625 RTLIB::getFPROUND(OpVT: Op.getOperand(i: 0).getValueType(), RetVT: MVT::bf16);
8626 SDValue Res =
8627 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
8628 if (Subtarget.is64Bit())
8629 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Res);
8630 return DAG.getBitcast(VT: MVT::i32, V: Res);
8631 }
8632 case ISD::BF16_TO_FP: {
8633 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalization");
8634 MVT VT = Op.getSimpleValueType();
8635 SDLoc DL(Op);
8636 Op = DAG.getNode(
8637 Opcode: ISD::SHL, DL, VT: Op.getOperand(i: 0).getValueType(), N1: Op.getOperand(i: 0),
8638 N2: DAG.getShiftAmountConstant(Val: 16, VT: Op.getOperand(i: 0).getValueType(), DL));
8639 SDValue Res = Subtarget.is64Bit()
8640 ? DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Op)
8641 : DAG.getBitcast(VT: MVT::f32, V: Op);
8642 // fp_extend if the target VT is bigger than f32.
8643 if (VT != MVT::f32)
8644 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Res);
8645 return Res;
8646 }
8647 case ISD::STRICT_FP_TO_FP16:
8648 case ISD::FP_TO_FP16: {
8649 // Custom lower to ensure the libcall return is passed in an FPR on hard
8650 // float ABIs.
8651 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8652 SDLoc DL(Op);
8653 MakeLibCallOptions CallOptions;
8654 bool IsStrict = Op->isStrictFPOpcode();
8655 SDValue Op0 = IsStrict ? Op.getOperand(i: 1) : Op.getOperand(i: 0);
8656 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8657 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op0.getValueType(), RetVT: MVT::f16);
8658 SDValue Res;
8659 std::tie(args&: Res, args&: Chain) =
8660 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op0, CallOptions, dl: DL, Chain);
8661 if (Subtarget.is64Bit())
8662 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Res);
8663 SDValue Result = DAG.getBitcast(VT: MVT::i32, V: IsStrict ? Res.getValue(R: 0) : Res);
8664 if (IsStrict)
8665 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
8666 return Result;
8667 }
8668 case ISD::STRICT_FP16_TO_FP:
8669 case ISD::FP16_TO_FP: {
8670 // Custom lower to ensure the libcall argument is passed in an FPR on hard
8671 // float ABIs.
8672 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8673 SDLoc DL(Op);
8674 MakeLibCallOptions CallOptions;
8675 bool IsStrict = Op->isStrictFPOpcode();
8676 SDValue Op0 = IsStrict ? Op.getOperand(i: 1) : Op.getOperand(i: 0);
8677 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8678 SDValue Arg = Subtarget.is64Bit()
8679 ? DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Op0)
8680 : DAG.getBitcast(VT: MVT::f32, V: Op0);
8681 SDValue Res;
8682 std::tie(args&: Res, args&: Chain) = makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Arg,
8683 CallOptions, dl: DL, Chain);
8684 if (IsStrict)
8685 return DAG.getMergeValues(Ops: {Res, Chain}, dl: DL);
8686 return Res;
8687 }
8688 case ISD::FTRUNC:
8689 case ISD::FCEIL:
8690 case ISD::FFLOOR:
8691 case ISD::FNEARBYINT:
8692 case ISD::FRINT:
8693 case ISD::FROUND:
8694 case ISD::FROUNDEVEN:
8695 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
8696 return SplitVectorOp(Op, DAG);
8697 return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
8698 case ISD::FCANONICALIZE: {
8699 MVT VT = Op.getSimpleValueType();
8700 assert(VT.isFixedLengthVector() && "Unexpected type");
8701 SDLoc DL(Op);
8702 MVT ContainerVT = getContainerForFixedLengthVector(VT);
8703 SDValue Src =
8704 convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i: 0), DAG, Subtarget);
8705 SDValue Res = DAG.getNode(Opcode: ISD::FCANONICALIZE, DL, VT: ContainerVT, Operand: Src);
8706 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
8707 }
8708 case ISD::LRINT:
8709 case ISD::LLRINT:
8710 case ISD::LROUND:
8711 case ISD::LLROUND: {
8712 if (Op.getValueType().isVector())
8713 return lowerVectorXRINT_XROUND(Op, DAG, Subtarget);
8714 assert(Op.getOperand(0).getValueType() == MVT::f16 &&
8715 "Unexpected custom legalisation");
8716 SDLoc DL(Op);
8717 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op.getOperand(i: 0));
8718 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: Ext);
8719 }
8720 case ISD::STRICT_LRINT:
8721 case ISD::STRICT_LLRINT:
8722 case ISD::STRICT_LROUND:
8723 case ISD::STRICT_LLROUND: {
8724 assert(Op.getOperand(1).getValueType() == MVT::f16 &&
8725 "Unexpected custom legalisation");
8726 SDLoc DL(Op);
8727 SDValue Ext = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
8728 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
8729 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
8730 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
8731 }
8732 case ISD::VECREDUCE_ADD:
8733 case ISD::VECREDUCE_UMAX:
8734 case ISD::VECREDUCE_SMAX:
8735 case ISD::VECREDUCE_UMIN:
8736 case ISD::VECREDUCE_SMIN:
8737 return lowerVECREDUCE(Op, DAG);
8738 case ISD::VECREDUCE_AND:
8739 case ISD::VECREDUCE_OR:
8740 case ISD::VECREDUCE_XOR:
8741 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8742 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
8743 return lowerVECREDUCE(Op, DAG);
8744 case ISD::VECREDUCE_SEQ_FADD:
8745 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 1), Subtarget, TLI: *this))
8746 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
8747 return lowerFPVECREDUCE(Op, DAG);
8748 case ISD::VECREDUCE_FADD:
8749 case ISD::VECREDUCE_FMIN:
8750 case ISD::VECREDUCE_FMAX:
8751 case ISD::VECREDUCE_FMAXIMUM:
8752 case ISD::VECREDUCE_FMINIMUM:
8753 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 0), Subtarget, TLI: *this))
8754 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ false);
8755 return lowerFPVECREDUCE(Op, DAG);
8756 case ISD::VP_REDUCE_ADD:
8757 case ISD::VP_REDUCE_UMAX:
8758 case ISD::VP_REDUCE_SMAX:
8759 case ISD::VP_REDUCE_UMIN:
8760 case ISD::VP_REDUCE_SMIN:
8761 case ISD::VP_REDUCE_FADD:
8762 case ISD::VP_REDUCE_SEQ_FADD:
8763 case ISD::VP_REDUCE_FMIN:
8764 case ISD::VP_REDUCE_FMAX:
8765 case ISD::VP_REDUCE_FMINIMUM:
8766 case ISD::VP_REDUCE_FMAXIMUM:
8767 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 1), Subtarget, TLI: *this))
8768 return SplitVectorReductionOp(Op, DAG, /*IsVP*/ true);
8769 return lowerVPREDUCE(Op, DAG);
8770 case ISD::VP_REDUCE_AND:
8771 case ISD::VP_REDUCE_OR:
8772 case ISD::VP_REDUCE_XOR:
8773 if (Op.getOperand(i: 1).getValueType().getVectorElementType() == MVT::i1)
8774 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
8775 return lowerVPREDUCE(Op, DAG);
8776 case ISD::VP_CTTZ_ELTS:
8777 case ISD::VP_CTTZ_ELTS_ZERO_POISON:
8778 return lowerVPCttzElements(Op, DAG);
8779 case ISD::UNDEF: {
8780 MVT ContainerVT = getContainerForFixedLengthVector(VT: Op.getSimpleValueType());
8781 return convertFromScalableVector(VT: Op.getSimpleValueType(),
8782 V: DAG.getUNDEF(VT: ContainerVT), DAG, Subtarget);
8783 }
8784 case ISD::INSERT_SUBVECTOR:
8785 return lowerINSERT_SUBVECTOR(Op, DAG);
8786 case ISD::EXTRACT_SUBVECTOR:
8787 return lowerEXTRACT_SUBVECTOR(Op, DAG);
8788 case ISD::VECTOR_DEINTERLEAVE:
8789 return lowerVECTOR_DEINTERLEAVE(Op, DAG);
8790 case ISD::VECTOR_INTERLEAVE:
8791 return lowerVECTOR_INTERLEAVE(Op, DAG);
8792 case ISD::STEP_VECTOR:
8793 return lowerSTEP_VECTOR(Op, DAG);
8794 case ISD::VECTOR_REVERSE:
8795 return lowerVECTOR_REVERSE(Op, DAG);
8796 case ISD::VECTOR_SPLICE_LEFT:
8797 case ISD::VECTOR_SPLICE_RIGHT:
8798 return lowerVECTOR_SPLICE(Op, DAG);
8799 case ISD::BUILD_VECTOR: {
8800 MVT VT = Op.getSimpleValueType();
8801 MVT EltVT = VT.getVectorElementType();
8802 if (!Subtarget.is64Bit() && EltVT == MVT::i64)
8803 return lowerBuildVectorViaVID(Op, DAG, Subtarget);
8804 return lowerBUILD_VECTOR(Op, DAG, Subtarget);
8805 }
8806 case ISD::SPLAT_VECTOR: {
8807 MVT VT = Op.getSimpleValueType();
8808 MVT EltVT = VT.getVectorElementType();
8809 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
8810 EltVT == MVT::bf16) {
8811 SDLoc DL(Op);
8812 SDValue Elt;
8813 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
8814 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
8815 Elt = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: Subtarget.getXLenVT(),
8816 Operand: Op.getOperand(i: 0));
8817 else
8818 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op.getOperand(i: 0));
8819 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
8820 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
8821 Operand: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: IVT, Operand: Elt));
8822 }
8823
8824 if (EltVT == MVT::i1)
8825 return lowerVectorMaskSplat(Op, DAG);
8826 return SDValue();
8827 }
8828 case ISD::VECTOR_SHUFFLE:
8829 return lowerVECTOR_SHUFFLE(Op, DAG);
8830 case ISD::CONCAT_VECTORS: {
8831 // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
8832 // better than going through the stack, as the default expansion does.
8833 SDLoc DL(Op);
8834 MVT VT = Op.getSimpleValueType();
8835 MVT ContainerVT = VT;
8836 if (VT.isFixedLengthVector())
8837 ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
8838
8839 // Recursively split concat_vectors with more than 2 operands:
8840 //
8841 // concat_vector op1, op2, op3, op4
8842 // ->
8843 // concat_vector (concat_vector op1, op2), (concat_vector op3, op4)
8844 //
8845 // This reduces the length of the chain of vslideups and allows us to
8846 // perform the vslideups at a smaller LMUL, limited to MF2.
8847 if (Op.getNumOperands() > 2 &&
8848 ContainerVT.bitsGE(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT))) {
8849 MVT HalfVT = VT.getHalfNumVectorElementsVT();
8850 assert(isPowerOf2_32(Op.getNumOperands()));
8851 size_t HalfNumOps = Op.getNumOperands() / 2;
8852 SDValue Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
8853 Ops: Op->ops().take_front(N: HalfNumOps));
8854 SDValue Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
8855 Ops: Op->ops().drop_front(N: HalfNumOps));
8856 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
8857 }
8858
8859 unsigned NumOpElts =
8860 Op.getOperand(i: 0).getSimpleValueType().getVectorMinNumElements();
8861 SDValue Vec = DAG.getUNDEF(VT);
8862 for (const auto &OpIdx : enumerate(First: Op->ops())) {
8863 SDValue SubVec = OpIdx.value();
8864 // Don't insert undef subvectors.
8865 if (SubVec.isUndef())
8866 continue;
8867 Vec = DAG.getInsertSubvector(DL, Vec, SubVec, Idx: OpIdx.index() * NumOpElts);
8868 }
8869 return Vec;
8870 }
8871 case ISD::LOAD: {
8872 auto *Load = cast<LoadSDNode>(Val&: Op);
8873 EVT VT = Load->getValueType(ResNo: 0);
8874 if (VT == MVT::f64) {
8875 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
8876 !Subtarget.is64Bit() && "Unexpected custom legalisation");
8877
8878 // Replace a double precision load with two i32 loads and a BuildPairF64.
8879 SDLoc DL(Op);
8880 SDValue BasePtr = Load->getBasePtr();
8881 SDValue Chain = Load->getChain();
8882
8883 SDValue Lo =
8884 DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo(),
8885 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
8886 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: 4));
8887 SDValue Hi = DAG.getLoad(
8888 VT: MVT::i32, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo().getWithOffset(O: 4),
8889 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
8890 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
8891 N2: Hi.getValue(R: 1));
8892
8893 // For big-endian, swap the order of Lo and Hi.
8894 if (!Subtarget.isLittleEndian())
8895 std::swap(a&: Lo, b&: Hi);
8896
8897 SDValue Pair = DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
8898 return DAG.getMergeValues(Ops: {Pair, Chain}, dl: DL);
8899 }
8900
8901 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
8902 (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
8903 assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
8904
8905 // Determine the half-size type
8906 MVT HalfVT;
8907 if (VT == MVT::v2i32)
8908 HalfVT = MVT::i32;
8909 else if (VT == MVT::v4i16)
8910 HalfVT = MVT::v2i16;
8911 else // VT == MVT::v8i8
8912 HalfVT = MVT::v4i8;
8913
8914 SDLoc DL(Op);
8915 SDValue BasePtr = Load->getBasePtr();
8916 SDValue Chain = Load->getChain();
8917
8918 // Create two loads for the lower and upper halves
8919 SDValue Lo =
8920 DAG.getLoad(VT: HalfVT, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo(),
8921 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
8922 unsigned HalfSize = HalfVT.getStoreSize();
8923 BasePtr =
8924 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: HalfSize));
8925 SDValue Hi =
8926 DAG.getLoad(VT: HalfVT, dl: DL, Chain, Ptr: BasePtr,
8927 PtrInfo: Load->getPointerInfo().getWithOffset(O: HalfSize),
8928 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
8929 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
8930 N2: Hi.getValue(R: 1));
8931
8932 // Combine the two halves into the result vector
8933 SDValue Result;
8934 if (VT == MVT::v2i32) {
8935 // For v2i32, build vector from two i32 scalars
8936 Result = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT, N1: Lo, N2: Hi);
8937 } else {
8938 // For v4i16 and v8i8, use CONCAT_VECTORS
8939 Result = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
8940 }
8941
8942 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
8943 }
8944
8945 if (VT == MVT::bf16)
8946 return lowerXAndesBfHCvtBFloat16Load(Op, DAG);
8947
8948 // Handle normal vector tuple load.
8949 if (VT.isRISCVVectorTuple()) {
8950 SDLoc DL(Op);
8951 MVT XLenVT = Subtarget.getXLenVT();
8952 unsigned NF = VT.getRISCVVectorTupleNumFields();
8953 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
8954 unsigned NumElts = Sz / (NF * 8);
8955 int Log2LMUL = Log2_64(Value: NumElts) - 3;
8956
8957 auto Flag = SDNodeFlags();
8958 Flag.setNoUnsignedWrap(true);
8959 SDValue Ret = DAG.getUNDEF(VT);
8960 SDValue BasePtr = Load->getBasePtr();
8961 SDValue VROffset = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
8962 VROffset =
8963 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VROffset,
8964 N2: DAG.getConstant(Val: std::max(a: Log2LMUL, b: 0), DL, VT: XLenVT));
8965 SmallVector<SDValue, 8> OutChains;
8966
8967 // Load NF vector registers and combine them to a vector tuple.
8968 for (unsigned i = 0; i < NF; ++i) {
8969 SDValue LoadVal = DAG.getLoad(
8970 VT: MVT::getScalableVectorVT(VT: MVT::i8, NumElements: NumElts), dl: DL, Chain: Load->getChain(),
8971 Ptr: BasePtr, PtrInfo: MachinePointerInfo(Load->getAddressSpace()), Alignment: Align(8));
8972 OutChains.push_back(Elt: LoadVal.getValue(R: 1));
8973 Ret = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT, N1: Ret, N2: LoadVal,
8974 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
8975 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: BasePtr, N2: VROffset, Flags: Flag);
8976 }
8977 return DAG.getMergeValues(
8978 Ops: {Ret, DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains)}, dl: DL);
8979 }
8980
8981 if (auto V = expandUnalignedRVVLoad(Op, DAG))
8982 return V;
8983 if (Op.getValueType().isFixedLengthVector())
8984 return lowerFixedLengthVectorLoadToRVV(Op, DAG);
8985 return Op;
8986 }
8987 case ISD::STORE: {
8988 auto *Store = cast<StoreSDNode>(Val&: Op);
8989 SDValue StoredVal = Store->getValue();
8990 EVT VT = StoredVal.getValueType();
8991
8992 if (VT == MVT::f64) {
8993 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
8994 !Subtarget.is64Bit() && "Unexpected custom legalisation");
8995
8996 // Replace a double precision store with a SplitF64 and i32 stores.
8997 SDValue DL(Op);
8998 SDValue BasePtr = Store->getBasePtr();
8999 SDValue Chain = Store->getChain();
9000 SDValue Split = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
9001 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: StoredVal);
9002
9003 SDValue Lo = Split.getValue(R: 0);
9004 SDValue Hi = Split.getValue(R: 1);
9005
9006 // For big-endian, swap the order of Lo and Hi before storing.
9007 if (!Subtarget.isLittleEndian())
9008 std::swap(a&: Lo, b&: Hi);
9009
9010 SDValue LoStore = DAG.getStore(
9011 Chain, dl: DL, Val: Lo, Ptr: BasePtr, PtrInfo: Store->getPointerInfo(),
9012 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9013 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: 4));
9014 SDValue HiStore = DAG.getStore(
9015 Chain, dl: DL, Val: Hi, Ptr: BasePtr, PtrInfo: Store->getPointerInfo().getWithOffset(O: 4),
9016 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9017 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: LoStore, N2: HiStore);
9018 }
9019 if (VT == MVT::i64) {
9020 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
9021 "Unexpected custom legalisation");
9022 if (Store->isTruncatingStore())
9023 return SDValue();
9024
9025 if (Store->getAlign() < Subtarget.getZilsdAlign())
9026 return SDValue();
9027
9028 SDLoc DL(Op);
9029 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i32, N1: StoredVal,
9030 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
9031 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i32, N1: StoredVal,
9032 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
9033
9034 return DAG.getMemIntrinsicNode(
9035 Opcode: RISCVISD::SD_RV32, dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
9036 Ops: {Store->getChain(), Lo, Hi, Store->getBasePtr()}, MemVT: MVT::i64,
9037 MMO: Store->getMemOperand());
9038 }
9039
9040 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9041 (VT == MVT::v2i32 || VT == MVT::v4i16 || VT == MVT::v8i8)) {
9042 assert(!Subtarget.is64Bit() && "Unexpected custom legalisation");
9043
9044 auto *Store = cast<StoreSDNode>(Val&: Op);
9045 SDValue Val = Store->getValue();
9046
9047 // Determine the half-size type
9048 MVT HalfVT;
9049 if (VT == MVT::v2i32)
9050 HalfVT = MVT::i32;
9051 else if (VT == MVT::v4i16)
9052 HalfVT = MVT::v2i16;
9053 else // VT == MVT::v8i8
9054 HalfVT = MVT::v4i8;
9055
9056 SDLoc DL(Op);
9057 SDValue BasePtr = Store->getBasePtr();
9058 SDValue Chain = Store->getChain();
9059
9060 // Extract the two halves from the vector
9061 SDValue Lo, Hi;
9062 if (VT == MVT::v2i32) {
9063 // For v2i32, extract two i32 scalars
9064 Lo = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Val,
9065 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9066 Hi = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: Val,
9067 N2: DAG.getVectorIdxConstant(Val: 1, DL));
9068 } else {
9069 // For v4i16 and v8i8, extract two vector halves
9070 Lo = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HalfVT, N1: Val,
9071 N2: DAG.getVectorIdxConstant(Val: 0, DL));
9072 unsigned HalfNumElts = HalfVT.getVectorNumElements();
9073 Hi = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: HalfVT, N1: Val,
9074 N2: DAG.getVectorIdxConstant(Val: HalfNumElts, DL));
9075 }
9076
9077 // Create two stores for the lower and upper halves
9078 SDValue LoStore = DAG.getStore(
9079 Chain, dl: DL, Val: Lo, Ptr: BasePtr, PtrInfo: Store->getPointerInfo(),
9080 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9081 unsigned HalfSize = HalfVT.getStoreSize();
9082 BasePtr =
9083 DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: HalfSize));
9084 SDValue HiStore = DAG.getStore(
9085 Chain, dl: DL, Val: Hi, Ptr: BasePtr,
9086 PtrInfo: Store->getPointerInfo().getWithOffset(O: HalfSize),
9087 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
9088
9089 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: LoStore, N2: HiStore);
9090 }
9091
9092 if (VT == MVT::bf16)
9093 return lowerXAndesBfHCvtBFloat16Store(Op, DAG);
9094
9095 // Handle normal vector tuple store.
9096 if (VT.isRISCVVectorTuple()) {
9097 SDLoc DL(Op);
9098 MVT XLenVT = Subtarget.getXLenVT();
9099 unsigned NF = VT.getRISCVVectorTupleNumFields();
9100 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
9101 unsigned NumElts = Sz / (NF * 8);
9102 int Log2LMUL = Log2_64(Value: NumElts) - 3;
9103
9104 auto Flag = SDNodeFlags();
9105 Flag.setNoUnsignedWrap(true);
9106 SDValue Ret;
9107 SDValue Chain = Store->getChain();
9108 SDValue BasePtr = Store->getBasePtr();
9109 SDValue VROffset = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
9110 VROffset =
9111 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VROffset,
9112 N2: DAG.getConstant(Val: std::max(a: Log2LMUL, b: 0), DL, VT: XLenVT));
9113
9114 // Extract subregisters in a vector tuple and store them individually.
9115 for (unsigned i = 0; i < NF; ++i) {
9116 auto Extract =
9117 DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL,
9118 VT: MVT::getScalableVectorVT(VT: MVT::i8, NumElements: NumElts), N1: StoredVal,
9119 N2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
9120 Ret = DAG.getStore(Chain, dl: DL, Val: Extract, Ptr: BasePtr,
9121 PtrInfo: MachinePointerInfo(Store->getAddressSpace()),
9122 Alignment: Store->getBaseAlign(),
9123 MMOFlags: Store->getMemOperand()->getFlags());
9124 Chain = Ret.getValue(R: 0);
9125 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: BasePtr, N2: VROffset, Flags: Flag);
9126 }
9127 return Ret;
9128 }
9129
9130 if (auto V = expandUnalignedRVVStore(Op, DAG))
9131 return V;
9132 if (Op.getOperand(i: 1).getValueType().isFixedLengthVector())
9133 return lowerFixedLengthVectorStoreToRVV(Op, DAG);
9134 return Op;
9135 }
9136 case ISD::VP_LOAD:
9137 if (SDValue V = expandUnalignedVPLoad(Op, DAG))
9138 return V;
9139 [[fallthrough]];
9140 case ISD::MLOAD:
9141 return lowerMaskedLoad(Op, DAG);
9142 case ISD::VP_LOAD_FF:
9143 return lowerLoadFF(Op, DAG);
9144 case ISD::VP_STORE:
9145 if (SDValue V = expandUnalignedVPStore(Op, DAG))
9146 return V;
9147 [[fallthrough]];
9148 case ISD::MSTORE:
9149 return lowerMaskedStore(Op, DAG);
9150 case ISD::VECTOR_COMPRESS:
9151 return lowerVectorCompress(Op, DAG);
9152 case ISD::SELECT_CC: {
9153 // This occurs because we custom legalize SETGT and SETUGT for setcc. That
9154 // causes LegalizeDAG to think we need to custom legalize select_cc. Expand
9155 // into separate SETCC+SELECT just like LegalizeDAG.
9156 SDValue Tmp1 = Op.getOperand(i: 0);
9157 SDValue Tmp2 = Op.getOperand(i: 1);
9158 SDValue True = Op.getOperand(i: 2);
9159 SDValue False = Op.getOperand(i: 3);
9160 EVT VT = Op.getValueType();
9161 SDValue CC = Op.getOperand(i: 4);
9162 EVT CmpVT = Tmp1.getValueType();
9163 EVT CCVT =
9164 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: CmpVT);
9165 SDLoc DL(Op);
9166 SDValue Cond =
9167 DAG.getNode(Opcode: ISD::SETCC, DL, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Op->getFlags());
9168 return DAG.getSelect(DL, VT, Cond, LHS: True, RHS: False);
9169 }
9170 case ISD::SETCC: {
9171 MVT OpVT = Op.getOperand(i: 0).getSimpleValueType();
9172 if (OpVT.isScalarInteger()) {
9173 MVT VT = Op.getSimpleValueType();
9174 SDValue LHS = Op.getOperand(i: 0);
9175 SDValue RHS = Op.getOperand(i: 1);
9176 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9177 assert((CCVal == ISD::SETGT || CCVal == ISD::SETUGT) &&
9178 "Unexpected CondCode");
9179
9180 SDLoc DL(Op);
9181
9182 // If the RHS is a constant in the range [-2049, 0) or (0, 2046], we can
9183 // convert this to the equivalent of (set(u)ge X, C+1) by using
9184 // (xori (slti(u) X, C+1), 1). This avoids materializing a small constant
9185 // in a register.
9186 if (isa<ConstantSDNode>(Val: RHS)) {
9187 int64_t Imm = cast<ConstantSDNode>(Val&: RHS)->getSExtValue();
9188 if (Imm != 0 && isInt<12>(x: (uint64_t)Imm + 1)) {
9189 // If this is an unsigned compare and the constant is -1, incrementing
9190 // the constant would change behavior. The result should be false.
9191 if (CCVal == ISD::SETUGT && Imm == -1)
9192 return DAG.getConstant(Val: 0, DL, VT);
9193 // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
9194 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
9195 SDValue SetCC = DAG.getSetCC(
9196 DL, VT, LHS, RHS: DAG.getSignedConstant(Val: Imm + 1, DL, VT: OpVT), Cond: CCVal);
9197 return DAG.getLogicalNOT(DL, Val: SetCC, VT);
9198 }
9199 // Lower (setugt X, 2047) as (setne (srl X, 11), 0).
9200 if (CCVal == ISD::SETUGT && Imm == 2047) {
9201 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: OpVT, N1: LHS,
9202 N2: DAG.getShiftAmountConstant(Val: 11, VT: OpVT, DL));
9203 return DAG.getSetCC(DL, VT, LHS: Shift, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT),
9204 Cond: ISD::SETNE);
9205 }
9206 }
9207
9208 // Not a constant we could handle, swap the operands and condition code to
9209 // SETLT/SETULT.
9210 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
9211 return DAG.getSetCC(DL, VT, LHS: RHS, RHS: LHS, Cond: CCVal);
9212 }
9213
9214 MVT VT = Op.getSimpleValueType();
9215 if (Subtarget.hasStdExtP() && VT.isFixedLengthVector()) {
9216 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
9217 SDValue LHS = Op.getOperand(i: 0);
9218 SDValue RHS = Op.getOperand(i: 1);
9219 SDLoc DL(Op);
9220 if (CCVal == ISD::SETNE) {
9221 // Convert setne X, 0 to setult 0, X.
9222 if (ISD::isConstantSplatVectorAllZeros(N: RHS.getNode())) {
9223 return DAG.getSetCC(DL, VT, LHS: RHS, RHS: LHS, Cond: ISD::SETULT);
9224 }
9225
9226 // Not a constant we could handle, convert to SETEQ+Invert
9227 SDValue SetCC = DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::SETEQ);
9228 return DAG.getLogicalNOT(DL, Val: SetCC, VT);
9229 }
9230
9231 if (CCVal == ISD::SETGT) {
9232 if (ISD::isConstantSplatVectorAllOnes(N: RHS.getNode())) {
9233 SDValue SetCC =
9234 DAG.getSetCC(DL, VT, LHS, RHS: DAG.getConstant(Val: 0, DL, VT), Cond: ISD::SETLT);
9235 return DAG.getLogicalNOT(DL, Val: SetCC, VT);
9236 }
9237
9238 // Not a constant we could handle, swap the operands and condition code
9239 // to SETLT.
9240 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
9241 return DAG.getSetCC(DL, VT, LHS: RHS, RHS: LHS, Cond: CCVal);
9242 }
9243
9244 return SDValue();
9245 }
9246
9247 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 0), Subtarget, TLI: *this))
9248 return SplitVectorOp(Op, DAG);
9249
9250 return lowerToScalableOp(Op, DAG);
9251 }
9252 case ISD::ADD:
9253 case ISD::SUB:
9254 case ISD::SDIV:
9255 case ISD::SREM:
9256 case ISD::UDIV:
9257 case ISD::UREM:
9258 case ISD::BSWAP:
9259 case ISD::CTPOP:
9260 return lowerToScalableOp(Op, DAG);
9261 case ISD::VSELECT: {
9262 EVT VT = Op.getValueType();
9263 // Split 64-bit vector VSELECT on RV32 with P extension for v4i16 and v8i8
9264 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9265 (VT == MVT::v4i16 || VT == MVT::v8i8)) {
9266 SDLoc DL(Op);
9267 SDValue Mask = Op.getOperand(i: 0);
9268 SDValue TrueVal = Op.getOperand(i: 1);
9269 SDValue FalseVal = Op.getOperand(i: 2);
9270
9271 // Split all three operands into two halves
9272 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Mask, DL);
9273 auto [TrueLo, TrueHi] = DAG.SplitVector(N: TrueVal, DL);
9274 auto [FalseLo, FalseHi] = DAG.SplitVector(N: FalseVal, DL);
9275
9276 // Perform VSELECT on each half
9277 SDValue ResLo = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: TrueLo.getValueType(),
9278 N1: MaskLo, N2: TrueLo, N3: FalseLo);
9279 SDValue ResHi = DAG.getNode(Opcode: ISD::VSELECT, DL, VT: TrueHi.getValueType(),
9280 N1: MaskHi, N2: TrueHi, N3: FalseHi);
9281
9282 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: ResLo, N2: ResHi);
9283 }
9284 return lowerToScalableOp(Op, DAG);
9285 }
9286 case ISD::AND:
9287 case ISD::OR:
9288 case ISD::XOR:
9289 case ISD::MUL:
9290 case ISD::MULHS:
9291 case ISD::MULHU: {
9292 EVT VT = Op.getValueType();
9293 unsigned Opc = Op.getOpcode();
9294 // Split 64-bit vector AND/OR/XOR/MUL/MULHS/MULHU on RV32 with P extension
9295 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9296 (VT == MVT::v4i16 || VT == MVT::v8i8)) {
9297 SDLoc DL(Op);
9298 SDValue LHS = Op.getOperand(i: 0);
9299 SDValue RHS = Op.getOperand(i: 1);
9300
9301 // Determine the half-size type
9302 MVT HalfVT = (VT == MVT::v4i16) ? MVT::v2i16 : MVT::v4i8;
9303
9304 // Extract the two halves from LHS
9305 auto [LHSLo, LHSHi] = DAG.SplitVector(N: LHS, DL, LoVT: HalfVT, HiVT: HalfVT);
9306
9307 // Extract the two halves from RHS
9308 auto [RHSLo, RHSHi] = DAG.SplitVector(N: RHS, DL, LoVT: HalfVT, HiVT: HalfVT);
9309
9310 // Perform the operation on each half
9311 SDValue ResLo = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: LHSLo, N2: RHSLo);
9312 SDValue ResHi = DAG.getNode(Opcode: Opc, DL, VT: HalfVT, N1: LHSHi, N2: RHSHi);
9313
9314 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: ResLo, N2: ResHi);
9315 }
9316 // Lower v4i8/v2i16 MUL/MULHS/MULHU via widening multiply + srl + truncate.
9317 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() &&
9318 (Opc == ISD::MUL || Opc == ISD::MULHS || Opc == ISD::MULHU) &&
9319 (VT == MVT::v4i8 || VT == MVT::v2i16)) {
9320 assert((VT == MVT::v4i8 || Opc == ISD::MUL) &&
9321 "Unexpected custom legalisation");
9322 SDLoc DL(Op);
9323 MVT WideVT = (VT == MVT::v4i8) ? MVT::v4i16 : MVT::v2i32;
9324 unsigned WMulOpc =
9325 (Opc == ISD::MULHU) ? RISCVISD::PWMULU : RISCVISD::PWMUL;
9326 SDValue Res =
9327 DAG.getNode(Opcode: WMulOpc, DL, VT: WideVT, N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
9328 if (Opc != ISD::MUL) {
9329 unsigned EltBits = VT.getVectorElementType().getSizeInBits();
9330 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: WideVT, N1: Res,
9331 N2: DAG.getConstant(Val: EltBits, DL, VT: WideVT));
9332 }
9333 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
9334 }
9335 return lowerToScalableOp(Op, DAG);
9336 }
9337 case ISD::ANY_EXTEND_VECTOR_INREG: {
9338 EVT VT = Op.getValueType();
9339 assert(Subtarget.hasStdExtP() && Subtarget.is64Bit() &&
9340 (VT == MVT::v2i32 || VT == MVT::v4i16) &&
9341 "Unexpected custom legalisation");
9342 return DAG.getNode(Opcode: ISD::ZERO_EXTEND_VECTOR_INREG, DL: SDLoc(Op), VT,
9343 Operand: Op.getOperand(i: 0));
9344 }
9345 case ISD::SHL:
9346 case ISD::SRL:
9347 case ISD::SRA:
9348 case ISD::SSHLSAT:
9349 if (Op.getSimpleValueType().isFixedLengthVector()) {
9350 if (Subtarget.hasStdExtP()) {
9351 SDValue ShAmtVec = Op.getOperand(i: 1);
9352 SDValue SplatVal;
9353 if (ShAmtVec.getOpcode() == ISD::SPLAT_VECTOR)
9354 SplatVal = ShAmtVec.getOperand(i: 0);
9355 else if (ShAmtVec.getOpcode() == ISD::BUILD_VECTOR)
9356 SplatVal = cast<BuildVectorSDNode>(Val&: ShAmtVec)->getSplatValue();
9357
9358 if (!SplatVal)
9359 return SDValue();
9360
9361 unsigned Opc;
9362 switch (Op.getOpcode()) {
9363 default:
9364 llvm_unreachable("Unexpected opcode");
9365 case ISD::SHL:
9366 Opc = RISCVISD::PSHL;
9367 break;
9368 case ISD::SRL:
9369 Opc = RISCVISD::PSRL;
9370 break;
9371 case ISD::SRA:
9372 Opc = RISCVISD::PSRA;
9373 break;
9374 case ISD::SSHLSAT:
9375 Opc = RISCVISD::PSSHA;
9376 break;
9377 }
9378 return DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: Op.getValueType(), N1: Op.getOperand(i: 0),
9379 N2: SplatVal);
9380 }
9381 return lowerToScalableOp(Op, DAG);
9382 }
9383 assert(Op.getOpcode() != ISD::SSHLSAT);
9384 // This can be called for an i32 shift amount that needs to be promoted.
9385 assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
9386 "Unexpected custom legalisation");
9387 return SDValue();
9388 case ISD::MASKED_UDIV:
9389 case ISD::MASKED_SDIV:
9390 case ISD::MASKED_UREM:
9391 case ISD::MASKED_SREM: {
9392 SDLoc DL(Op);
9393 MVT VT = Op.getSimpleValueType();
9394 MVT ContainerVT = getContainerForFixedLengthVector(VT);
9395 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
9396 SDValue Res = DAG.getNode(
9397 Opcode: getRISCVVLOp(Op), DL, VT: ContainerVT,
9398 N1: convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i: 0), DAG, Subtarget),
9399 N2: convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i: 1), DAG, Subtarget),
9400 N3: DAG.getUNDEF(VT: ContainerVT),
9401 N4: convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Op.getOperand(i: 2),
9402 DAG, Subtarget),
9403 N5: VL);
9404 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
9405 }
9406 case ISD::FABS:
9407 case ISD::FNEG:
9408 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
9409 return lowerFABSorFNEG(Op, DAG, Subtarget);
9410 [[fallthrough]];
9411 case ISD::FADD:
9412 case ISD::FSUB:
9413 case ISD::FMUL:
9414 case ISD::FDIV:
9415 case ISD::FSQRT:
9416 case ISD::FMA:
9417 case ISD::FMINNUM:
9418 case ISD::FMAXNUM:
9419 case ISD::FMINIMUMNUM:
9420 case ISD::FMAXIMUMNUM:
9421 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
9422 return SplitVectorOp(Op, DAG);
9423 [[fallthrough]];
9424 case ISD::AVGFLOORS:
9425 case ISD::AVGFLOORU:
9426 case ISD::AVGCEILS:
9427 case ISD::AVGCEILU:
9428 case ISD::SMIN:
9429 case ISD::SMAX:
9430 case ISD::UMIN:
9431 case ISD::UMAX:
9432 case ISD::UADDSAT:
9433 case ISD::USUBSAT:
9434 case ISD::SADDSAT:
9435 case ISD::SSUBSAT:
9436 return lowerToScalableOp(Op, DAG);
9437 case ISD::ABDS:
9438 case ISD::ABDU: {
9439 EVT VT = Op->getValueType(ResNo: 0);
9440 // Only SEW=8/16 are supported in Zvabd.
9441 if (Subtarget.hasStdExtZvabd() && VT.isVector() &&
9442 (VT.getVectorElementType() == MVT::i8 ||
9443 VT.getVectorElementType() == MVT::i16))
9444 return lowerToScalableOp(Op, DAG);
9445
9446 SDLoc dl(Op);
9447 SDValue LHS = DAG.getFreeze(V: Op->getOperand(Num: 0));
9448 SDValue RHS = DAG.getFreeze(V: Op->getOperand(Num: 1));
9449 bool IsSigned = Op->getOpcode() == ISD::ABDS;
9450
9451 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
9452 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
9453 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
9454 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
9455 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
9456 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
9457 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
9458 }
9459 case ISD::ABS:
9460 case ISD::ABS_MIN_POISON:
9461 return lowerABS(Op, DAG);
9462 case ISD::CTLZ:
9463 case ISD::CTLZ_ZERO_POISON:
9464 case ISD::CTTZ:
9465 case ISD::CTTZ_ZERO_POISON:
9466 if (Subtarget.hasStdExtZvbb())
9467 return lowerToScalableOp(Op, DAG);
9468 assert(Op.getOpcode() != ISD::CTTZ);
9469 return lowerCTLZ_CTTZ_ZERO_POISON(Op, DAG);
9470 case ISD::CLMUL: {
9471 MVT VT = Op.getSimpleValueType();
9472 assert(VT.isScalableVector() && Subtarget.hasStdExtZvbc() &&
9473 "Unexpected custom legalisation");
9474 // Promote to i64 vector.
9475 MVT I64VecVT = VT.changeVectorElementType(EltVT: MVT::i64);
9476 SDLoc DL(Op);
9477 SDValue Op0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: I64VecVT, Operand: Op.getOperand(i: 0));
9478 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: I64VecVT, Operand: Op.getOperand(i: 1));
9479 SDValue CLMUL = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: I64VecVT, N1: Op0, N2: Op1);
9480 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CLMUL);
9481 }
9482 case ISD::FCOPYSIGN:
9483 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
9484 return lowerFCOPYSIGN(Op, DAG, Subtarget);
9485 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
9486 return SplitVectorOp(Op, DAG);
9487 return lowerToScalableOp(Op, DAG);
9488 case ISD::STRICT_FADD:
9489 case ISD::STRICT_FSUB:
9490 case ISD::STRICT_FMUL:
9491 case ISD::STRICT_FDIV:
9492 case ISD::STRICT_FSQRT:
9493 case ISD::STRICT_FMA:
9494 if (isPromotedOpNeedingSplit(Op, Subtarget, TLI: *this))
9495 return SplitStrictFPVectorOp(Op, DAG);
9496 return lowerToScalableOp(Op, DAG);
9497 case ISD::STRICT_FSETCC:
9498 case ISD::STRICT_FSETCCS:
9499 return lowerVectorStrictFSetcc(Op, DAG);
9500 case ISD::STRICT_FCEIL:
9501 case ISD::STRICT_FRINT:
9502 case ISD::STRICT_FFLOOR:
9503 case ISD::STRICT_FTRUNC:
9504 case ISD::STRICT_FNEARBYINT:
9505 case ISD::STRICT_FROUND:
9506 case ISD::STRICT_FROUNDEVEN:
9507 return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
9508 case ISD::MGATHER:
9509 case ISD::VP_GATHER:
9510 return lowerMaskedGather(Op, DAG);
9511 case ISD::MSCATTER:
9512 case ISD::VP_SCATTER:
9513 return lowerMaskedScatter(Op, DAG);
9514 case ISD::GET_ROUNDING:
9515 return lowerGET_ROUNDING(Op, DAG);
9516 case ISD::SET_ROUNDING:
9517 return lowerSET_ROUNDING(Op, DAG);
9518 case ISD::GET_FPENV:
9519 return lowerGET_FPENV(Op, DAG);
9520 case ISD::SET_FPENV:
9521 return lowerSET_FPENV(Op, DAG);
9522 case ISD::RESET_FPENV:
9523 return lowerRESET_FPENV(Op, DAG);
9524 case ISD::GET_FPMODE:
9525 return lowerGET_FPMODE(Op, DAG);
9526 case ISD::SET_FPMODE:
9527 return lowerSET_FPMODE(Op, DAG);
9528 case ISD::RESET_FPMODE:
9529 return lowerRESET_FPMODE(Op, DAG);
9530 case ISD::EH_DWARF_CFA:
9531 return lowerEH_DWARF_CFA(Op, DAG);
9532 case ISD::VP_MERGE:
9533 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
9534 return lowerVPMergeMask(Op, DAG);
9535 [[fallthrough]];
9536 case ISD::VP_SDIV:
9537 case ISD::VP_UDIV:
9538 case ISD::VP_SREM:
9539 case ISD::VP_UREM:
9540 return lowerVPOp(Op, DAG);
9541 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9542 return lowerVPStridedLoad(Op, DAG);
9543 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9544 return lowerVPStridedStore(Op, DAG);
9545 case ISD::EXPERIMENTAL_VP_SPLICE:
9546 return lowerVPSpliceExperimental(Op, DAG);
9547 case ISD::EXPERIMENTAL_VP_REVERSE:
9548 return lowerVPReverseExperimental(Op, DAG);
9549 case ISD::CLEAR_CACHE: {
9550 assert(getTargetMachine().getTargetTriple().isOSLinux() &&
9551 "llvm.clear_cache only needs custom lower on Linux targets");
9552 SDLoc DL(Op);
9553 SDValue Flags = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
9554 return emitFlushICache(DAG, InChain: Op.getOperand(i: 0), Start: Op.getOperand(i: 1),
9555 End: Op.getOperand(i: 2), Flags, DL);
9556 }
9557 case ISD::DYNAMIC_STACKALLOC:
9558 return lowerDYNAMIC_STACKALLOC(Op, DAG);
9559 case ISD::INIT_TRAMPOLINE:
9560 return lowerINIT_TRAMPOLINE(Op, DAG);
9561 case ISD::ADJUST_TRAMPOLINE:
9562 return lowerADJUST_TRAMPOLINE(Op, DAG);
9563 case ISD::PARTIAL_REDUCE_UMLA:
9564 case ISD::PARTIAL_REDUCE_SMLA:
9565 case ISD::PARTIAL_REDUCE_SUMLA:
9566 return lowerPARTIAL_REDUCE_MLA(Op, DAG);
9567 case ISD::CTTZ_ELTS:
9568 case ISD::CTTZ_ELTS_ZERO_POISON:
9569 return lowerCttzElts(Op, DAG, Subtarget);
9570 }
9571}
9572
9573SDValue RISCVTargetLowering::emitFlushICache(SelectionDAG &DAG, SDValue InChain,
9574 SDValue Start, SDValue End,
9575 SDValue Flags, SDLoc DL) const {
9576 MakeLibCallOptions CallOptions;
9577 std::pair<SDValue, SDValue> CallResult =
9578 makeLibCall(DAG, LC: RTLIB::RISCV_FLUSH_ICACHE, RetVT: MVT::isVoid,
9579 Ops: {Start, End, Flags}, CallOptions, dl: DL, Chain: InChain);
9580
9581 // This function returns void so only the out chain matters.
9582 return CallResult.second;
9583}
9584
9585SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
9586 SelectionDAG &DAG) const {
9587 if (!Subtarget.is64Bit())
9588 llvm::reportFatalUsageError(reason: "Trampolines only implemented for RV64");
9589
9590 // Create an MCCodeEmitter to encode instructions.
9591 TargetLoweringObjectFile *TLO = getTargetMachine().getObjFileLowering();
9592 assert(TLO);
9593 MCContext &MCCtx = TLO->getContext();
9594
9595 std::unique_ptr<MCCodeEmitter> CodeEmitter(
9596 createRISCVMCCodeEmitter(MCII: *getTargetMachine().getMCInstrInfo(), Ctx&: MCCtx));
9597
9598 SDValue Root = Op.getOperand(i: 0);
9599 SDValue Trmp = Op.getOperand(i: 1); // trampoline
9600 SDLoc dl(Op);
9601
9602 const Value *TrmpAddr = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
9603
9604 // We store in the trampoline buffer the following instructions and data.
9605 // Offset:
9606 // 0: auipc t2, 0
9607 // 4: ld t0, 24(t2)
9608 // 8: ld t2, 16(t2)
9609 // 12: jalr t0
9610 // 16: <StaticChainOffset>
9611 // 24: <FunctionAddressOffset>
9612 // 32:
9613 // Offset with branch control flow protection enabled:
9614 // 0: lpad <imm20>
9615 // 4: auipc t3, 0
9616 // 8: ld t2, 28(t3)
9617 // 12: ld t3, 20(t3)
9618 // 16: jalr t2
9619 // 20: <StaticChainOffset>
9620 // 28: <FunctionAddressOffset>
9621 // 36:
9622
9623 const MachineFunction &MF = DAG.getMachineFunction();
9624 const bool HasCFBranch =
9625 MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
9626 const unsigned StaticChainIdx = HasCFBranch ? 5 : 4;
9627 const unsigned StaticChainOffset = StaticChainIdx * 4;
9628 const unsigned FunctionAddressOffset = StaticChainOffset + 8;
9629
9630 const MCSubtargetInfo &STI = getTargetMachine().getMCSubtargetInfo();
9631 auto GetEncoding = [&](const MCInst &MC) {
9632 SmallVector<char, 4> CB;
9633 SmallVector<MCFixup> Fixups;
9634 CodeEmitter->encodeInstruction(Inst: MC, CB, Fixups, STI);
9635 uint32_t Encoding = support::endian::read32le(P: CB.data());
9636 return Encoding;
9637 };
9638
9639 SmallVector<SDValue> OutChains;
9640
9641 SmallVector<uint32_t> Encodings;
9642 if (!HasCFBranch) {
9643 Encodings.append(
9644 IL: {// auipc t2, 0
9645 // Loads the current PC into t2.
9646 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X7).addImm(Val: 0)),
9647 // ld t0, 24(t2)
9648 // Loads the function address into t0. Note that we are using offsets
9649 // pc-relative to the first instruction of the trampoline.
9650 GetEncoding(MCInstBuilder(RISCV::LD)
9651 .addReg(Reg: RISCV::X5)
9652 .addReg(Reg: RISCV::X7)
9653 .addImm(Val: FunctionAddressOffset)),
9654 // ld t2, 16(t2)
9655 // Load the value of the static chain.
9656 GetEncoding(MCInstBuilder(RISCV::LD)
9657 .addReg(Reg: RISCV::X7)
9658 .addReg(Reg: RISCV::X7)
9659 .addImm(Val: StaticChainOffset)),
9660 // jalr t0
9661 // Jump to the function.
9662 GetEncoding(MCInstBuilder(RISCV::JALR)
9663 .addReg(Reg: RISCV::X0)
9664 .addReg(Reg: RISCV::X5)
9665 .addImm(Val: 0))});
9666 } else {
9667 Encodings.append(
9668 IL: {// auipc x0, <imm20> (lpad <imm20>)
9669 // Landing pad.
9670 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X0).addImm(Val: 0)),
9671 // auipc t3, 0
9672 // Loads the current PC into t3.
9673 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X28).addImm(Val: 0)),
9674 // ld t2, (FunctionAddressOffset - 4)(t3)
9675 // Loads the function address into t2. Note that we are using offsets
9676 // pc-relative to the SECOND instruction of the trampoline.
9677 GetEncoding(MCInstBuilder(RISCV::LD)
9678 .addReg(Reg: RISCV::X7)
9679 .addReg(Reg: RISCV::X28)
9680 .addImm(Val: FunctionAddressOffset - 4)),
9681 // ld t3, (StaticChainOffset - 4)(t3)
9682 // Load the value of the static chain.
9683 GetEncoding(MCInstBuilder(RISCV::LD)
9684 .addReg(Reg: RISCV::X28)
9685 .addReg(Reg: RISCV::X28)
9686 .addImm(Val: StaticChainOffset - 4)),
9687 // jalr t2
9688 // Software-guarded jump to the function.
9689 GetEncoding(MCInstBuilder(RISCV::JALR)
9690 .addReg(Reg: RISCV::X0)
9691 .addReg(Reg: RISCV::X7)
9692 .addImm(Val: 0))});
9693 }
9694
9695 // Store encoded instructions.
9696 for (auto [Idx, Encoding] : llvm::enumerate(First&: Encodings)) {
9697 SDValue Addr = Idx > 0 ? DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Trmp,
9698 N2: DAG.getConstant(Val: Idx * 4, DL: dl, VT: MVT::i64))
9699 : Trmp;
9700 OutChains.push_back(Elt: DAG.getTruncStore(
9701 Chain: Root, dl, Val: DAG.getConstant(Val: Encoding, DL: dl, VT: MVT::i64), Ptr: Addr,
9702 PtrInfo: MachinePointerInfo(TrmpAddr, Idx * 4), SVT: MVT::i32));
9703 }
9704
9705 // Now store the variable part of the trampoline.
9706 SDValue FunctionAddress = Op.getOperand(i: 2);
9707 SDValue StaticChain = Op.getOperand(i: 3);
9708
9709 // Store the given static chain and function pointer in the trampoline buffer.
9710 struct OffsetValuePair {
9711 const unsigned Offset;
9712 const SDValue Value;
9713 SDValue Addr = SDValue(); // Used to cache the address.
9714 } OffsetValues[] = {
9715 {.Offset: StaticChainOffset, .Value: StaticChain},
9716 {.Offset: FunctionAddressOffset, .Value: FunctionAddress},
9717 };
9718 for (auto &OffsetValue : OffsetValues) {
9719 SDValue Addr =
9720 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Trmp,
9721 N2: DAG.getConstant(Val: OffsetValue.Offset, DL: dl, VT: MVT::i64));
9722 OffsetValue.Addr = Addr;
9723 OutChains.push_back(
9724 Elt: DAG.getStore(Chain: Root, dl, Val: OffsetValue.Value, Ptr: Addr,
9725 PtrInfo: MachinePointerInfo(TrmpAddr, OffsetValue.Offset)));
9726 }
9727
9728 assert(OutChains.size() == StaticChainIdx + 2 &&
9729 "Size of OutChains mismatch");
9730 SDValue StoreToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9731
9732 // The end of instructions of trampoline is the same as the static chain
9733 // address that we computed earlier.
9734 SDValue EndOfTrmp = OffsetValues[0].Addr;
9735
9736 // Call clear cache on the trampoline instructions.
9737 SDValue Chain = DAG.getNode(Opcode: ISD::CLEAR_CACHE, DL: dl, VT: MVT::Other, N1: StoreToken,
9738 N2: Trmp, N3: EndOfTrmp);
9739
9740 return Chain;
9741}
9742
9743SDValue RISCVTargetLowering::lowerADJUST_TRAMPOLINE(SDValue Op,
9744 SelectionDAG &DAG) const {
9745 if (!Subtarget.is64Bit())
9746 llvm::reportFatalUsageError(reason: "Trampolines only implemented for RV64");
9747
9748 return Op.getOperand(i: 0);
9749}
9750
9751SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
9752 SelectionDAG &DAG) const {
9753 // Currently, only the vdot4a and vdot4au case (from zvdot4a8i) should be
9754 // legal.
9755 // TODO: There are many other sub-cases we could potentially lower, are
9756 // any of them worthwhile? Ex: via vredsum, vwredsum, vwwmaccu, etc..
9757 SDLoc DL(Op);
9758 MVT VT = Op.getSimpleValueType();
9759 SDValue Accum = Op.getOperand(i: 0);
9760 assert(Accum.getSimpleValueType() == VT &&
9761 VT.getVectorElementType() == MVT::i32);
9762 SDValue A = Op.getOperand(i: 1);
9763 SDValue B = Op.getOperand(i: 2);
9764 MVT ArgVT = A.getSimpleValueType();
9765 assert(ArgVT == B.getSimpleValueType() &&
9766 ArgVT.getVectorElementType() == MVT::i8);
9767 (void)ArgVT;
9768
9769 // The zvdot4a8i pseudos are defined with sources and destination both
9770 // being i32. This cast is needed for correctness to avoid incorrect
9771 // .vx matching of i8 splats.
9772 A = DAG.getBitcast(VT, V: A);
9773 B = DAG.getBitcast(VT, V: B);
9774
9775 MVT ContainerVT = VT;
9776 if (VT.isFixedLengthVector()) {
9777 ContainerVT = getContainerForFixedLengthVector(VT);
9778 Accum = convertToScalableVector(VT: ContainerVT, V: Accum, DAG, Subtarget);
9779 A = convertToScalableVector(VT: ContainerVT, V: A, DAG, Subtarget);
9780 B = convertToScalableVector(VT: ContainerVT, V: B, DAG, Subtarget);
9781 }
9782
9783 unsigned Opc;
9784 switch (Op.getOpcode()) {
9785 case ISD::PARTIAL_REDUCE_SMLA:
9786 Opc = RISCVISD::VDOT4A_VL;
9787 break;
9788 case ISD::PARTIAL_REDUCE_UMLA:
9789 Opc = RISCVISD::VDOT4AU_VL;
9790 break;
9791 case ISD::PARTIAL_REDUCE_SUMLA:
9792 Opc = RISCVISD::VDOT4ASU_VL;
9793 break;
9794 default:
9795 llvm_unreachable("Unexpected opcode");
9796 }
9797 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
9798 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, Ops: {A, B, Accum, Mask, VL});
9799 if (VT.isFixedLengthVector())
9800 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
9801 return Res;
9802}
9803
9804static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty,
9805 SelectionDAG &DAG, unsigned Flags) {
9806 return DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: Flags);
9807}
9808
9809static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
9810 SelectionDAG &DAG, unsigned Flags) {
9811 return DAG.getTargetBlockAddress(BA: N->getBlockAddress(), VT: Ty, Offset: N->getOffset(),
9812 TargetFlags: Flags);
9813}
9814
9815static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
9816 SelectionDAG &DAG, unsigned Flags) {
9817 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
9818 Offset: N->getOffset(), TargetFlags: Flags);
9819}
9820
9821static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
9822 SelectionDAG &DAG, unsigned Flags) {
9823 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
9824}
9825
9826static SDValue getLargeGlobalAddress(GlobalAddressSDNode *N, const SDLoc &DL,
9827 EVT Ty, SelectionDAG &DAG) {
9828 RISCVConstantPoolValue *CPV = RISCVConstantPoolValue::Create(GV: N->getGlobal());
9829 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: Ty, Align: Align(8));
9830 SDValue LC = DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: CPAddr);
9831 return DAG.getLoad(
9832 VT: Ty, dl: DL, Chain: DAG.getEntryNode(), Ptr: LC,
9833 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
9834}
9835
9836static SDValue getLargeExternalSymbol(ExternalSymbolSDNode *N, const SDLoc &DL,
9837 EVT Ty, SelectionDAG &DAG) {
9838 RISCVConstantPoolValue *CPV =
9839 RISCVConstantPoolValue::Create(C&: *DAG.getContext(), S: N->getSymbol());
9840 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: Ty, Align: Align(8));
9841 SDValue LC = DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: CPAddr);
9842 return DAG.getLoad(
9843 VT: Ty, dl: DL, Chain: DAG.getEntryNode(), Ptr: LC,
9844 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
9845}
9846
9847template <class NodeTy>
9848SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
9849 bool IsLocal, bool IsExternWeak) const {
9850 SDLoc DL(N);
9851 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
9852
9853 // When HWASAN is used and tagging of global variables is enabled
9854 // they should be accessed via the GOT, since the tagged address of a global
9855 // is incompatible with existing code models. This also applies to non-pic
9856 // mode.
9857 if (isPositionIndependent() || Subtarget.allowTaggedGlobals()) {
9858 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9859 if (IsLocal && !Subtarget.allowTaggedGlobals())
9860 // Use PC-relative addressing to access the symbol. This generates the
9861 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
9862 // %pcrel_lo(auipc)).
9863 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
9864
9865 // Use PC-relative addressing to access the GOT for this symbol, then load
9866 // the address from the GOT. This generates the pattern (PseudoLGA sym),
9867 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
9868 SDValue Load =
9869 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLGA, dl: DL, VT: Ty, Op1: Addr), 0);
9870 MachineFunction &MF = DAG.getMachineFunction();
9871 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9872 PtrInfo: MachinePointerInfo::getGOT(MF),
9873 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9874 MachineMemOperand::MOInvariant,
9875 MemTy: LLT(Ty.getSimpleVT()), base_alignment: Align(Ty.getFixedSizeInBits() / 8));
9876 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
9877 return Load;
9878 }
9879
9880 switch (getTargetMachine().getCodeModel()) {
9881 default:
9882 reportFatalUsageError(reason: "Unsupported code model for lowering");
9883 case CodeModel::Small: {
9884 // Generate a sequence for accessing addresses within the first 2 GiB of
9885 // address space.
9886 if (Subtarget.hasVendorXqcili()) {
9887 // Use QC.E.LI to generate the address, as this is easier to relax than
9888 // LUI/ADDI.
9889 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9890 return DAG.getNode(Opcode: RISCVISD::QC_E_LI, DL, VT: Ty, Operand: Addr);
9891 }
9892
9893 // This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
9894 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
9895 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
9896 SDValue MNHi = DAG.getNode(Opcode: RISCVISD::HI, DL, VT: Ty, Operand: AddrHi);
9897 return DAG.getNode(Opcode: RISCVISD::ADD_LO, DL, VT: Ty, N1: MNHi, N2: AddrLo);
9898 }
9899 case CodeModel::Medium: {
9900 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9901 if (IsExternWeak) {
9902 // An extern weak symbol may be undefined, i.e. have value 0, which may
9903 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
9904 // symbol. This generates the pattern (PseudoLGA sym), which expands to
9905 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
9906 SDValue Load =
9907 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLGA, dl: DL, VT: Ty, Op1: Addr), 0);
9908 MachineFunction &MF = DAG.getMachineFunction();
9909 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9910 PtrInfo: MachinePointerInfo::getGOT(MF),
9911 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9912 MachineMemOperand::MOInvariant,
9913 MemTy: LLT(Ty.getSimpleVT()), base_alignment: Align(Ty.getFixedSizeInBits() / 8));
9914 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
9915 return Load;
9916 }
9917
9918 // Generate a sequence for accessing addresses within any 2GiB range within
9919 // the address space. This generates the pattern (PseudoLLA sym), which
9920 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
9921 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
9922 }
9923 case CodeModel::Large: {
9924 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N))
9925 return getLargeGlobalAddress(N: G, DL, Ty, DAG);
9926
9927 // Using pc-relative mode for other node type.
9928 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9929 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
9930 }
9931 }
9932}
9933
9934SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
9935 SelectionDAG &DAG) const {
9936 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
9937 assert(N->getOffset() == 0 && "unexpected offset in global node");
9938 const GlobalValue *GV = N->getGlobal();
9939 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(GV);
9940 return getAddr(N, DAG, IsLocal, IsExternWeak: GV->hasExternalWeakLinkage());
9941}
9942
9943SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
9944 SelectionDAG &DAG) const {
9945 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Val&: Op);
9946
9947 return getAddr(N, DAG);
9948}
9949
9950SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
9951 SelectionDAG &DAG) const {
9952 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Val&: Op);
9953
9954 return getAddr(N, DAG);
9955}
9956
9957SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
9958 SelectionDAG &DAG) const {
9959 JumpTableSDNode *N = cast<JumpTableSDNode>(Val&: Op);
9960
9961 return getAddr(N, DAG);
9962}
9963
9964SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
9965 SelectionDAG &DAG,
9966 bool UseGOT) const {
9967 SDLoc DL(N);
9968 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
9969 const GlobalValue *GV = N->getGlobal();
9970 MVT XLenVT = Subtarget.getXLenVT();
9971
9972 if (UseGOT) {
9973 // Use PC-relative addressing to access the GOT for this TLS symbol, then
9974 // load the address from the GOT and add the thread pointer. This generates
9975 // the pattern (PseudoLA_TLS_IE sym), which expands to
9976 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
9977 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
9978 SDValue Load =
9979 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLS_IE, dl: DL, VT: Ty, Op1: Addr), 0);
9980 MachineFunction &MF = DAG.getMachineFunction();
9981 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9982 PtrInfo: MachinePointerInfo::getGOT(MF),
9983 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9984 MachineMemOperand::MOInvariant,
9985 MemTy: LLT(Ty.getSimpleVT()), base_alignment: Align(Ty.getFixedSizeInBits() / 8));
9986 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
9987
9988 // Add the thread pointer.
9989 SDValue TPReg = DAG.getRegister(Reg: RISCV::X4, VT: XLenVT);
9990 return DAG.getNode(Opcode: ISD::ADD, DL, VT: Ty, N1: Load, N2: TPReg);
9991 }
9992
9993 // Generate a sequence for accessing the address relative to the thread
9994 // pointer, with the appropriate adjustment for the thread pointer offset.
9995 // This generates the pattern
9996 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
9997 SDValue AddrHi =
9998 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_HI);
9999 SDValue AddrAdd =
10000 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_ADD);
10001 SDValue AddrLo =
10002 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_LO);
10003
10004 SDValue MNHi = DAG.getNode(Opcode: RISCVISD::HI, DL, VT: Ty, Operand: AddrHi);
10005 SDValue TPReg = DAG.getRegister(Reg: RISCV::X4, VT: XLenVT);
10006 SDValue MNAdd =
10007 DAG.getNode(Opcode: RISCVISD::ADD_TPREL, DL, VT: Ty, N1: MNHi, N2: TPReg, N3: AddrAdd);
10008 return DAG.getNode(Opcode: RISCVISD::ADD_LO, DL, VT: Ty, N1: MNAdd, N2: AddrLo);
10009}
10010
10011SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
10012 SelectionDAG &DAG) const {
10013 SDLoc DL(N);
10014 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
10015 IntegerType *CallTy = Type::getIntNTy(C&: *DAG.getContext(), N: Ty.getSizeInBits());
10016 const GlobalValue *GV = N->getGlobal();
10017
10018 // Use a PC-relative addressing mode to access the global dynamic GOT address.
10019 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
10020 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
10021 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
10022 SDValue Load =
10023 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLS_GD, dl: DL, VT: Ty, Op1: Addr), 0);
10024
10025 // Prepare argument list to generate call.
10026 ArgListTy Args;
10027 Args.emplace_back(args&: Load, args&: CallTy);
10028
10029 // Setup call to __tls_get_addr.
10030 TargetLowering::CallLoweringInfo CLI(DAG);
10031 CLI.setDebugLoc(DL)
10032 .setChain(DAG.getEntryNode())
10033 .setLibCallee(CC: CallingConv::C, ResultType: CallTy,
10034 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: Ty),
10035 ArgsList: std::move(Args));
10036
10037 return LowerCallTo(CLI).first;
10038}
10039
10040SDValue RISCVTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
10041 SelectionDAG &DAG) const {
10042 SDLoc DL(N);
10043 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
10044 const GlobalValue *GV = N->getGlobal();
10045
10046 // Use a PC-relative addressing mode to access the global dynamic GOT address.
10047 // This generates the pattern (PseudoLA_TLSDESC sym), which expands to
10048 //
10049 // auipc tX, %tlsdesc_hi(symbol) // R_RISCV_TLSDESC_HI20(symbol)
10050 // lw tY, tX, %tlsdesc_load_lo(label) // R_RISCV_TLSDESC_LOAD_LO12(label)
10051 // addi a0, tX, %tlsdesc_add_lo(label) // R_RISCV_TLSDESC_ADD_LO12(label)
10052 // jalr t0, tY // R_RISCV_TLSDESC_CALL(label)
10053 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
10054 return SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLSDESC, dl: DL, VT: Ty, Op1: Addr), 0);
10055}
10056
10057SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
10058 SelectionDAG &DAG) const {
10059 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
10060 assert(N->getOffset() == 0 && "unexpected offset in global node");
10061
10062 if (DAG.getTarget().useEmulatedTLS())
10063 return LowerToTLSEmulatedModel(GA: N, DAG);
10064
10065 TLSModel::Model Model = getTargetMachine().getTLSModel(GV: N->getGlobal());
10066
10067 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
10068 CallingConv::GHC)
10069 reportFatalUsageError(reason: "In GHC calling convention TLS is not supported");
10070
10071 SDValue Addr;
10072 switch (Model) {
10073 case TLSModel::LocalExec:
10074 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
10075 break;
10076 case TLSModel::InitialExec:
10077 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
10078 break;
10079 case TLSModel::LocalDynamic:
10080 case TLSModel::GeneralDynamic:
10081 Addr = DAG.getTarget().useTLSDESC() ? getTLSDescAddr(N, DAG)
10082 : getDynamicTLSAddr(N, DAG);
10083 break;
10084 }
10085
10086 return Addr;
10087}
10088
10089// Return true if Val is equal to (setcc LHS, RHS, CC).
10090// Return false if Val is the inverse of (setcc LHS, RHS, CC).
10091// Otherwise, return std::nullopt.
10092static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
10093 ISD::CondCode CC, SDValue Val) {
10094 assert(Val->getOpcode() == ISD::SETCC);
10095 SDValue LHS2 = Val.getOperand(i: 0);
10096 SDValue RHS2 = Val.getOperand(i: 1);
10097 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
10098
10099 if (LHS == LHS2 && RHS == RHS2) {
10100 if (CC == CC2)
10101 return true;
10102 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
10103 return false;
10104 } else if (LHS == RHS2 && RHS == LHS2) {
10105 CC2 = ISD::getSetCCSwappedOperands(Operation: CC2);
10106 if (CC == CC2)
10107 return true;
10108 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
10109 return false;
10110 }
10111
10112 return std::nullopt;
10113}
10114
10115static bool isSimm12Constant(SDValue V) {
10116 return isa<ConstantSDNode>(Val: V) && V->getAsAPIntVal().isSignedIntN(N: 12);
10117}
10118
10119static SDValue lowerSelectToBinOp(SDNode *N, SelectionDAG &DAG,
10120 const RISCVSubtarget &Subtarget) {
10121 SDValue CondV = N->getOperand(Num: 0);
10122 SDValue TrueV = N->getOperand(Num: 1);
10123 SDValue FalseV = N->getOperand(Num: 2);
10124 MVT VT = N->getSimpleValueType(ResNo: 0);
10125 SDLoc DL(N);
10126
10127 if (!Subtarget.hasConditionalMoveFusion()) {
10128 // (select c, -1, y) -> -c | y
10129 if (isAllOnesConstant(V: TrueV)) {
10130 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
10131 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
10132 }
10133 // (select c, y, -1) -> (c-1) | y
10134 if (isAllOnesConstant(V: FalseV)) {
10135 SDValue Neg = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV,
10136 N2: DAG.getAllOnesConstant(DL, VT));
10137 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
10138 }
10139
10140 const bool HasCZero = VT.isScalarInteger() && Subtarget.hasCZEROLike();
10141
10142 // (select c, 0, y) -> (c-1) & y
10143 if (isNullConstant(V: TrueV) && (!HasCZero || isSimm12Constant(V: FalseV))) {
10144 SDValue Neg =
10145 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
10146 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
10147 }
10148 if (isNullConstant(V: FalseV)) {
10149 if (auto *TrueC = dyn_cast<ConstantSDNode>(Val&: TrueV)) {
10150 // (select c, y, 0) -> (c * (y - 1)) + c
10151 int64_t MulImm = TrueC->getSExtValue();
10152 if (MulImm != INT64_MIN && isInt<12>(x: MulImm - 1) &&
10153 Subtarget.hasVendorXqciac())
10154 return DAG.getNode(Opcode: RISCVISD::QC_MULIADD, DL, VT, N1: CondV, N2: CondV,
10155 N3: DAG.getSignedTargetConstant(Val: MulImm - 1, DL, VT));
10156
10157 // (select c, (1 << ShAmount) + 1, 0) -> (c << ShAmount) + c
10158 uint64_t TrueM1 = TrueC->getZExtValue() - 1;
10159 if (isPowerOf2_64(Value: TrueM1)) {
10160 unsigned ShAmount = Log2_64(Value: TrueM1);
10161 if (Subtarget.hasShlAdd(ShAmt: ShAmount))
10162 return DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: CondV,
10163 N2: DAG.getTargetConstant(Val: ShAmount, DL, VT), N3: CondV);
10164 }
10165 }
10166 // (select c, y, 0) -> -c & y
10167 if (!HasCZero || isSimm12Constant(V: TrueV)) {
10168 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
10169 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
10170 }
10171 }
10172 }
10173
10174 // select c, ~x, x --> xor -c, x
10175 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
10176 const APInt &TrueVal = TrueV->getAsAPIntVal();
10177 const APInt &FalseVal = FalseV->getAsAPIntVal();
10178 if (~TrueVal == FalseVal) {
10179 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
10180 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Neg, N2: FalseV);
10181 }
10182 }
10183
10184 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
10185 // when both truev and falsev are also setcc.
10186 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
10187 FalseV.getOpcode() == ISD::SETCC) {
10188 SDValue LHS = CondV.getOperand(i: 0);
10189 SDValue RHS = CondV.getOperand(i: 1);
10190 ISD::CondCode CC = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10191
10192 // (select x, x, y) -> x | y
10193 // (select !x, x, y) -> x & y
10194 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: TrueV)) {
10195 return DAG.getNode(Opcode: *MatchResult ? ISD::OR : ISD::AND, DL, VT, N1: TrueV,
10196 N2: DAG.getFreeze(V: FalseV));
10197 }
10198 // (select x, y, x) -> x & y
10199 // (select !x, y, x) -> x | y
10200 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: FalseV)) {
10201 return DAG.getNode(Opcode: *MatchResult ? ISD::AND : ISD::OR, DL, VT,
10202 N1: DAG.getFreeze(V: TrueV), N2: FalseV);
10203 }
10204 }
10205
10206 return SDValue();
10207}
10208
10209// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
10210// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
10211// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
10212// being `0` or `-1`. In such cases we can replace `select` with `and`.
10213// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
10214// than `c0`?
10215static SDValue
10216foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
10217 const RISCVSubtarget &Subtarget) {
10218 if (Subtarget.hasShortForwardBranchIALU())
10219 return SDValue();
10220
10221 unsigned SelOpNo = 0;
10222 SDValue Sel = BO->getOperand(Num: 0);
10223 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
10224 SelOpNo = 1;
10225 Sel = BO->getOperand(Num: 1);
10226 }
10227
10228 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
10229 return SDValue();
10230
10231 unsigned ConstSelOpNo = 1;
10232 unsigned OtherSelOpNo = 2;
10233 if (!isa<ConstantSDNode>(Val: Sel->getOperand(Num: ConstSelOpNo))) {
10234 ConstSelOpNo = 2;
10235 OtherSelOpNo = 1;
10236 }
10237 SDValue ConstSelOp = Sel->getOperand(Num: ConstSelOpNo);
10238 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(Val&: ConstSelOp);
10239 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
10240 return SDValue();
10241
10242 SDValue ConstBinOp = BO->getOperand(Num: SelOpNo ^ 1);
10243 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(Val&: ConstBinOp);
10244 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
10245 return SDValue();
10246
10247 SDLoc DL(Sel);
10248 EVT VT = BO->getValueType(ResNo: 0);
10249
10250 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
10251 if (SelOpNo == 1)
10252 std::swap(a&: NewConstOps[0], b&: NewConstOps[1]);
10253
10254 SDValue NewConstOp =
10255 DAG.FoldConstantArithmetic(Opcode: BO->getOpcode(), DL, VT, Ops: NewConstOps);
10256 if (!NewConstOp)
10257 return SDValue();
10258
10259 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
10260 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
10261 return SDValue();
10262
10263 SDValue OtherSelOp = Sel->getOperand(Num: OtherSelOpNo);
10264 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
10265 if (SelOpNo == 1)
10266 std::swap(a&: NewNonConstOps[0], b&: NewNonConstOps[1]);
10267 SDValue NewNonConstOp = DAG.getNode(Opcode: BO->getOpcode(), DL, VT, Ops: NewNonConstOps);
10268
10269 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
10270 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
10271 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewT, RHS: NewF);
10272}
10273
10274SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
10275 SDValue CondV = Op.getOperand(i: 0);
10276 SDValue TrueV = Op.getOperand(i: 1);
10277 SDValue FalseV = Op.getOperand(i: 2);
10278 SDLoc DL(Op);
10279 MVT VT = Op.getSimpleValueType();
10280 MVT XLenVT = Subtarget.getXLenVT();
10281
10282 // Handle P extension packed types by bitcasting to an integer of
10283 // matching width and reusing the scalar selection mechanism.
10284 // Reachable cases:
10285 // RV32: v4i8/v2i16 -> select on i32
10286 // RV32: v8i8/v4i16 -> select on i64 (legalizes to two i32 selects)
10287 // RV64: v8i8/v4i16/v2i32 -> select on i64
10288 if (Subtarget.isPExtPackedType(VT)) {
10289 MVT IntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
10290 SDValue TrueVInt = DAG.getBitcast(VT: IntVT, V: TrueV);
10291 SDValue FalseVInt = DAG.getBitcast(VT: IntVT, V: FalseV);
10292 SDValue ResultInt =
10293 DAG.getNode(Opcode: ISD::SELECT, DL, VT: IntVT, N1: CondV, N2: TrueVInt, N3: FalseVInt);
10294 return DAG.getBitcast(VT, V: ResultInt);
10295 }
10296
10297 // Lower vector SELECTs to VSELECTs by splatting the condition.
10298 if (VT.isVector()) {
10299 MVT SplatCondVT = VT.changeVectorElementType(EltVT: MVT::i1);
10300 SDValue CondSplat = DAG.getSplat(VT: SplatCondVT, DL, Op: CondV);
10301 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CondSplat, N2: TrueV, N3: FalseV);
10302 }
10303
10304 // Try some other optimizations before falling back to generic lowering.
10305 if (SDValue V = lowerSelectToBinOp(N: Op.getNode(), DAG, Subtarget))
10306 return V;
10307
10308 // When there is no cost for GPR <-> FPR, we can use zicond select for
10309 // floating value when CondV is int type
10310 bool FPinGPR = Subtarget.hasStdExtZfinx();
10311
10312 // We can handle FGPR without spliting into hi/lo parts
10313 bool FitsInGPR = TypeSize::isKnownLE(LHS: VT.getSizeInBits(),
10314 RHS: Subtarget.getXLenVT().getSizeInBits());
10315
10316 bool UseZicondForFPSel = Subtarget.hasStdExtZicond() && FPinGPR &&
10317 VT.isFloatingPoint() && FitsInGPR;
10318
10319 if (UseZicondForFPSel) {
10320
10321 auto CastToInt = [&](SDValue V) -> SDValue {
10322 // Treat +0.0 as int 0 to enable single 'czero' instruction generation.
10323 if (isNullFPConstant(V))
10324 return DAG.getConstant(Val: 0, DL, VT: XLenVT);
10325
10326 if (VT == MVT::f16)
10327 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: V);
10328
10329 if (VT == MVT::f32 && Subtarget.is64Bit())
10330 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: XLenVT, Operand: V);
10331
10332 return DAG.getBitcast(VT: XLenVT, V);
10333 };
10334
10335 SDValue TrueVInt = CastToInt(TrueV);
10336 SDValue FalseVInt = CastToInt(FalseV);
10337
10338 // Emit integer SELECT (lowers to Zicond)
10339 SDValue ResultInt =
10340 DAG.getNode(Opcode: ISD::SELECT, DL, VT: XLenVT, N1: CondV, N2: TrueVInt, N3: FalseVInt);
10341
10342 // Convert back to floating VT
10343 if (VT == MVT::f32 && Subtarget.is64Bit())
10344 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT, Operand: ResultInt);
10345
10346 if (VT == MVT::f16)
10347 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: ResultInt);
10348
10349 return DAG.getBitcast(VT, V: ResultInt);
10350 }
10351
10352 // When Zicond or XVentanaCondOps is present, emit CZERO_EQZ and CZERO_NEZ
10353 // nodes to implement the SELECT. Performing the lowering here allows for
10354 // greater control over when CZERO_{EQZ/NEZ} are used vs another branchless
10355 // sequence or RISCVISD::SELECT_CC node (branch-based select).
10356 if (Subtarget.hasCZEROLike() && VT.isScalarInteger()) {
10357
10358 // (select c, t, 0) -> (czero_eqz t, c)
10359 if (isNullConstant(V: FalseV))
10360 return DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV);
10361 // (select c, 0, f) -> (czero_nez f, c)
10362 if (isNullConstant(V: TrueV))
10363 return DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV);
10364
10365 // Check to see if a given operation is a 'NOT', if so return the negated
10366 // operand
10367 auto getNotOperand = [](const SDValue &Op) -> std::optional<const SDValue> {
10368 using namespace llvm::SDPatternMatch;
10369 SDValue Xor;
10370 if (sd_match(N: Op, P: m_OneUse(P: m_Not(V: m_Value(N&: Xor))))) {
10371 return Xor;
10372 }
10373 return std::nullopt;
10374 };
10375 // (select c, (and f, x), f) -> (or (and f, x), (czero_nez f, c))
10376 // (select c, (and f, ~x), f) -> (andn f, (czero_eqz x, c))
10377 if (TrueV.getOpcode() == ISD::AND &&
10378 (TrueV.getOperand(i: 0) == FalseV || TrueV.getOperand(i: 1) == FalseV)) {
10379 auto NotOperand = (TrueV.getOperand(i: 0) == FalseV)
10380 ? getNotOperand(TrueV.getOperand(i: 1))
10381 : getNotOperand(TrueV.getOperand(i: 0));
10382 if (NotOperand) {
10383 SDValue CMOV =
10384 DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: *NotOperand, N2: CondV);
10385 SDValue NOT = DAG.getNOT(DL, Val: CMOV, VT);
10386 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseV, N2: NOT);
10387 }
10388 return DAG.getNode(
10389 Opcode: ISD::OR, DL, VT, N1: TrueV,
10390 N2: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV));
10391 }
10392
10393 // (select c, t, (and t, x)) -> (or (czero_eqz t, c), (and t, x))
10394 // (select c, t, (and t, ~x)) -> (andn t, (czero_nez x, c))
10395 if (FalseV.getOpcode() == ISD::AND &&
10396 (FalseV.getOperand(i: 0) == TrueV || FalseV.getOperand(i: 1) == TrueV)) {
10397 auto NotOperand = (FalseV.getOperand(i: 0) == TrueV)
10398 ? getNotOperand(FalseV.getOperand(i: 1))
10399 : getNotOperand(FalseV.getOperand(i: 0));
10400 if (NotOperand) {
10401 SDValue CMOV =
10402 DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: *NotOperand, N2: CondV);
10403 SDValue NOT = DAG.getNOT(DL, Val: CMOV, VT);
10404 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueV, N2: NOT);
10405 }
10406 return DAG.getNode(
10407 Opcode: ISD::OR, DL, VT, N1: FalseV,
10408 N2: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV));
10409 }
10410
10411 // (select c, c1, c2) -> (add (czero_nez c2 - c1, c), c1)
10412 // (select c, c1, c2) -> (add (czero_eqz c1 - c2, c), c2)
10413 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
10414 const APInt &TrueVal = TrueV->getAsAPIntVal();
10415 const APInt &FalseVal = FalseV->getAsAPIntVal();
10416
10417 // Prefer these over Zicond to avoid materializing an immediate:
10418 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
10419 // (select (x > -1), z, y) -> x >> (XLEN - 1) & (y - z) + z
10420 if (CondV.getOpcode() == ISD::SETCC &&
10421 CondV.getOperand(i: 0).getValueType() == VT && CondV.hasOneUse()) {
10422 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10423 if ((CCVal == ISD::SETLT && isNullConstant(V: CondV.getOperand(i: 1))) ||
10424 (CCVal == ISD::SETGT && isAllOnesConstant(V: CondV.getOperand(i: 1)))) {
10425 int64_t TrueImm = TrueVal.getSExtValue();
10426 int64_t FalseImm = FalseVal.getSExtValue();
10427 if (CCVal == ISD::SETGT)
10428 std::swap(a&: TrueImm, b&: FalseImm);
10429 if (isInt<12>(x: TrueImm) && isInt<12>(x: FalseImm) &&
10430 isInt<12>(x: TrueImm - FalseImm)) {
10431 SDValue SRA =
10432 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CondV.getOperand(i: 0),
10433 N2: DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT));
10434 SDValue AND =
10435 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
10436 N2: DAG.getSignedConstant(Val: TrueImm - FalseImm, DL, VT));
10437 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND,
10438 N2: DAG.getSignedConstant(Val: FalseImm, DL, VT));
10439 }
10440 }
10441 }
10442
10443 // Use SHL/ADDI (and possible XORI) to avoid having to materialize
10444 // a constant in register
10445 if ((TrueVal - FalseVal).isPowerOf2() && FalseVal.isSignedIntN(N: 12)) {
10446 SDValue Log2 = DAG.getConstant(Val: (TrueVal - FalseVal).logBase2(), DL, VT);
10447 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10448 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FalseV, N2: BitDiff);
10449 }
10450 if ((FalseVal - TrueVal).isPowerOf2() && TrueVal.isSignedIntN(N: 12)) {
10451 SDValue Log2 = DAG.getConstant(Val: (FalseVal - TrueVal).logBase2(), DL, VT);
10452 CondV = DAG.getLogicalNOT(DL, Val: CondV, VT: CondV->getValueType(ResNo: 0));
10453 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10454 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: TrueV, N2: BitDiff);
10455 }
10456
10457 auto getCost = [&](const APInt &Delta, const APInt &Addend) {
10458 const int DeltaCost = RISCVMatInt::getIntMatCost(
10459 Val: Delta, Size: Subtarget.getXLen(), STI: Subtarget, /*CompressionCost=*/true);
10460 // Does the addend fold into an ADDI
10461 if (Addend.isSignedIntN(N: 12))
10462 return DeltaCost;
10463 const int AddendCost = RISCVMatInt::getIntMatCost(
10464 Val: Addend, Size: Subtarget.getXLen(), STI: Subtarget, /*CompressionCost=*/true);
10465 return AddendCost + DeltaCost;
10466 };
10467 bool IsCZERO_NEZ = getCost(FalseVal - TrueVal, TrueVal) <=
10468 getCost(TrueVal - FalseVal, FalseVal);
10469 SDValue LHSVal = DAG.getConstant(
10470 Val: IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
10471 SDValue CMOV =
10472 DAG.getNode(Opcode: IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10473 DL, VT, N1: LHSVal, N2: CondV);
10474 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CMOV, N2: IsCZERO_NEZ ? TrueV : FalseV);
10475 }
10476
10477 // (select c, c1, t) -> (add (czero_nez t - c1, c), c1)
10478 // (select c, t, c1) -> (add (czero_eqz t - c1, c), c1)
10479 if (isa<ConstantSDNode>(Val: TrueV) != isa<ConstantSDNode>(Val: FalseV)) {
10480 bool IsCZERO_NEZ = isa<ConstantSDNode>(Val: TrueV);
10481 SDValue ConstVal = IsCZERO_NEZ ? TrueV : FalseV;
10482 SDValue RegV = IsCZERO_NEZ ? FalseV : TrueV;
10483 int64_t RawConstVal = cast<ConstantSDNode>(Val&: ConstVal)->getSExtValue();
10484 // Efficient only if the constant and its negation fit into `ADDI`
10485 // Prefer Add/Sub over Xor since can be compressed for small immediates
10486 if (isInt<12>(x: RawConstVal)) {
10487 // Fall back to XORI if Const == -0x800 since we don't have SUBI.
10488 unsigned SubOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::SUB;
10489 unsigned AddOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::ADD;
10490 SDValue SubOp = DAG.getNode(Opcode: SubOpc, DL, VT, N1: RegV, N2: ConstVal);
10491 SDValue CZERO =
10492 DAG.getNode(Opcode: IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10493 DL, VT, N1: SubOp, N2: CondV);
10494 return DAG.getNode(Opcode: AddOpc, DL, VT, N1: CZERO, N2: ConstVal);
10495 }
10496 }
10497
10498 // (select c, t, f) -> (or (czero_eqz t, c), (czero_nez f, c))
10499 // Unless we have the short forward branch optimization.
10500 if (!Subtarget.hasConditionalMoveFusion())
10501 return DAG.getNode(
10502 Opcode: ISD::OR, DL, VT,
10503 N1: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV),
10504 N2: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV),
10505 Flags: SDNodeFlags::Disjoint);
10506 }
10507
10508 if (Op.hasOneUse()) {
10509 unsigned UseOpc = Op->user_begin()->getOpcode();
10510 if (isBinOp(Opcode: UseOpc) && DAG.isSafeToSpeculativelyExecute(Opcode: UseOpc)) {
10511 SDNode *BinOp = *Op->user_begin();
10512 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(BO: *Op->user_begin(),
10513 DAG, Subtarget)) {
10514 DAG.ReplaceAllUsesWith(From: BinOp, To: &NewSel);
10515 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
10516 // may return a constant node and cause crash in lowerSELECT.
10517 if (NewSel.getOpcode() == ISD::SELECT)
10518 return lowerSELECT(Op: NewSel, DAG);
10519 return NewSel;
10520 }
10521 }
10522 }
10523
10524 // (select cc, 1.0, 0.0) -> (sint_to_fp (zext cc))
10525 // (select cc, 0.0, 1.0) -> (sint_to_fp (zext (xor cc, 1)))
10526 const ConstantFPSDNode *FPTV = dyn_cast<ConstantFPSDNode>(Val&: TrueV);
10527 const ConstantFPSDNode *FPFV = dyn_cast<ConstantFPSDNode>(Val&: FalseV);
10528 if (FPTV && FPFV) {
10529 if (FPTV->isOne() && FPFV->isPosZero())
10530 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: CondV);
10531 if (FPTV->isPosZero() && FPFV->isOne()) {
10532 SDValue XOR = DAG.getNode(Opcode: ISD::XOR, DL, VT: XLenVT, N1: CondV,
10533 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
10534 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: XOR);
10535 }
10536 }
10537
10538 // If the condition is not an integer SETCC which operates on XLenVT, we need
10539 // to emit a RISCVISD::SELECT_CC comparing the condition to zero. i.e.:
10540 // (select condv, truev, falsev)
10541 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
10542 if (CondV.getOpcode() != ISD::SETCC ||
10543 CondV.getOperand(i: 0).getSimpleValueType() != XLenVT) {
10544 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
10545 SDValue SetNE = DAG.getCondCode(Cond: ISD::SETNE);
10546
10547 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
10548
10549 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, Ops);
10550 }
10551
10552 // If the CondV is the output of a SETCC node which operates on XLenVT inputs,
10553 // then merge the SETCC node into the lowered RISCVISD::SELECT_CC to take
10554 // advantage of the integer compare+branch instructions. i.e.:
10555 // (select (setcc lhs, rhs, cc), truev, falsev)
10556 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
10557 SDValue LHS = CondV.getOperand(i: 0);
10558 SDValue RHS = CondV.getOperand(i: 1);
10559 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10560
10561 // Special case for a select of 2 constants that have a difference of 1.
10562 // Normally this is done by DAGCombine, but if the select is introduced by
10563 // type legalization or op legalization, we miss it. Restricting to SETLT
10564 // case for now because that is what signed saturating add/sub need.
10565 // FIXME: We don't need the condition to be SETLT or even a SETCC,
10566 // but we would probably want to swap the true/false values if the condition
10567 // is SETGE/SETLE to avoid an XORI.
10568 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
10569 CCVal == ISD::SETLT) {
10570 const APInt &TrueVal = TrueV->getAsAPIntVal();
10571 const APInt &FalseVal = FalseV->getAsAPIntVal();
10572 if (TrueVal - 1 == FalseVal)
10573 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: FalseV);
10574 if (TrueVal + 1 == FalseVal)
10575 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: CondV);
10576 }
10577
10578 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
10579 // 1 < x ? x : 1 -> 0 < x ? x : 1
10580 if (isOneConstant(V: LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
10581 RHS == TrueV && LHS == FalseV) {
10582 LHS = DAG.getConstant(Val: 0, DL, VT);
10583 // 0 <u x is the same as x != 0.
10584 if (CCVal == ISD::SETULT) {
10585 std::swap(a&: LHS, b&: RHS);
10586 CCVal = ISD::SETNE;
10587 }
10588 }
10589
10590 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
10591 if (isAllOnesConstant(V: RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
10592 RHS == FalseV) {
10593 RHS = DAG.getConstant(Val: 0, DL, VT);
10594 }
10595
10596 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
10597
10598 if (isa<ConstantSDNode>(Val: TrueV) && !isa<ConstantSDNode>(Val: FalseV)) {
10599 // (select (setcc lhs, rhs, CC), constant, falsev)
10600 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
10601 std::swap(a&: TrueV, b&: FalseV);
10602 TargetCC = DAG.getCondCode(Cond: ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType()));
10603 }
10604
10605 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
10606 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, Ops);
10607}
10608
10609SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
10610 SDValue CondV = Op.getOperand(i: 1);
10611 SDLoc DL(Op);
10612 MVT XLenVT = Subtarget.getXLenVT();
10613
10614 if (CondV.getOpcode() == ISD::SETCC &&
10615 CondV.getOperand(i: 0).getValueType() == XLenVT) {
10616 SDValue LHS = CondV.getOperand(i: 0);
10617 SDValue RHS = CondV.getOperand(i: 1);
10618 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10619
10620 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
10621
10622 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
10623 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
10624 N2: LHS, N3: RHS, N4: TargetCC, N5: Op.getOperand(i: 2));
10625 }
10626
10627 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
10628 N2: CondV, N3: DAG.getConstant(Val: 0, DL, VT: XLenVT),
10629 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: Op.getOperand(i: 2));
10630}
10631
10632SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
10633 MachineFunction &MF = DAG.getMachineFunction();
10634 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
10635
10636 SDLoc DL(Op);
10637 SDValue FI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
10638 VT: getPointerTy(DL: MF.getDataLayout()));
10639
10640 // vastart just stores the address of the VarArgsFrameIndex slot into the
10641 // memory location argument.
10642 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
10643 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FI, Ptr: Op.getOperand(i: 1),
10644 PtrInfo: MachinePointerInfo(SV));
10645}
10646
10647SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
10648 SelectionDAG &DAG) const {
10649 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10650 MachineFunction &MF = DAG.getMachineFunction();
10651 MachineFrameInfo &MFI = MF.getFrameInfo();
10652 MFI.setFrameAddressIsTaken(true);
10653 Register FrameReg = RI.getFrameRegister(MF);
10654 int XLenInBytes = Subtarget.getXLen() / 8;
10655
10656 EVT VT = Op.getValueType();
10657 SDLoc DL(Op);
10658 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT);
10659 unsigned Depth = Op.getConstantOperandVal(i: 0);
10660 while (Depth--) {
10661 int Offset = -(XLenInBytes * 2);
10662 SDValue Ptr = DAG.getNode(
10663 Opcode: ISD::ADD, DL, VT, N1: FrameAddr,
10664 N2: DAG.getSignedConstant(Val: Offset, DL, VT: getPointerTy(DL: DAG.getDataLayout())));
10665 FrameAddr =
10666 DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: MachinePointerInfo());
10667 }
10668 return FrameAddr;
10669}
10670
10671SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
10672 SelectionDAG &DAG) const {
10673 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10674 MachineFunction &MF = DAG.getMachineFunction();
10675 MachineFrameInfo &MFI = MF.getFrameInfo();
10676 MFI.setReturnAddressIsTaken(true);
10677 MVT XLenVT = Subtarget.getXLenVT();
10678 int XLenInBytes = Subtarget.getXLen() / 8;
10679
10680 EVT VT = Op.getValueType();
10681 SDLoc DL(Op);
10682 unsigned Depth = Op.getConstantOperandVal(i: 0);
10683 if (Depth) {
10684 int Off = -XLenInBytes;
10685 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
10686 SDValue Offset = DAG.getSignedConstant(Val: Off, DL, VT);
10687 return DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(),
10688 Ptr: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr, N2: Offset),
10689 PtrInfo: MachinePointerInfo());
10690 }
10691
10692 // Return the value of the return address register, marking it an implicit
10693 // live-in.
10694 Register Reg = MF.addLiveIn(PReg: RI.getRARegister(), RC: getRegClassFor(VT: XLenVT));
10695 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT: XLenVT);
10696}
10697
10698SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
10699 SelectionDAG &DAG) const {
10700 SDLoc DL(Op);
10701 SDValue Lo = Op.getOperand(i: 0);
10702 SDValue Hi = Op.getOperand(i: 1);
10703 SDValue Shamt = Op.getOperand(i: 2);
10704 EVT VT = Lo.getValueType();
10705 unsigned XLen = Subtarget.getXLen();
10706
10707 // With P extension, use SLX (FSHL) for the high part.
10708 if (Subtarget.hasStdExtP()) {
10709 // HiRes = fshl(Hi, Lo, Shamt) - correct when Shamt < XLen
10710 SDValue HiRes = DAG.getNode(Opcode: ISD::FSHL, DL, VT, N1: Hi, N2: Lo, N3: Shamt);
10711 // LoRes = Lo << Shamt - correct Lo when Shamt < XLen,
10712 // Mask shift amount to avoid UB when Shamt >= XLen.
10713 SDValue ShamtMasked =
10714 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shamt, N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10715 SDValue LoRes = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMasked);
10716
10717 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
10718 // FIXME: We should use a select and let LowerSelect make the
10719 // optimizations.
10720 SDValue ShAmtExt =
10721 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Shamt,
10722 N2: DAG.getConstant(Val: XLen - Log2_32(Value: XLen) - 1, DL, VT));
10723 SDValue Mask = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: ShAmtExt,
10724 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10725
10726 // When Shamt >= XLen: HiRes = LoRes, LoRes = 0
10727 // HiRes = (HiRes & ~Mask) | (LoRes & Mask)
10728 SDValue HiMasked =
10729 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10730 SDValue LoMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: Mask);
10731 HiRes =
10732 DAG.getNode(Opcode: ISD::OR, DL, VT, N1: HiMasked, N2: LoMasked, Flags: SDNodeFlags::Disjoint);
10733
10734 // LoRes = LoRes & ~Mask (clear when Shamt >= XLen)
10735 LoRes = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10736
10737 return DAG.getMergeValues(Ops: {LoRes, HiRes}, dl: DL);
10738 }
10739
10740 // if Shamt-XLEN < 0: // Shamt < XLEN
10741 // Lo = Lo << Shamt
10742 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
10743 // else:
10744 // Lo = 0
10745 // Hi = Lo << (Shamt-XLEN)
10746
10747 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
10748 SDValue One = DAG.getConstant(Val: 1, DL, VT);
10749 SDValue MinusXLen = DAG.getSignedConstant(Val: -(int)XLen, DL, VT);
10750 SDValue XLenMinus1 = DAG.getConstant(Val: XLen - 1, DL, VT);
10751 SDValue ShamtMinusXLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusXLen);
10752 SDValue XLenMinus1Shamt = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: XLenMinus1, N2: Shamt);
10753
10754 SDValue LoTrue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: Shamt);
10755 SDValue ShiftRight1Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: One);
10756 SDValue ShiftRightLo =
10757 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShiftRight1Lo, N2: XLenMinus1Shamt);
10758 SDValue ShiftLeftHi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: Shamt);
10759 SDValue HiTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftLeftHi, N2: ShiftRightLo);
10760 SDValue HiFalse = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMinusXLen);
10761
10762 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusXLen, RHS: Zero, Cond: ISD::SETLT);
10763
10764 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: Zero);
10765 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
10766
10767 SDValue Parts[2] = {Lo, Hi};
10768 return DAG.getMergeValues(Ops: Parts, dl: DL);
10769}
10770
10771SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
10772 bool IsSRA) const {
10773 SDLoc DL(Op);
10774 SDValue Lo = Op.getOperand(i: 0);
10775 SDValue Hi = Op.getOperand(i: 1);
10776 SDValue Shamt = Op.getOperand(i: 2);
10777 EVT VT = Lo.getValueType();
10778
10779 // With P extension, use NSRL/NSRA for RV32 or FSHR (SRX) for RV64.
10780 if (Subtarget.hasStdExtP()) {
10781 unsigned XLen = Subtarget.getXLen();
10782
10783 SDValue LoRes;
10784 if (Subtarget.is64Bit()) {
10785 // On RV64, use FSHR (SRX instruction) for the low part. We will need
10786 // to fix this later if ShAmt >= 64.
10787 LoRes = DAG.getNode(Opcode: ISD::FSHR, DL, VT, N1: Hi, N2: Lo, N3: Shamt);
10788 } else {
10789 // On RV32, use NSRL/NSRA for the low part.
10790 // NSRL/NSRA read 6 bits of shift amount, so they handle Shamt >= 32
10791 // correctly.
10792 LoRes = DAG.getNode(Opcode: IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL, VT, N1: Lo,
10793 N2: Hi, N3: Shamt);
10794 }
10795
10796 // Mask shift amount to avoid UB when Shamt >= XLen.
10797 SDValue ShamtMasked =
10798 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shamt, N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10799 SDValue HiRes =
10800 DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL, VT, N1: Hi, N2: ShamtMasked);
10801
10802 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
10803 // FIXME: We should use a select and let LowerSelect make the
10804 // optimizations.
10805 SDValue ShAmtExt =
10806 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Shamt,
10807 N2: DAG.getConstant(Val: XLen - Log2_32(Value: XLen) - 1, DL, VT));
10808 SDValue Mask = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: ShAmtExt,
10809 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10810
10811 if (Subtarget.is64Bit()) {
10812 // On RV64, FSHR masks shift amount to 63. We need to replace LoRes
10813 // with HiRes when Shamt >= 64.
10814 // LoRes = (LoRes & ~Mask) | (HiRes & Mask)
10815 SDValue LoMasked =
10816 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10817 SDValue HiMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: Mask);
10818 LoRes = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoMasked, N2: HiMasked,
10819 Flags: SDNodeFlags::Disjoint);
10820 }
10821
10822 // If ShAmt >= XLen, we need to replace HiRes with 0 or sign bits.
10823 if (IsSRA) {
10824 // sra hi, hi, (mask & (XLen-1)) - shifts by XLen-1 when shamt >= XLen
10825 SDValue MaskAmt = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mask,
10826 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10827 HiRes = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: HiRes, N2: MaskAmt);
10828 } else {
10829 // andn hi, hi, mask - clears hi when shamt >= XLen
10830 HiRes = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10831 }
10832
10833 return DAG.getMergeValues(Ops: {LoRes, HiRes}, dl: DL);
10834 }
10835
10836 // SRA expansion:
10837 // if Shamt-XLEN < 0: // Shamt < XLEN
10838 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
10839 // Hi = Hi >>s Shamt
10840 // else:
10841 // Lo = Hi >>s (Shamt-XLEN);
10842 // Hi = Hi >>s (XLEN-1)
10843 //
10844 // SRL expansion:
10845 // if Shamt-XLEN < 0: // Shamt < XLEN
10846 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
10847 // Hi = Hi >>u Shamt
10848 // else:
10849 // Lo = Hi >>u (Shamt-XLEN);
10850 // Hi = 0;
10851
10852 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
10853
10854 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
10855 SDValue One = DAG.getConstant(Val: 1, DL, VT);
10856 SDValue MinusXLen = DAG.getSignedConstant(Val: -(int)Subtarget.getXLen(), DL, VT);
10857 SDValue XLenMinus1 = DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT);
10858 SDValue ShamtMinusXLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusXLen);
10859 SDValue XLenMinus1Shamt = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: XLenMinus1, N2: Shamt);
10860
10861 SDValue ShiftRightLo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: Shamt);
10862 SDValue ShiftLeftHi1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: One);
10863 SDValue ShiftLeftHi =
10864 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShiftLeftHi1, N2: XLenMinus1Shamt);
10865 SDValue LoTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftRightLo, N2: ShiftLeftHi);
10866 SDValue HiTrue = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: Shamt);
10867 SDValue LoFalse = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: ShamtMinusXLen);
10868 SDValue HiFalse =
10869 IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Hi, N2: XLenMinus1) : Zero;
10870
10871 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusXLen, RHS: Zero, Cond: ISD::SETLT);
10872
10873 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: LoFalse);
10874 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
10875
10876 SDValue Parts[2] = {Lo, Hi};
10877 return DAG.getMergeValues(Ops: Parts, dl: DL);
10878}
10879
10880// Lower splats of i1 types to SETCC. For each mask vector type, we have a
10881// legal equivalently-sized i8 type, so we can use that as a go-between.
10882SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
10883 SelectionDAG &DAG) const {
10884 SDLoc DL(Op);
10885 MVT VT = Op.getSimpleValueType();
10886 SDValue SplatVal = Op.getOperand(i: 0);
10887 // All-zeros or all-ones splats are handled specially.
10888 if (ISD::isConstantSplatVectorAllOnes(N: Op.getNode())) {
10889 SDValue VL = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget).second;
10890 return DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT, Operand: VL);
10891 }
10892 if (ISD::isConstantSplatVectorAllZeros(N: Op.getNode())) {
10893 SDValue VL = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget).second;
10894 return DAG.getNode(Opcode: RISCVISD::VMCLR_VL, DL, VT, Operand: VL);
10895 }
10896 MVT InterVT = VT.changeVectorElementType(EltVT: MVT::i8);
10897 SplatVal = DAG.getNode(Opcode: ISD::AND, DL, VT: SplatVal.getValueType(), N1: SplatVal,
10898 N2: DAG.getConstant(Val: 1, DL, VT: SplatVal.getValueType()));
10899 SDValue LHS = DAG.getSplatVector(VT: InterVT, DL, Op: SplatVal);
10900 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: InterVT);
10901 return DAG.getSetCC(DL, VT, LHS, RHS: Zero, Cond: ISD::SETNE);
10902}
10903
10904// Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
10905// illegal (currently only vXi64 RV32).
10906// FIXME: We could also catch non-constant sign-extended i32 values and lower
10907// them to VMV_V_X_VL.
10908SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
10909 SelectionDAG &DAG) const {
10910 SDLoc DL(Op);
10911 MVT VecVT = Op.getSimpleValueType();
10912 assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
10913 "Unexpected SPLAT_VECTOR_PARTS lowering");
10914
10915 assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
10916 SDValue Lo = Op.getOperand(i: 0);
10917 SDValue Hi = Op.getOperand(i: 1);
10918
10919 MVT ContainerVT = VecVT;
10920 if (VecVT.isFixedLengthVector())
10921 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
10922
10923 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
10924
10925 SDValue Res =
10926 splatPartsI64WithVL(DL, VT: ContainerVT, Passthru: SDValue(), Lo, Hi, VL, DAG);
10927
10928 if (VecVT.isFixedLengthVector())
10929 Res = convertFromScalableVector(VT: VecVT, V: Res, DAG, Subtarget);
10930
10931 return Res;
10932}
10933
10934// Custom-lower extensions from mask vectors by using a vselect either with 1
10935// for zero/any-extension or -1 for sign-extension:
10936// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
10937// Note that any-extension is lowered identically to zero-extension.
10938SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
10939 int64_t ExtTrueVal) const {
10940 SDLoc DL(Op);
10941 MVT VecVT = Op.getSimpleValueType();
10942 SDValue Src = Op.getOperand(i: 0);
10943 // Only custom-lower extensions from mask types
10944 assert(Src.getValueType().isVector() &&
10945 Src.getValueType().getVectorElementType() == MVT::i1);
10946
10947 if (VecVT.isScalableVector()) {
10948 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: VecVT);
10949 SDValue SplatTrueVal = DAG.getSignedConstant(Val: ExtTrueVal, DL, VT: VecVT);
10950 if (Src.getOpcode() == ISD::XOR &&
10951 ISD::isConstantSplatVectorAllOnes(N: Src.getOperand(i: 1).getNode()))
10952 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Src.getOperand(i: 0), N2: SplatZero,
10953 N3: SplatTrueVal);
10954 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Src, N2: SplatTrueVal, N3: SplatZero);
10955 }
10956
10957 MVT ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
10958 MVT I1ContainerVT =
10959 MVT::getVectorVT(VT: MVT::i1, EC: ContainerVT.getVectorElementCount());
10960
10961 SDValue CC = convertToScalableVector(VT: I1ContainerVT, V: Src, DAG, Subtarget);
10962
10963 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
10964
10965 MVT XLenVT = Subtarget.getXLenVT();
10966 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
10967 SDValue SplatTrueVal = DAG.getSignedConstant(Val: ExtTrueVal, DL, VT: XLenVT);
10968
10969 if (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
10970 SDValue Xor = Src.getOperand(i: 0);
10971 if (Xor.getOpcode() == RISCVISD::VMXOR_VL) {
10972 SDValue ScalableOnes = Xor.getOperand(i: 1);
10973 if (ScalableOnes.getOpcode() == ISD::INSERT_SUBVECTOR &&
10974 ScalableOnes.getOperand(i: 0).isUndef() &&
10975 ISD::isConstantSplatVectorAllOnes(
10976 N: ScalableOnes.getOperand(i: 1).getNode())) {
10977 CC = Xor.getOperand(i: 0);
10978 std::swap(a&: SplatZero, b&: SplatTrueVal);
10979 }
10980 }
10981 }
10982
10983 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
10984 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatZero, N3: VL);
10985 SplatTrueVal = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
10986 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatTrueVal, N3: VL);
10987 SDValue Select =
10988 DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: CC, N2: SplatTrueVal,
10989 N3: SplatZero, N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
10990
10991 return convertFromScalableVector(VT: VecVT, V: Select, DAG, Subtarget);
10992}
10993
10994// Custom-lower truncations from vectors to mask vectors by using a mask and a
10995// setcc operation:
10996// (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
10997SDValue RISCVTargetLowering::lowerVectorMaskTrunc(SDValue Op,
10998 SelectionDAG &DAG) const {
10999 SDLoc DL(Op);
11000 EVT MaskVT = Op.getValueType();
11001 // Only expect to custom-lower truncations to mask types
11002 assert(MaskVT.isVectorOf(MVT::i1) &&
11003 "Unexpected type for vector mask lowering");
11004 SDValue Src = Op.getOperand(i: 0);
11005 MVT VecVT = Src.getSimpleValueType();
11006 // If this is a fixed vector, we need to convert it to a scalable vector.
11007 MVT ContainerVT = VecVT;
11008
11009 if (VecVT.isFixedLengthVector()) {
11010 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11011 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
11012 }
11013
11014 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11015
11016 SDValue SplatOne = DAG.getConstant(Val: 1, DL, VT: Subtarget.getXLenVT());
11017 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
11018
11019 SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
11020 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatOne, N3: VL);
11021 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
11022 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatZero, N3: VL);
11023
11024 MVT MaskContainerVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
11025 SDValue Trunc = DAG.getNode(Opcode: RISCVISD::AND_VL, DL, VT: ContainerVT, N1: Src, N2: SplatOne,
11026 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
11027 Trunc = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: MaskContainerVT,
11028 Ops: {Trunc, SplatZero, DAG.getCondCode(Cond: ISD::SETNE),
11029 DAG.getUNDEF(VT: MaskContainerVT), Mask, VL});
11030 if (MaskVT.isFixedLengthVector())
11031 Trunc = convertFromScalableVector(VT: MaskVT, V: Trunc, DAG, Subtarget);
11032 return Trunc;
11033}
11034
11035SDValue RISCVTargetLowering::lowerVectorTrunc(SDValue Op,
11036 SelectionDAG &DAG) const {
11037 unsigned Opc = Op.getOpcode();
11038 SDLoc DL(Op);
11039
11040 MVT VT = Op.getSimpleValueType();
11041 // Only custom-lower vector truncates
11042 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
11043
11044 // Truncates to mask types are handled differently
11045 if (VT.getVectorElementType() == MVT::i1)
11046 return lowerVectorMaskTrunc(Op, DAG);
11047
11048 // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
11049 // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
11050 // truncate by one power of two at a time.
11051 MVT DstEltVT = VT.getVectorElementType();
11052
11053 SDValue Src = Op.getOperand(i: 0);
11054 MVT SrcVT = Src.getSimpleValueType();
11055 MVT SrcEltVT = SrcVT.getVectorElementType();
11056
11057 assert(DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) &&
11058 isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
11059 "Unexpected vector truncate lowering");
11060
11061 MVT ContainerVT = SrcVT;
11062 if (SrcVT.isFixedLengthVector()) {
11063 ContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
11064 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
11065 }
11066
11067 SDValue Result = Src;
11068 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
11069
11070 unsigned NewOpc;
11071 if (Opc == ISD::TRUNCATE_SSAT_S)
11072 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
11073 else if (Opc == ISD::TRUNCATE_USAT_U)
11074 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
11075 else
11076 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL;
11077
11078 do {
11079 SrcEltVT = MVT::getIntegerVT(BitWidth: SrcEltVT.getSizeInBits() / 2);
11080 MVT ResultVT = ContainerVT.changeVectorElementType(EltVT: SrcEltVT);
11081 Result = DAG.getNode(Opcode: NewOpc, DL, VT: ResultVT, N1: Result, N2: Mask, N3: VL);
11082 } while (SrcEltVT != DstEltVT);
11083
11084 if (SrcVT.isFixedLengthVector())
11085 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
11086
11087 return Result;
11088}
11089
11090SDValue
11091RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(SDValue Op,
11092 SelectionDAG &DAG) const {
11093 SDLoc DL(Op);
11094 SDValue Chain = Op.getOperand(i: 0);
11095 SDValue Src = Op.getOperand(i: 1);
11096 MVT VT = Op.getSimpleValueType();
11097 MVT SrcVT = Src.getSimpleValueType();
11098 MVT ContainerVT = VT;
11099 if (VT.isFixedLengthVector()) {
11100 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
11101 ContainerVT =
11102 SrcContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType());
11103 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
11104 }
11105
11106 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
11107
11108 // RVV can only widen/truncate fp to types double/half the size as the source.
11109 if ((VT.getVectorElementType() == MVT::f64 &&
11110 (SrcVT.getVectorElementType() == MVT::f16 ||
11111 SrcVT.getVectorElementType() == MVT::bf16)) ||
11112 ((VT.getVectorElementType() == MVT::f16 ||
11113 VT.getVectorElementType() == MVT::bf16) &&
11114 SrcVT.getVectorElementType() == MVT::f64)) {
11115 // For double rounding, the intermediate rounding should be round-to-odd.
11116 unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
11117 ? RISCVISD::STRICT_FP_EXTEND_VL
11118 : RISCVISD::STRICT_VFNCVT_ROD_VL;
11119 MVT InterVT = ContainerVT.changeVectorElementType(EltVT: MVT::f32);
11120 Src = DAG.getNode(Opcode: InterConvOpc, DL, VTList: DAG.getVTList(VT1: InterVT, VT2: MVT::Other),
11121 N1: Chain, N2: Src, N3: Mask, N4: VL);
11122 Chain = Src.getValue(R: 1);
11123 }
11124
11125 unsigned ConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
11126 ? RISCVISD::STRICT_FP_EXTEND_VL
11127 : RISCVISD::STRICT_FP_ROUND_VL;
11128 SDValue Res = DAG.getNode(Opcode: ConvOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
11129 N1: Chain, N2: Src, N3: Mask, N4: VL);
11130 if (VT.isFixedLengthVector()) {
11131 // StrictFP operations have two result values. Their lowered result should
11132 // have same result count.
11133 SDValue SubVec = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
11134 Res = DAG.getMergeValues(Ops: {SubVec, Res.getValue(R: 1)}, dl: DL);
11135 }
11136 return Res;
11137}
11138
11139SDValue
11140RISCVTargetLowering::lowerVectorFPExtendOrRound(SDValue Op,
11141 SelectionDAG &DAG) const {
11142 bool IsExtend = Op.getOpcode() == ISD::FP_EXTEND;
11143 // RVV can only do truncate fp to types half the size as the source. We
11144 // custom-lower f64->f16 rounds via RVV's round-to-odd float
11145 // conversion instruction.
11146 SDLoc DL(Op);
11147 MVT VT = Op.getSimpleValueType();
11148
11149 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
11150
11151 SDValue Src = Op.getOperand(i: 0);
11152 MVT SrcVT = Src.getSimpleValueType();
11153
11154 bool IsDirectExtend =
11155 IsExtend && (VT.getVectorElementType() != MVT::f64 ||
11156 (SrcVT.getVectorElementType() != MVT::f16 &&
11157 SrcVT.getVectorElementType() != MVT::bf16));
11158 bool IsDirectTrunc = !IsExtend && ((VT.getVectorElementType() != MVT::f16 &&
11159 VT.getVectorElementType() != MVT::bf16) ||
11160 SrcVT.getVectorElementType() != MVT::f64);
11161
11162 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
11163
11164 // We have regular SD node patterns for direct non-VL extends.
11165 if (VT.isScalableVector() && IsDirectConv)
11166 return Op;
11167
11168 // Prepare any fixed-length vector operands.
11169 MVT ContainerVT = VT;
11170 if (VT.isFixedLengthVector()) {
11171 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
11172 ContainerVT =
11173 SrcContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType());
11174 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
11175 }
11176
11177 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
11178
11179 unsigned ConvOpc = IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::FP_ROUND_VL;
11180
11181 if (IsDirectConv) {
11182 Src = DAG.getNode(Opcode: ConvOpc, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
11183 if (VT.isFixedLengthVector())
11184 Src = convertFromScalableVector(VT, V: Src, DAG, Subtarget);
11185 return Src;
11186 }
11187
11188 unsigned InterConvOpc =
11189 IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::VFNCVT_ROD_VL;
11190
11191 MVT InterVT = ContainerVT.changeVectorElementType(EltVT: MVT::f32);
11192 SDValue IntermediateConv =
11193 DAG.getNode(Opcode: InterConvOpc, DL, VT: InterVT, N1: Src, N2: Mask, N3: VL);
11194 SDValue Result =
11195 DAG.getNode(Opcode: ConvOpc, DL, VT: ContainerVT, N1: IntermediateConv, N2: Mask, N3: VL);
11196 if (VT.isFixedLengthVector())
11197 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
11198 return Result;
11199}
11200
11201// Given a scalable vector type and an index into it, returns the type for the
11202// smallest subvector that the index fits in. This can be used to reduce LMUL
11203// for operations like vslidedown.
11204//
11205// E.g. With Zvl128b, index 3 in a nxv4i32 fits within the first nxv2i32.
11206static std::optional<MVT>
11207getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG,
11208 const RISCVSubtarget &Subtarget) {
11209 assert(VecVT.isScalableVector());
11210 const unsigned EltSize = VecVT.getScalarSizeInBits();
11211 const unsigned VectorBitsMin = Subtarget.getRealMinVLen();
11212 const unsigned MinVLMAX = VectorBitsMin / EltSize;
11213 MVT SmallerVT;
11214 if (MaxIdx < MinVLMAX)
11215 SmallerVT = RISCVTargetLowering::getM1VT(VT: VecVT);
11216 else if (MaxIdx < MinVLMAX * 2)
11217 SmallerVT =
11218 RISCVTargetLowering::getM1VT(VT: VecVT).getDoubleNumVectorElementsVT();
11219 else if (MaxIdx < MinVLMAX * 4)
11220 SmallerVT = RISCVTargetLowering::getM1VT(VT: VecVT)
11221 .getDoubleNumVectorElementsVT()
11222 .getDoubleNumVectorElementsVT();
11223 if (!SmallerVT.isValid() || !VecVT.bitsGT(VT: SmallerVT))
11224 return std::nullopt;
11225 return SmallerVT;
11226}
11227
11228// Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
11229// first position of a vector, and that vector is slid up to the insert index.
11230// By limiting the active vector length to index+1 and merging with the
11231// original vector (with an undisturbed tail policy for elements >= VL), we
11232// achieve the desired result of leaving all elements untouched except the one
11233// at VL-1, which is replaced with the desired value.
11234SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
11235 SelectionDAG &DAG) const {
11236 SDLoc DL(Op);
11237 MVT VecVT = Op.getSimpleValueType();
11238 MVT XLenVT = Subtarget.getXLenVT();
11239 SDValue Vec = Op.getOperand(i: 0);
11240 SDValue Val = Op.getOperand(i: 1);
11241 MVT ValVT = Val.getSimpleValueType();
11242 SDValue Idx = Op.getOperand(i: 2);
11243
11244 if (VecVT.getVectorElementType() == MVT::i1) {
11245 // FIXME: For now we just promote to an i8 vector and insert into that,
11246 // but this is probably not optimal.
11247 MVT WideVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
11248 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Vec);
11249 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: WideVT, N1: Vec, N2: Val, N3: Idx);
11250 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VecVT, Operand: Vec);
11251 }
11252
11253 if ((ValVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
11254 (ValVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
11255 // If we don't have vfmv.s.f for f16/bf16, use fmv.x.h first.
11256 MVT IntVT = VecVT.changeTypeToInteger();
11257 SDValue IntInsert = DAG.getNode(
11258 Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVT, N1: DAG.getBitcast(VT: IntVT, V: Vec),
11259 N2: DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Val), N3: Idx);
11260 return DAG.getBitcast(VT: VecVT, V: IntInsert);
11261 }
11262
11263 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
11264 auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
11265 if (!IdxC)
11266 return SDValue();
11267
11268 unsigned IdxVal = IdxC->getZExtValue();
11269 unsigned NumElts = VecVT.getVectorNumElements();
11270 MVT EltVT = VecVT.getVectorElementType();
11271
11272 if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
11273 unsigned HalfNumElts = NumElts / 2;
11274 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
11275 MVT HalfVT = Lo.getSimpleValueType();
11276 if (IdxVal < HalfNumElts) {
11277 SDValue NewLo =
11278 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: HalfVT, N1: Lo, N2: Val, N3: Idx);
11279 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: NewLo, N2: Hi);
11280 }
11281 SDValue NewHi =
11282 DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: HalfVT, N1: Hi, N2: Val,
11283 N3: DAG.getVectorIdxConstant(Val: IdxVal - HalfNumElts, DL));
11284 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo, N2: NewHi);
11285 }
11286
11287 Vec = DAG.getBitcast(VT: XLenVT, V: Vec);
11288 SDValue ExtVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Val);
11289
11290 // For 2-element vectors, BUILD_VECTOR is more efficient since it only needs
11291 // at most 2 instructions.
11292 if (NumElts == 2) {
11293 unsigned EltBits = EltVT.getSizeInBits();
11294 SDValue Elt0, Elt1;
11295 if (IdxVal == 0) {
11296 Elt0 = ExtVal;
11297 Elt1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Vec,
11298 N2: DAG.getConstant(Val: EltBits, DL, VT: XLenVT));
11299 } else {
11300 Elt0 = Vec;
11301 Elt1 = ExtVal;
11302 }
11303 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: VecVT, N1: Elt0, N2: Elt1);
11304 }
11305
11306 // For 4/8-element vectors, use MVM(or MERGE) instruction which does bitwise
11307 // select: rd = (~mask & rd) | (mask & rs1).
11308 // This generates: slli + lui/li + mvm
11309 if (NumElts == 4 || NumElts == 8) {
11310 unsigned EltBits = EltVT.getSizeInBits();
11311 unsigned ShiftAmt = IdxVal * EltBits;
11312 uint64_t PosMask = ((1ULL << EltBits) - 1) << ShiftAmt;
11313
11314 SDValue ShiftedVal = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: ExtVal,
11315 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: XLenVT));
11316 SDValue Mask = DAG.getConstant(Val: PosMask, DL, VT: XLenVT);
11317 SDValue Result =
11318 DAG.getNode(Opcode: RISCVISD::MERGE, DL, VT: XLenVT, N1: Mask, N2: Vec, N3: ShiftedVal);
11319 return DAG.getBitcast(VT: VecVT, V: Result);
11320 }
11321
11322 return SDValue();
11323 }
11324
11325 MVT ContainerVT = VecVT;
11326 // If the operand is a fixed-length vector, convert to a scalable one.
11327 if (VecVT.isFixedLengthVector()) {
11328 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11329 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11330 }
11331
11332 // If we know the index we're going to insert at, we can shrink Vec so that
11333 // we're performing the scalar inserts and slideup on a smaller LMUL.
11334 SDValue OrigVec = Vec;
11335 std::optional<unsigned> AlignedIdx;
11336 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx)) {
11337 const unsigned OrigIdx = IdxC->getZExtValue();
11338 // Do we know an upper bound on LMUL?
11339 if (auto ShrunkVT = getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: OrigIdx,
11340 DL, DAG, Subtarget)) {
11341 ContainerVT = *ShrunkVT;
11342 AlignedIdx = 0;
11343 }
11344
11345 // If we're compiling for an exact VLEN value, we can always perform
11346 // the insert in m1 as we can determine the register corresponding to
11347 // the index in the register group.
11348 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
11349 if (auto VLEN = Subtarget.getRealVLen(); VLEN && ContainerVT.bitsGT(VT: M1VT)) {
11350 EVT ElemVT = VecVT.getVectorElementType();
11351 unsigned ElemsPerVReg = *VLEN / ElemVT.getFixedSizeInBits();
11352 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11353 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11354 AlignedIdx = SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11355 Idx = DAG.getVectorIdxConstant(Val: RemIdx, DL);
11356 ContainerVT = M1VT;
11357 }
11358
11359 if (AlignedIdx)
11360 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: *AlignedIdx);
11361 }
11362
11363 bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
11364 // Even i64-element vectors on RV32 can be lowered without scalar
11365 // legalization if the most-significant 32 bits of the value are not affected
11366 // by the sign-extension of the lower 32 bits. This applies to i32 constants
11367 // and sign_extend of i32 values.
11368 if (!IsLegalInsert) {
11369 if (isa<ConstantSDNode>(Val)) {
11370 const auto *CVal = cast<ConstantSDNode>(Val);
11371 if (isInt<32>(x: CVal->getSExtValue())) {
11372 IsLegalInsert = true;
11373 Val = DAG.getSignedConstant(Val: CVal->getSExtValue(), DL, VT: MVT::i32);
11374 }
11375 } else if (Val.getOpcode() == ISD::SIGN_EXTEND &&
11376 Val.getOperand(i: 0).getValueType() == MVT::i32) {
11377 IsLegalInsert = true;
11378 Val = Val.getOperand(i: 0);
11379 }
11380 }
11381
11382 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11383
11384 SDValue ValInVec;
11385
11386 if (IsLegalInsert) {
11387 unsigned Opc =
11388 VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
11389 if (isNullConstant(V: Idx)) {
11390 if (!VecVT.isFloatingPoint())
11391 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Val);
11392 Vec = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: Vec, N2: Val, N3: VL);
11393
11394 if (AlignedIdx)
11395 Vec = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: Vec, Idx: *AlignedIdx);
11396 if (!VecVT.isFixedLengthVector())
11397 return Vec;
11398 return convertFromScalableVector(VT: VecVT, V: Vec, DAG, Subtarget);
11399 }
11400
11401 ValInVec = lowerScalarInsert(Scalar: Val, VL, VT: ContainerVT, DL, DAG, Subtarget);
11402 } else {
11403 // On RV32, i64-element vectors must be specially handled to place the
11404 // value at element 0, by using two vslide1down instructions in sequence on
11405 // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
11406 // this.
11407 SDValue ValLo, ValHi;
11408 std::tie(args&: ValLo, args&: ValHi) = DAG.SplitScalar(N: Val, DL, LoVT: MVT::i32, HiVT: MVT::i32);
11409 MVT I32ContainerVT =
11410 MVT::getVectorVT(VT: MVT::i32, EC: ContainerVT.getVectorElementCount() * 2);
11411 SDValue I32Mask =
11412 getDefaultScalableVLOps(VecVT: I32ContainerVT, DL, DAG, Subtarget).first;
11413 // Limit the active VL to two.
11414 SDValue InsertI64VL = DAG.getConstant(Val: 2, DL, VT: XLenVT);
11415 // If the Idx is 0 we can insert directly into the vector.
11416 if (isNullConstant(V: Idx)) {
11417 // First slide in the lo value, then the hi in above it. We use slide1down
11418 // to avoid the register group overlap constraint of vslide1up.
11419 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11420 N1: Vec, N2: Vec, N3: ValLo, N4: I32Mask, N5: InsertI64VL);
11421 // If the source vector is undef don't pass along the tail elements from
11422 // the previous slide1down.
11423 SDValue Tail = Vec.isUndef() ? Vec : ValInVec;
11424 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11425 N1: Tail, N2: ValInVec, N3: ValHi, N4: I32Mask, N5: InsertI64VL);
11426 // Bitcast back to the right container type.
11427 ValInVec = DAG.getBitcast(VT: ContainerVT, V: ValInVec);
11428
11429 if (AlignedIdx)
11430 ValInVec = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: ValInVec, Idx: *AlignedIdx);
11431 if (!VecVT.isFixedLengthVector())
11432 return ValInVec;
11433 return convertFromScalableVector(VT: VecVT, V: ValInVec, DAG, Subtarget);
11434 }
11435
11436 // First slide in the lo value, then the hi in above it. We use slide1down
11437 // to avoid the register group overlap constraint of vslide1up.
11438 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11439 N1: DAG.getUNDEF(VT: I32ContainerVT),
11440 N2: DAG.getUNDEF(VT: I32ContainerVT), N3: ValLo,
11441 N4: I32Mask, N5: InsertI64VL);
11442 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11443 N1: DAG.getUNDEF(VT: I32ContainerVT), N2: ValInVec, N3: ValHi,
11444 N4: I32Mask, N5: InsertI64VL);
11445 // Bitcast back to the right container type.
11446 ValInVec = DAG.getBitcast(VT: ContainerVT, V: ValInVec);
11447 }
11448
11449 // Now that the value is in a vector, slide it into position.
11450 SDValue InsertVL =
11451 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: Idx, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11452
11453 // Use tail agnostic policy if Idx is the last index of Vec.
11454 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
11455 if (VecVT.isFixedLengthVector() && isa<ConstantSDNode>(Val: Idx) &&
11456 Idx->getAsZExtVal() + 1 == VecVT.getVectorNumElements())
11457 Policy = RISCVVType::TAIL_AGNOSTIC;
11458 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Vec, Op: ValInVec,
11459 Offset: Idx, Mask, VL: InsertVL, Policy);
11460
11461 if (AlignedIdx)
11462 Slideup = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: Slideup, Idx: *AlignedIdx);
11463 if (!VecVT.isFixedLengthVector())
11464 return Slideup;
11465 return convertFromScalableVector(VT: VecVT, V: Slideup, DAG, Subtarget);
11466}
11467
11468// Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
11469// extract the first element: (extractelt (slidedown vec, idx), 0). For integer
11470// types this is done using VMV_X_S to allow us to glean information about the
11471// sign bits of the result.
11472SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
11473 SelectionDAG &DAG) const {
11474 SDLoc DL(Op);
11475 SDValue Idx = Op.getOperand(i: 1);
11476 SDValue Vec = Op.getOperand(i: 0);
11477 EVT EltVT = Op.getValueType();
11478 MVT VecVT = Vec.getSimpleValueType();
11479 MVT XLenVT = Subtarget.getXLenVT();
11480
11481 if (VecVT.getVectorElementType() == MVT::i1) {
11482 // Use vfirst.m to extract the first bit.
11483 if (isNullConstant(V: Idx)) {
11484 MVT ContainerVT = VecVT;
11485 if (VecVT.isFixedLengthVector()) {
11486 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11487 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11488 }
11489 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11490 SDValue Vfirst =
11491 DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
11492 SDValue Res = DAG.getSetCC(DL, VT: XLenVT, LHS: Vfirst,
11493 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
11494 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
11495 }
11496 if (VecVT.isFixedLengthVector()) {
11497 unsigned NumElts = VecVT.getVectorNumElements();
11498 if (NumElts >= 8) {
11499 MVT WideEltVT;
11500 unsigned WidenVecLen;
11501 SDValue ExtractElementIdx;
11502 SDValue ExtractBitIdx;
11503 unsigned MaxEEW = Subtarget.getELen();
11504 MVT LargestEltVT = MVT::getIntegerVT(
11505 BitWidth: std::min(a: MaxEEW, b: unsigned(XLenVT.getSizeInBits())));
11506 if (NumElts <= LargestEltVT.getSizeInBits()) {
11507 assert(isPowerOf2_32(NumElts) &&
11508 "the number of elements should be power of 2");
11509 WideEltVT = MVT::getIntegerVT(BitWidth: NumElts);
11510 WidenVecLen = 1;
11511 ExtractElementIdx = DAG.getConstant(Val: 0, DL, VT: XLenVT);
11512 ExtractBitIdx = Idx;
11513 } else {
11514 WideEltVT = LargestEltVT;
11515 WidenVecLen = NumElts / WideEltVT.getSizeInBits();
11516 // extract element index = index / element width
11517 ExtractElementIdx = DAG.getNode(
11518 Opcode: ISD::SRL, DL, VT: XLenVT, N1: Idx,
11519 N2: DAG.getConstant(Val: Log2_64(Value: WideEltVT.getSizeInBits()), DL, VT: XLenVT));
11520 // mask bit index = index % element width
11521 ExtractBitIdx = DAG.getNode(
11522 Opcode: ISD::AND, DL, VT: XLenVT, N1: Idx,
11523 N2: DAG.getConstant(Val: WideEltVT.getSizeInBits() - 1, DL, VT: XLenVT));
11524 }
11525 MVT WideVT = MVT::getVectorVT(VT: WideEltVT, NumElements: WidenVecLen);
11526 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Vec);
11527 SDValue ExtractElt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: XLenVT,
11528 N1: Vec, N2: ExtractElementIdx);
11529 // Extract the bit from GPR.
11530 SDValue ShiftRight =
11531 DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: ExtractElt, N2: ExtractBitIdx);
11532 SDValue Res = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: ShiftRight,
11533 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11534 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
11535 }
11536 }
11537 // Otherwise, promote to an i8 vector and extract from that.
11538 MVT WideVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
11539 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Vec);
11540 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Vec, N2: Idx);
11541 }
11542
11543 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
11544 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
11545 // If we don't have vfmv.f.s for f16/bf16, extract to a gpr then use fmv.h.x
11546 MVT IntVT = VecVT.changeTypeToInteger();
11547 SDValue IntVec = DAG.getBitcast(VT: IntVT, V: Vec);
11548 SDValue IntExtract =
11549 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: XLenVT, N1: IntVec, N2: Idx);
11550 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT: EltVT, Operand: IntExtract);
11551 }
11552
11553 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
11554 if (VecVT != MVT::v4i16 && VecVT != MVT::v2i16 && VecVT != MVT::v8i8 &&
11555 VecVT != MVT::v4i8 && VecVT != MVT::v2i32)
11556 return SDValue();
11557
11558 if (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)) {
11559 auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
11560 if (!IdxC)
11561 return SDValue();
11562 unsigned IdxVal = IdxC->getZExtValue();
11563 unsigned HalfNumElts = VecVT.getVectorNumElements() / 2;
11564 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
11565 if (IdxVal < HalfNumElts)
11566 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Lo, N2: Idx);
11567 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Hi,
11568 N2: DAG.getVectorIdxConstant(Val: IdxVal - HalfNumElts, DL));
11569 }
11570
11571 SDValue Extracted = DAG.getBitcast(VT: XLenVT, V: Vec);
11572 unsigned ElemWidth = VecVT.getVectorElementType().getSizeInBits();
11573 SDValue Shamt = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Idx,
11574 N2: DAG.getConstant(Val: ElemWidth, DL, VT: XLenVT));
11575 return DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Extracted, N2: Shamt);
11576 }
11577
11578 // If this is a fixed vector, we need to convert it to a scalable vector.
11579 MVT ContainerVT = VecVT;
11580 if (VecVT.isFixedLengthVector()) {
11581 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11582 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11583 }
11584
11585 // If we're compiling for an exact VLEN value and we have a known
11586 // constant index, we can always perform the extract in m1 (or
11587 // smaller) as we can determine the register corresponding to
11588 // the index in the register group.
11589 const auto VLen = Subtarget.getRealVLen();
11590 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
11591 IdxC && VLen && VecVT.getSizeInBits().getKnownMinValue() > *VLen) {
11592 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
11593 unsigned OrigIdx = IdxC->getZExtValue();
11594 EVT ElemVT = VecVT.getVectorElementType();
11595 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
11596 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11597 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11598 unsigned ExtractIdx =
11599 SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11600 Vec = DAG.getExtractSubvector(DL, VT: M1VT, Vec, Idx: ExtractIdx);
11601 Idx = DAG.getVectorIdxConstant(Val: RemIdx, DL);
11602 ContainerVT = M1VT;
11603 }
11604
11605 // Reduce the LMUL of our slidedown and vmv.x.s to the smallest LMUL which
11606 // contains our index.
11607 std::optional<uint64_t> MaxIdx;
11608 if (VecVT.isFixedLengthVector())
11609 MaxIdx = VecVT.getVectorNumElements() - 1;
11610 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx))
11611 MaxIdx = IdxC->getZExtValue();
11612 if (MaxIdx) {
11613 if (auto SmallerVT =
11614 getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: *MaxIdx, DL, DAG, Subtarget)) {
11615 ContainerVT = *SmallerVT;
11616 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: 0);
11617 }
11618 }
11619
11620 // If after narrowing, the required slide is still greater than LMUL2,
11621 // fallback to generic expansion and go through the stack. This is done
11622 // for a subtle reason: extracting *all* elements out of a vector is
11623 // widely expected to be linear in vector size, but because vslidedown
11624 // is linear in LMUL, performing N extracts using vslidedown becomes
11625 // O(n^2) / (VLEN/ETYPE) work. On the surface, going through the stack
11626 // seems to have the same problem (the store is linear in LMUL), but the
11627 // generic expansion *memoizes* the store, and thus for many extracts of
11628 // the same vector we end up with one store and a bunch of loads.
11629 // TODO: We don't have the same code for insert_vector_elt because we
11630 // have BUILD_VECTOR and handle the degenerate case there. Should we
11631 // consider adding an inverse BUILD_VECTOR node?
11632 MVT LMUL2VT =
11633 RISCVTargetLowering::getM1VT(VT: ContainerVT).getDoubleNumVectorElementsVT();
11634 if (ContainerVT.bitsGT(VT: LMUL2VT) && VecVT.isFixedLengthVector())
11635 return SDValue();
11636
11637 // If the index is 0, the vector is already in the right position.
11638 if (!isNullConstant(V: Idx)) {
11639 // Use a VL of 1 to avoid processing more elements than we need.
11640 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT, DL, DAG, Subtarget);
11641 Vec = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
11642 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: Idx, Mask, VL);
11643 }
11644
11645 if (!EltVT.isInteger()) {
11646 // Floating-point extracts are handled in TableGen.
11647 return DAG.getExtractVectorElt(DL, VT: EltVT, Vec, Idx: 0);
11648 }
11649
11650 SDValue Elt0 = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
11651 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Elt0);
11652}
11653
11654// Some RVV intrinsics may claim that they want an integer operand to be
11655// promoted or expanded.
11656static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
11657 const RISCVSubtarget &Subtarget) {
11658 assert((Op.getOpcode() == ISD::INTRINSIC_VOID ||
11659 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
11660 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
11661 "Unexpected opcode");
11662
11663 if (!Subtarget.hasVInstructions())
11664 return SDValue();
11665
11666 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
11667 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
11668 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
11669
11670 SDLoc DL(Op);
11671
11672 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
11673 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
11674 if (!II || !II->hasScalarOperand())
11675 return SDValue();
11676
11677 unsigned SplatOp = II->ScalarOperand + 1 + HasChain;
11678 assert(SplatOp < Op.getNumOperands());
11679
11680 SmallVector<SDValue, 8> Operands(Op->ops());
11681 SDValue &ScalarOp = Operands[SplatOp];
11682 MVT OpVT = ScalarOp.getSimpleValueType();
11683 MVT XLenVT = Subtarget.getXLenVT();
11684
11685 // If this isn't a scalar, or its type is XLenVT we're done.
11686 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
11687 return SDValue();
11688
11689 // Simplest case is that the operand needs to be promoted to XLenVT.
11690 if (OpVT.bitsLT(VT: XLenVT)) {
11691 // If the operand is a constant, sign extend to increase our chances
11692 // of being able to use a .vi instruction. ANY_EXTEND would become a
11693 // a zero extend and the simm5 check in isel would fail.
11694 // FIXME: Should we ignore the upper bits in isel instead?
11695 unsigned ExtOpc =
11696 isa<ConstantSDNode>(Val: ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
11697 ScalarOp = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: ScalarOp);
11698 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
11699 }
11700
11701 // Use the previous operand to get the vXi64 VT. The result might be a mask
11702 // VT for compares. Using the previous operand assumes that the previous
11703 // operand will never have a smaller element size than a scalar operand and
11704 // that a widening operation never uses SEW=64.
11705 // NOTE: If this fails the below assert, we can probably just find the
11706 // element count from any operand or result and use it to construct the VT.
11707 assert(II->ScalarOperand > 0 && "Unexpected splat operand!");
11708 MVT VT = Op.getOperand(i: SplatOp - 1).getSimpleValueType();
11709
11710 // The more complex case is when the scalar is larger than XLenVT.
11711 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
11712 VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
11713
11714 // If this is a sign-extended 32-bit value, we can truncate it and rely on the
11715 // instruction to sign-extend since SEW>XLEN.
11716 if (DAG.ComputeNumSignBits(Op: ScalarOp) > 32) {
11717 ScalarOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: ScalarOp);
11718 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
11719 }
11720
11721 switch (IntNo) {
11722 case Intrinsic::riscv_vslide1up:
11723 case Intrinsic::riscv_vslide1down:
11724 case Intrinsic::riscv_vslide1up_mask:
11725 case Intrinsic::riscv_vslide1down_mask: {
11726 // We need to special case these when the scalar is larger than XLen.
11727 unsigned NumOps = Op.getNumOperands();
11728 bool IsMasked = NumOps == 7;
11729
11730 // Convert the vector source to the equivalent nxvXi32 vector.
11731 MVT I32VT = MVT::getVectorVT(VT: MVT::i32, EC: VT.getVectorElementCount() * 2);
11732 SDValue Vec = DAG.getBitcast(VT: I32VT, V: Operands[2]);
11733 SDValue ScalarLo, ScalarHi;
11734 std::tie(args&: ScalarLo, args&: ScalarHi) =
11735 DAG.SplitScalar(N: ScalarOp, DL, LoVT: MVT::i32, HiVT: MVT::i32);
11736
11737 // Double the VL since we halved SEW.
11738 SDValue AVL = getVLOperand(Op);
11739 SDValue I32VL;
11740
11741 // Optimize for constant AVL
11742 if (isa<ConstantSDNode>(Val: AVL)) {
11743 const auto [MinVLMAX, MaxVLMAX] =
11744 RISCVTargetLowering::computeVLMAXBounds(VecVT: VT, Subtarget);
11745
11746 uint64_t AVLInt = AVL->getAsZExtVal();
11747 if (AVLInt <= MinVLMAX) {
11748 I32VL = DAG.getConstant(Val: 2 * AVLInt, DL, VT: XLenVT);
11749 } else if (AVLInt >= 2 * MaxVLMAX) {
11750 // Just set vl to VLMAX in this situation
11751 I32VL = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
11752 } else {
11753 // For AVL between (MinVLMAX, 2 * MaxVLMAX), the actual working vl
11754 // is related to the hardware implementation.
11755 // So let the following code handle
11756 }
11757 }
11758 if (!I32VL) {
11759 RISCVVType::VLMUL Lmul = RISCVTargetLowering::getLMUL(VT);
11760 SDValue LMUL = DAG.getConstant(Val: Lmul, DL, VT: XLenVT);
11761 unsigned Sew = RISCVVType::encodeSEW(SEW: VT.getScalarSizeInBits());
11762 SDValue SEW = DAG.getConstant(Val: Sew, DL, VT: XLenVT);
11763 SDValue SETVL =
11764 DAG.getTargetConstant(Val: Intrinsic::riscv_vsetvli, DL, VT: MVT::i32);
11765 // Using vsetvli instruction to get actually used length which related to
11766 // the hardware implementation
11767 SDValue VL = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: XLenVT, N1: SETVL, N2: AVL,
11768 N3: SEW, N4: LMUL);
11769 I32VL =
11770 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VL, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11771 }
11772
11773 SDValue I32Mask = getAllOnesMask(VecVT: I32VT, VL: I32VL, DL, DAG);
11774
11775 // Shift the two scalar parts in using SEW=32 slide1up/slide1down
11776 // instructions.
11777 SDValue Passthru;
11778 if (IsMasked)
11779 Passthru = DAG.getUNDEF(VT: I32VT);
11780 else
11781 Passthru = DAG.getBitcast(VT: I32VT, V: Operands[1]);
11782
11783 if (IntNo == Intrinsic::riscv_vslide1up ||
11784 IntNo == Intrinsic::riscv_vslide1up_mask) {
11785 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1UP_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11786 N3: ScalarHi, N4: I32Mask, N5: I32VL);
11787 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1UP_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11788 N3: ScalarLo, N4: I32Mask, N5: I32VL);
11789 } else {
11790 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11791 N3: ScalarLo, N4: I32Mask, N5: I32VL);
11792 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11793 N3: ScalarHi, N4: I32Mask, N5: I32VL);
11794 }
11795
11796 // Convert back to nxvXi64.
11797 Vec = DAG.getBitcast(VT, V: Vec);
11798
11799 if (!IsMasked)
11800 return Vec;
11801 // Apply mask after the operation.
11802 SDValue Mask = Operands[NumOps - 3];
11803 SDValue MaskedOff = Operands[1];
11804 // Assume Policy operand is the last operand.
11805 uint64_t Policy = Operands[NumOps - 1]->getAsZExtVal();
11806 // We don't need to select maskedoff if it's undef.
11807 if (MaskedOff.isUndef())
11808 return Vec;
11809 // TAMU
11810 if (Policy == RISCVVType::TAIL_AGNOSTIC)
11811 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: Mask, N2: Vec, N3: MaskedOff,
11812 N4: DAG.getUNDEF(VT), N5: AVL);
11813 // TUMA or TUMU: Currently we always emit tumu policy regardless of tuma.
11814 // It's fine because vmerge does not care mask policy.
11815 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: Mask, N2: Vec, N3: MaskedOff,
11816 N4: MaskedOff, N5: AVL);
11817 }
11818 }
11819
11820 // We need to convert the scalar to a splat vector.
11821 SDValue VL = getVLOperand(Op);
11822 assert(VL.getValueType() == XLenVT);
11823 ScalarOp = splatSplitI64WithVL(DL, VT, Passthru: SDValue(), Scalar: ScalarOp, VL, DAG);
11824 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
11825}
11826
11827// Lower the llvm.get.vector.length intrinsic to vsetvli. We only support
11828// scalable vector llvm.get.vector.length for now.
11829//
11830// We need to convert from a scalable VF to a vsetvli with VLMax equal to
11831// (vscale * VF). The vscale and VF are independent of element width. We use
11832// SEW=8 for the vsetvli because it is the only element width that supports all
11833// fractional LMULs. The LMUL is chosen so that with SEW=8 the VLMax is
11834// (vscale * VF). Where vscale is defined as VLEN/RVVBitsPerBlock. The
11835// InsertVSETVLI pass can fix up the vtype of the vsetvli if a different
11836// SEW and LMUL are better for the surrounding vector instructions.
11837static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
11838 const RISCVSubtarget &Subtarget) {
11839 MVT XLenVT = Subtarget.getXLenVT();
11840
11841 // The smallest LMUL is only valid for the smallest element width.
11842 const unsigned ElementWidth = 8;
11843
11844 // Determine the VF that corresponds to LMUL 1 for ElementWidth.
11845 unsigned LMul1VF = RISCV::RVVBitsPerBlock / ElementWidth;
11846 // We don't support VF==1 with ELEN==32.
11847 [[maybe_unused]] unsigned MinVF =
11848 RISCV::RVVBitsPerBlock / Subtarget.getELen();
11849
11850 [[maybe_unused]] unsigned VF = N->getConstantOperandVal(Num: 2);
11851 assert(VF >= MinVF && VF <= (LMul1VF * 8) && isPowerOf2_32(VF) &&
11852 "Unexpected VF");
11853
11854 bool Fractional = VF < LMul1VF;
11855 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
11856 unsigned VLMUL = (unsigned)RISCVVType::encodeLMUL(LMUL: LMulVal, Fractional);
11857 unsigned VSEW = RISCVVType::encodeSEW(SEW: ElementWidth);
11858
11859 SDLoc DL(N);
11860
11861 SDValue LMul = DAG.getTargetConstant(Val: VLMUL, DL, VT: XLenVT);
11862 SDValue Sew = DAG.getTargetConstant(Val: VSEW, DL, VT: XLenVT);
11863
11864 SDValue AVL = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 1));
11865
11866 SDValue ID = DAG.getTargetConstant(Val: Intrinsic::riscv_vsetvli, DL, VT: XLenVT);
11867 SDValue Res =
11868 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: XLenVT, N1: ID, N2: AVL, N3: Sew, N4: LMul);
11869 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: Res);
11870}
11871
11872static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
11873 const RISCVSubtarget &Subtarget) {
11874 SDValue Op0 = Op.getOperand(i: 0);
11875 MVT OpVT = Op0.getSimpleValueType();
11876 MVT ContainerVT = OpVT;
11877 if (OpVT.isFixedLengthVector()) {
11878 ContainerVT = getContainerForFixedLengthVector(VT: OpVT, Subtarget);
11879 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
11880 }
11881 MVT XLenVT = Subtarget.getXLenVT();
11882 SDLoc DL(Op);
11883 auto [Mask, VL] = getDefaultVLOps(VecVT: OpVT, ContainerVT, DL, DAG, Subtarget);
11884 SDValue Res = DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Op0, N2: Mask, N3: VL);
11885 if (Op.getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
11886 return Res;
11887
11888 // Convert -1 to VL.
11889 SDValue Setcc =
11890 DAG.getSetCC(DL, VT: XLenVT, LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETLT);
11891 VL = DAG.getElementCount(DL, VT: XLenVT, EC: OpVT.getVectorElementCount());
11892 return DAG.getSelect(DL, VT: XLenVT, Cond: Setcc, LHS: VL, RHS: Res);
11893}
11894
11895static inline void promoteVCIXScalar(SDValue Op,
11896 MutableArrayRef<SDValue> Operands,
11897 SelectionDAG &DAG) {
11898 const RISCVSubtarget &Subtarget =
11899 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
11900
11901 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
11902 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
11903 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
11904 SDLoc DL(Op);
11905
11906 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
11907 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
11908 if (!II || !II->hasScalarOperand())
11909 return;
11910
11911 unsigned SplatOp = II->ScalarOperand + 1;
11912 assert(SplatOp < Op.getNumOperands());
11913
11914 SDValue &ScalarOp = Operands[SplatOp];
11915 MVT OpVT = ScalarOp.getSimpleValueType();
11916 MVT XLenVT = Subtarget.getXLenVT();
11917
11918 // The code below is partially copied from lowerVectorIntrinsicScalars.
11919 // If this isn't a scalar, or its type is XLenVT we're done.
11920 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
11921 return;
11922
11923 // Manually emit promote operation for scalar operation.
11924 if (OpVT.bitsLT(VT: XLenVT)) {
11925 unsigned ExtOpc =
11926 isa<ConstantSDNode>(Val: ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
11927 ScalarOp = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: ScalarOp);
11928 }
11929}
11930
11931static void processVCIXOperands(SDValue OrigOp,
11932 MutableArrayRef<SDValue> Operands,
11933 SelectionDAG &DAG) {
11934 promoteVCIXScalar(Op: OrigOp, Operands, DAG);
11935 const RISCVSubtarget &Subtarget =
11936 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
11937 for (SDValue &V : Operands) {
11938 EVT ValType = V.getValueType();
11939 if (ValType.isVector() && ValType.isFloatingPoint()) {
11940 MVT InterimIVT =
11941 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ValType.getScalarSizeInBits()),
11942 EC: ValType.getVectorElementCount());
11943 V = DAG.getBitcast(VT: InterimIVT, V);
11944 }
11945 if (ValType.isFixedLengthVector()) {
11946 MVT OpContainerVT =
11947 getContainerForFixedLengthVector(VT: V.getSimpleValueType(), Subtarget);
11948 V = convertToScalableVector(VT: OpContainerVT, V, DAG, Subtarget);
11949 }
11950 }
11951}
11952
11953// LMUL * VLEN should be greater than or equal to EGS * SEW
11954static inline bool isValidEGW(int EGS, EVT VT,
11955 const RISCVSubtarget &Subtarget) {
11956 return (Subtarget.getRealMinVLen() *
11957 VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >=
11958 EGS * VT.getScalarSizeInBits();
11959}
11960
11961static unsigned getRVPShiftOpcode(Intrinsic::ID IntNo) {
11962 switch (IntNo) {
11963 default:
11964 llvm_unreachable(
11965 "Unexpected RISC-V packed saturating and rounding shift intrinsic");
11966 case Intrinsic::riscv_pssha:
11967 return RISCVISD::PSSHA;
11968 case Intrinsic::riscv_psshar:
11969 return RISCVISD::PSSHAR;
11970 case Intrinsic::riscv_psshl:
11971 return RISCVISD::PSSHL;
11972 case Intrinsic::riscv_psshlr:
11973 return RISCVISD::PSSHLR;
11974 }
11975}
11976
11977SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11978 SelectionDAG &DAG) const {
11979 unsigned IntNo = Op.getConstantOperandVal(i: 0);
11980 SDLoc DL(Op);
11981 MVT XLenVT = Subtarget.getXLenVT();
11982
11983 switch (IntNo) {
11984 default:
11985 break; // Don't custom lower most intrinsics.
11986 case Intrinsic::riscv_tuple_insert: {
11987 SDValue Vec = Op.getOperand(i: 1);
11988 SDValue SubVec = Op.getOperand(i: 2);
11989 SDValue Index = Op.getOperand(i: 3);
11990
11991 return DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: Op.getValueType(), N1: Vec,
11992 N2: SubVec, N3: Index);
11993 }
11994 case Intrinsic::riscv_tuple_extract: {
11995 SDValue Vec = Op.getOperand(i: 1);
11996 SDValue Index = Op.getOperand(i: 2);
11997
11998 return DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: Op.getValueType(), N1: Vec,
11999 N2: Index);
12000 }
12001 case Intrinsic::thread_pointer: {
12002 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
12003 return DAG.getRegister(Reg: RISCV::X4, VT: PtrVT);
12004 }
12005 case Intrinsic::riscv_pas:
12006 case Intrinsic::riscv_psa:
12007 case Intrinsic::riscv_psas:
12008 case Intrinsic::riscv_pssa:
12009 case Intrinsic::riscv_paas:
12010 case Intrinsic::riscv_pasa: {
12011 // v2i32 has no paired instruction on RV32; split into a pair of i32 ops
12012 // with cross-lane operands. The exchange shape is: even result uses
12013 // (S1[0], S2[1]); odd result uses (S1[1], S2[0]).
12014 if (Subtarget.is64Bit() || Op.getSimpleValueType() != MVT::v2i32)
12015 break;
12016
12017 unsigned EvenOpc, OddOpc;
12018 switch (IntNo) {
12019 case Intrinsic::riscv_pas:
12020 EvenOpc = ISD::SUB;
12021 OddOpc = ISD::ADD;
12022 break;
12023 case Intrinsic::riscv_psa:
12024 EvenOpc = ISD::ADD;
12025 OddOpc = ISD::SUB;
12026 break;
12027 case Intrinsic::riscv_psas:
12028 EvenOpc = ISD::SSUBSAT;
12029 OddOpc = ISD::SADDSAT;
12030 break;
12031 case Intrinsic::riscv_pssa:
12032 EvenOpc = ISD::SADDSAT;
12033 OddOpc = ISD::SSUBSAT;
12034 break;
12035 case Intrinsic::riscv_paas:
12036 EvenOpc = RISCVISD::ASUB;
12037 OddOpc = ISD::AVGFLOORS;
12038 break;
12039 case Intrinsic::riscv_pasa:
12040 EvenOpc = ISD::AVGFLOORS;
12041 OddOpc = RISCVISD::ASUB;
12042 break;
12043 default:
12044 llvm_unreachable("Unexpected exchanged add/sub intrinsic");
12045 }
12046
12047 SDValue S1 = Op.getOperand(i: 1);
12048 SDValue S2 = Op.getOperand(i: 2);
12049 SDValue S1Even = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S1, Idx: 0);
12050 SDValue S1Odd = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S1, Idx: 1);
12051 SDValue S2Even = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S2, Idx: 0);
12052 SDValue S2Odd = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: S2, Idx: 1);
12053
12054 SDValue REven = DAG.getNode(Opcode: EvenOpc, DL, VT: MVT::i32, N1: S1Even, N2: S2Odd);
12055 SDValue ROdd = DAG.getNode(Opcode: OddOpc, DL, VT: MVT::i32, N1: S1Odd, N2: S2Even);
12056 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: MVT::v2i32, N1: REven, N2: ROdd);
12057 }
12058 case Intrinsic::riscv_orc_b:
12059 case Intrinsic::riscv_brev8:
12060 case Intrinsic::riscv_sha256sig0:
12061 case Intrinsic::riscv_sha256sig1:
12062 case Intrinsic::riscv_sha256sum0:
12063 case Intrinsic::riscv_sha256sum1:
12064 case Intrinsic::riscv_sm3p0:
12065 case Intrinsic::riscv_sm3p1: {
12066 unsigned Opc;
12067 switch (IntNo) {
12068 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
12069 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
12070 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
12071 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
12072 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
12073 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
12074 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
12075 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
12076 }
12077
12078 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
12079 }
12080 case Intrinsic::riscv_sm4ks:
12081 case Intrinsic::riscv_sm4ed: {
12082 unsigned Opc =
12083 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
12084
12085 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
12086 N3: Op.getOperand(i: 3));
12087 }
12088 case Intrinsic::riscv_zip:
12089 case Intrinsic::riscv_unzip: {
12090 unsigned Opc =
12091 IntNo == Intrinsic::riscv_zip ? RISCVISD::ZIP : RISCVISD::UNZIP;
12092 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
12093 }
12094 case Intrinsic::riscv_mopr:
12095 return DAG.getNode(Opcode: RISCVISD::MOP_R, DL, VT: XLenVT, N1: Op.getOperand(i: 1),
12096 N2: Op.getOperand(i: 2));
12097
12098 case Intrinsic::riscv_moprr: {
12099 return DAG.getNode(Opcode: RISCVISD::MOP_RR, DL, VT: XLenVT, N1: Op.getOperand(i: 1),
12100 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
12101 }
12102 case Intrinsic::riscv_clmulh:
12103 case Intrinsic::riscv_clmulr: {
12104 unsigned Opc = IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
12105 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
12106 }
12107 case Intrinsic::riscv_paadd:
12108 case Intrinsic::riscv_paaddu:
12109 case Intrinsic::riscv_pasub:
12110 case Intrinsic::riscv_pasubu:
12111 case Intrinsic::riscv_pabd:
12112 case Intrinsic::riscv_pabdu:
12113 case Intrinsic::riscv_psabs: {
12114 unsigned Opc;
12115 switch (IntNo) {
12116 case Intrinsic::riscv_paadd:
12117 Opc = ISD::AVGFLOORS;
12118 break;
12119 case Intrinsic::riscv_paaddu:
12120 Opc = ISD::AVGFLOORU;
12121 break;
12122 case Intrinsic::riscv_pasub:
12123 Opc = RISCVISD::ASUB;
12124 break;
12125 case Intrinsic::riscv_pasubu:
12126 Opc = RISCVISD::ASUBU;
12127 break;
12128 case Intrinsic::riscv_pabd:
12129 Opc = ISD::ABDS;
12130 break;
12131 case Intrinsic::riscv_pabdu:
12132 Opc = ISD::ABDU;
12133 break;
12134 case Intrinsic::riscv_psabs:
12135 Opc = RISCVISD::PSABS;
12136 break;
12137 }
12138
12139 if (IntNo == Intrinsic::riscv_psabs)
12140 return DAG.getNode(Opcode: Opc, DL, VT: Op.getValueType(), Operand: Op.getOperand(i: 1));
12141
12142 return DAG.getNode(Opcode: Opc, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 1),
12143 N2: Op.getOperand(i: 2));
12144 }
12145 case Intrinsic::riscv_pssha:
12146 case Intrinsic::riscv_psshar:
12147 case Intrinsic::riscv_psshl:
12148 case Intrinsic::riscv_psshlr: {
12149 SDValue ShAmt = Op.getOperand(i: 2);
12150 ShAmt = DAG.getAnyExtOrTrunc(Op: ShAmt, DL, VT: XLenVT);
12151 return DAG.getNode(Opcode: getRVPShiftOpcode(IntNo), DL, VT: Op.getValueType(),
12152 N1: Op.getOperand(i: 1), N2: ShAmt);
12153 }
12154 case Intrinsic::riscv_pabdsumu:
12155 case Intrinsic::riscv_pabdsumau: {
12156 // On RV32 an i32-result absolute difference sum over a 64-bit (GPRPair)
12157 // source has no paired instruction. Split into two v4i8 halves: reduce the
12158 // low half (folding in rd when accumulating), then accumulate the high half
12159 // into that partial sum.
12160 SDValue Rs1 = Op.getOperand(i: Op.getNumOperands() - 2);
12161 SDValue Rs2 = Op.getOperand(i: Op.getNumOperands() - 1);
12162 if (Subtarget.is64Bit() || Rs1.getSimpleValueType() != MVT::v8i8)
12163 return SDValue();
12164 bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
12165 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
12166 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
12167 SDValue AbdsumuId =
12168 DAG.getTargetConstant(Val: Intrinsic::riscv_pabdsumu, DL, VT: MVT::i32);
12169 SDValue AbdsumauId =
12170 DAG.getTargetConstant(Val: Intrinsic::riscv_pabdsumau, DL, VT: MVT::i32);
12171 SDValue Lo = IsAcc ? DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
12172 N1: AbdsumauId, N2: Op.getOperand(i: 1), N3: Rs1Lo, N4: Rs2Lo)
12173 : DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
12174 N1: AbdsumuId, N2: Rs1Lo, N3: Rs2Lo);
12175 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32, N1: AbdsumauId, N2: Lo,
12176 N3: Rs1Hi, N4: Rs2Hi);
12177 }
12178 case Intrinsic::riscv_pmerge: {
12179 EVT VT = Op.getValueType();
12180 auto buildMerge = [&](SDValue Rs1, SDValue Rs2, SDValue Mask,
12181 EVT ResultVT) {
12182 MVT IntVT = MVT::getIntegerVT(BitWidth: ResultVT.getSizeInBits());
12183 SDValue Res =
12184 DAG.getNode(Opcode: RISCVISD::MERGE, DL, VT: IntVT, N1: DAG.getBitcast(VT: IntVT, V: Mask),
12185 N2: DAG.getBitcast(VT: IntVT, V: Rs1), N3: DAG.getBitcast(VT: IntVT, V: Rs2));
12186 return DAG.getBitcast(VT: ResultVT, V: Res);
12187 };
12188
12189 // 64-bit packed types on RV32: split into two 32-bit halves. v2i32 has no
12190 // legal 32-bit vector half, so bitcast it to v4i16 (same 64 bits) first;
12191 // the merge result is identical.
12192 if (!Subtarget.is64Bit() &&
12193 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32)) {
12194 EVT WorkVT = VT == MVT::v2i32 ? EVT(MVT::v4i16) : VT;
12195 SDValue Rs1 = DAG.getBitcast(VT: WorkVT, V: Op.getOperand(i: 1));
12196 SDValue Rs2 = DAG.getBitcast(VT: WorkVT, V: Op.getOperand(i: 2));
12197 SDValue Mask = DAG.getBitcast(VT: WorkVT, V: Op.getOperand(i: 3));
12198 MVT HalfVT = WorkVT == MVT::v8i8 ? MVT::v4i8 : MVT::v2i16;
12199 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL, LoVT: HalfVT, HiVT: HalfVT);
12200 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL, LoVT: HalfVT, HiVT: HalfVT);
12201 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Mask, DL, LoVT: HalfVT, HiVT: HalfVT);
12202 SDValue ResLo = buildMerge(Rs1Lo, Rs2Lo, MaskLo, HalfVT);
12203 SDValue ResHi = buildMerge(Rs1Hi, Rs2Hi, MaskHi, HalfVT);
12204 SDValue Res = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WorkVT, N1: ResLo, N2: ResHi);
12205 return DAG.getBitcast(VT, V: Res);
12206 }
12207
12208 return buildMerge(Op.getOperand(i: 1), Op.getOperand(i: 2), Op.getOperand(i: 3), VT);
12209 }
12210 case Intrinsic::experimental_get_vector_length:
12211 return lowerGetVectorLength(N: Op.getNode(), DAG, Subtarget);
12212 case Intrinsic::riscv_vmv_x_s: {
12213 SDValue Res = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
12214 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: Res);
12215 }
12216 case Intrinsic::riscv_vfmv_f_s:
12217 return DAG.getExtractVectorElt(DL, VT: Op.getValueType(), Vec: Op.getOperand(i: 1), Idx: 0);
12218 case Intrinsic::riscv_vmv_v_x:
12219 return lowerScalarSplat(Passthru: Op.getOperand(i: 1), Scalar: Op.getOperand(i: 2),
12220 VL: Op.getOperand(i: 3), VT: Op.getSimpleValueType(), DL, DAG,
12221 Subtarget);
12222 case Intrinsic::riscv_vfmv_v_f:
12223 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: Op.getValueType(),
12224 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
12225 case Intrinsic::riscv_vmv_s_x: {
12226 SDValue Scalar = Op.getOperand(i: 2);
12227
12228 if (Scalar.getValueType().bitsLE(VT: XLenVT)) {
12229 Scalar = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Scalar);
12230 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT: Op.getValueType(),
12231 N1: Op.getOperand(i: 1), N2: Scalar, N3: Op.getOperand(i: 3));
12232 }
12233
12234 assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
12235
12236 // This is an i64 value that lives in two scalar registers. We have to
12237 // insert this in a convoluted way. First we build vXi64 splat containing
12238 // the two values that we assemble using some bit math. Next we'll use
12239 // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
12240 // to merge element 0 from our splat into the source vector.
12241 // FIXME: This is probably not the best way to do this, but it is
12242 // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
12243 // point.
12244 // sw lo, (a0)
12245 // sw hi, 4(a0)
12246 // vlse vX, (a0)
12247 //
12248 // vid.v vVid
12249 // vmseq.vx mMask, vVid, 0
12250 // vmerge.vvm vDest, vSrc, vVal, mMask
12251 MVT VT = Op.getSimpleValueType();
12252 SDValue Vec = Op.getOperand(i: 1);
12253 SDValue VL = getVLOperand(Op);
12254
12255 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Passthru: SDValue(), Scalar, VL, DAG);
12256 if (Op.getOperand(i: 1).isUndef())
12257 return SplattedVal;
12258 SDValue SplattedIdx =
12259 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
12260 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N3: VL);
12261
12262 MVT MaskVT = getMaskTypeFor(VecVT: VT);
12263 SDValue Mask = getAllOnesMask(VecVT: VT, VL, DL, DAG);
12264 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT, N1: Mask, N2: VL);
12265 SDValue SelectCond =
12266 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: MaskVT,
12267 Ops: {VID, SplattedIdx, DAG.getCondCode(Cond: ISD::SETEQ),
12268 DAG.getUNDEF(VT: MaskVT), Mask, VL});
12269 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: SelectCond, N2: SplattedVal,
12270 N3: Vec, N4: DAG.getUNDEF(VT), N5: VL);
12271 }
12272 case Intrinsic::riscv_vfmv_s_f:
12273 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT: Op.getValueType(),
12274 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
12275 // EGS * EEW >= 128 bits
12276 case Intrinsic::riscv_vaesdf_vv:
12277 case Intrinsic::riscv_vaesdf_vs:
12278 case Intrinsic::riscv_vaesdm_vv:
12279 case Intrinsic::riscv_vaesdm_vs:
12280 case Intrinsic::riscv_vaesef_vv:
12281 case Intrinsic::riscv_vaesef_vs:
12282 case Intrinsic::riscv_vaesem_vv:
12283 case Intrinsic::riscv_vaesem_vs:
12284 case Intrinsic::riscv_vaeskf1:
12285 case Intrinsic::riscv_vaeskf2:
12286 case Intrinsic::riscv_vaesz_vs:
12287 case Intrinsic::riscv_vsm4k:
12288 case Intrinsic::riscv_vsm4r_vv:
12289 case Intrinsic::riscv_vsm4r_vs: {
12290 if (!isValidEGW(EGS: 4, VT: Op.getSimpleValueType(), Subtarget) ||
12291 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget) ||
12292 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 2).getSimpleValueType(), Subtarget))
12293 reportFatalUsageError(reason: "EGW should be greater than or equal to 4 * SEW.");
12294 return Op;
12295 }
12296 // EGS * EEW >= 256 bits
12297 case Intrinsic::riscv_vsm3c:
12298 case Intrinsic::riscv_vsm3me: {
12299 if (!isValidEGW(EGS: 8, VT: Op.getSimpleValueType(), Subtarget) ||
12300 !isValidEGW(EGS: 8, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget))
12301 reportFatalUsageError(reason: "EGW should be greater than or equal to 8 * SEW.");
12302 return Op;
12303 }
12304 // zvknha(SEW=32)/zvknhb(SEW=[32|64])
12305 case Intrinsic::riscv_vsha2ch:
12306 case Intrinsic::riscv_vsha2cl:
12307 case Intrinsic::riscv_vsha2ms: {
12308 if (Op->getSimpleValueType(ResNo: 0).getScalarSizeInBits() == 64 &&
12309 !Subtarget.hasStdExtZvknhb())
12310 reportFatalUsageError(reason: "SEW=64 needs Zvknhb to be enabled.");
12311 if (!isValidEGW(EGS: 4, VT: Op.getSimpleValueType(), Subtarget) ||
12312 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget) ||
12313 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 2).getSimpleValueType(), Subtarget))
12314 reportFatalUsageError(reason: "EGW should be greater than or equal to 4 * SEW.");
12315 return Op;
12316 }
12317 case Intrinsic::riscv_sf_vc_v_x:
12318 case Intrinsic::riscv_sf_vc_v_i:
12319 case Intrinsic::riscv_sf_vc_v_xv:
12320 case Intrinsic::riscv_sf_vc_v_iv:
12321 case Intrinsic::riscv_sf_vc_v_vv:
12322 case Intrinsic::riscv_sf_vc_v_fv:
12323 case Intrinsic::riscv_sf_vc_v_xvv:
12324 case Intrinsic::riscv_sf_vc_v_ivv:
12325 case Intrinsic::riscv_sf_vc_v_vvv:
12326 case Intrinsic::riscv_sf_vc_v_fvv:
12327 case Intrinsic::riscv_sf_vc_v_xvw:
12328 case Intrinsic::riscv_sf_vc_v_ivw:
12329 case Intrinsic::riscv_sf_vc_v_vvw:
12330 case Intrinsic::riscv_sf_vc_v_fvw: {
12331 MVT VT = Op.getSimpleValueType();
12332
12333 SmallVector<SDValue> Operands{Op->op_values()};
12334 processVCIXOperands(OrigOp: Op, Operands, DAG);
12335
12336 MVT RetVT = VT;
12337 if (VT.isFixedLengthVector())
12338 RetVT = getContainerForFixedLengthVector(VT);
12339 else if (VT.isFloatingPoint())
12340 RetVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits()),
12341 EC: VT.getVectorElementCount());
12342
12343 SDValue NewNode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: RetVT, Ops: Operands);
12344
12345 if (VT.isFixedLengthVector())
12346 NewNode = convertFromScalableVector(VT, V: NewNode, DAG, Subtarget);
12347 else if (VT.isFloatingPoint())
12348 NewNode = DAG.getBitcast(VT, V: NewNode);
12349
12350 if (Op == NewNode)
12351 break;
12352
12353 return NewNode;
12354 }
12355 }
12356
12357 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12358}
12359
12360static inline SDValue getVCIXISDNodeWCHAIN(SDValue Op, SelectionDAG &DAG,
12361 unsigned Type) {
12362 SDLoc DL(Op);
12363 SmallVector<SDValue> Operands{Op->op_values()};
12364 Operands.erase(CI: Operands.begin() + 1);
12365
12366 const RISCVSubtarget &Subtarget =
12367 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
12368 MVT VT = Op.getSimpleValueType();
12369 MVT RetVT = VT;
12370 MVT FloatVT = VT;
12371
12372 if (VT.isFloatingPoint()) {
12373 RetVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits()),
12374 EC: VT.getVectorElementCount());
12375 FloatVT = RetVT;
12376 }
12377 if (VT.isFixedLengthVector())
12378 RetVT = getContainerForFixedLengthVector(VT: RetVT, Subtarget);
12379
12380 processVCIXOperands(OrigOp: Op, Operands, DAG);
12381
12382 SDVTList VTs = DAG.getVTList(VTs: {RetVT, MVT::Other});
12383 SDValue NewNode = DAG.getNode(Opcode: Type, DL, VTList: VTs, Ops: Operands);
12384 SDValue Chain = NewNode.getValue(R: 1);
12385
12386 if (VT.isFixedLengthVector())
12387 NewNode = convertFromScalableVector(VT: FloatVT, V: NewNode, DAG, Subtarget);
12388 if (VT.isFloatingPoint())
12389 NewNode = DAG.getBitcast(VT, V: NewNode);
12390
12391 NewNode = DAG.getMergeValues(Ops: {NewNode, Chain}, dl: DL);
12392
12393 return NewNode;
12394}
12395
12396static inline SDValue getVCIXISDNodeVOID(SDValue Op, SelectionDAG &DAG,
12397 unsigned Type) {
12398 SmallVector<SDValue> Operands{Op->op_values()};
12399 Operands.erase(CI: Operands.begin() + 1);
12400 processVCIXOperands(OrigOp: Op, Operands, DAG);
12401
12402 return DAG.getNode(Opcode: Type, DL: SDLoc(Op), VT: Op.getValueType(), Ops: Operands);
12403}
12404
12405static SDValue
12406lowerFixedVectorSegLoadIntrinsics(unsigned IntNo, SDValue Op,
12407 const RISCVSubtarget &Subtarget,
12408 SelectionDAG &DAG) {
12409 bool IsStrided;
12410 switch (IntNo) {
12411 case Intrinsic::riscv_seg2_load_mask:
12412 case Intrinsic::riscv_seg3_load_mask:
12413 case Intrinsic::riscv_seg4_load_mask:
12414 case Intrinsic::riscv_seg5_load_mask:
12415 case Intrinsic::riscv_seg6_load_mask:
12416 case Intrinsic::riscv_seg7_load_mask:
12417 case Intrinsic::riscv_seg8_load_mask:
12418 IsStrided = false;
12419 break;
12420 case Intrinsic::riscv_sseg2_load_mask:
12421 case Intrinsic::riscv_sseg3_load_mask:
12422 case Intrinsic::riscv_sseg4_load_mask:
12423 case Intrinsic::riscv_sseg5_load_mask:
12424 case Intrinsic::riscv_sseg6_load_mask:
12425 case Intrinsic::riscv_sseg7_load_mask:
12426 case Intrinsic::riscv_sseg8_load_mask:
12427 IsStrided = true;
12428 break;
12429 default:
12430 llvm_unreachable("unexpected intrinsic ID");
12431 };
12432
12433 static const Intrinsic::ID VlsegInts[7] = {
12434 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
12435 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
12436 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
12437 Intrinsic::riscv_vlseg8_mask};
12438 static const Intrinsic::ID VlssegInts[7] = {
12439 Intrinsic::riscv_vlsseg2_mask, Intrinsic::riscv_vlsseg3_mask,
12440 Intrinsic::riscv_vlsseg4_mask, Intrinsic::riscv_vlsseg5_mask,
12441 Intrinsic::riscv_vlsseg6_mask, Intrinsic::riscv_vlsseg7_mask,
12442 Intrinsic::riscv_vlsseg8_mask};
12443
12444 SDLoc DL(Op);
12445 unsigned NF = Op->getNumValues() - 1;
12446 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
12447 MVT XLenVT = Subtarget.getXLenVT();
12448 MVT VT = Op->getSimpleValueType(ResNo: 0);
12449 MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
12450 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
12451 ContainerVT.getScalarSizeInBits();
12452 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: NF);
12453
12454 // Operands: (chain, int_id, pointer, mask, vl) or
12455 // (chain, int_id, pointer, offset, mask, vl)
12456 SDValue VL = Op.getOperand(i: Op.getNumOperands() - 1);
12457 SDValue Mask = Op.getOperand(i: Op.getNumOperands() - 2);
12458 MVT MaskVT = Mask.getSimpleValueType();
12459 MVT MaskContainerVT = ::getContainerForFixedLengthVector(VT: MaskVT, Subtarget);
12460 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
12461
12462 SDValue IntID = DAG.getTargetConstant(
12463 Val: IsStrided ? VlssegInts[NF - 2] : VlsegInts[NF - 2], DL, VT: XLenVT);
12464 auto *Load = cast<MemIntrinsicSDNode>(Val&: Op);
12465
12466 SDVTList VTs = DAG.getVTList(VTs: {VecTupTy, MVT::Other});
12467 SmallVector<SDValue, 9> Ops = {
12468 Load->getChain(),
12469 IntID,
12470 DAG.getUNDEF(VT: VecTupTy),
12471 Op.getOperand(i: 2),
12472 Mask,
12473 VL,
12474 DAG.getTargetConstant(
12475 Val: RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, VT: XLenVT),
12476 DAG.getTargetConstant(Val: Log2_64(Value: VT.getScalarSizeInBits()), DL, VT: XLenVT)};
12477 // Insert the stride operand.
12478 if (IsStrided)
12479 Ops.insert(I: std::next(x: Ops.begin(), n: 4), Elt: Op.getOperand(i: 3));
12480
12481 SDValue Result =
12482 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
12483 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
12484 SmallVector<SDValue, 9> Results;
12485 for (unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
12486 SDValue SubVec = DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: ContainerVT,
12487 N1: Result.getValue(R: 0),
12488 N2: DAG.getTargetConstant(Val: RetIdx, DL, VT: MVT::i32));
12489 Results.push_back(Elt: convertFromScalableVector(VT, V: SubVec, DAG, Subtarget));
12490 }
12491 Results.push_back(Elt: Result.getValue(R: 1));
12492 return DAG.getMergeValues(Ops: Results, dl: DL);
12493}
12494
12495SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
12496 SelectionDAG &DAG) const {
12497 unsigned IntNo = Op.getConstantOperandVal(i: 1);
12498 switch (IntNo) {
12499 default:
12500 break;
12501 case Intrinsic::riscv_seg2_load_mask:
12502 case Intrinsic::riscv_seg3_load_mask:
12503 case Intrinsic::riscv_seg4_load_mask:
12504 case Intrinsic::riscv_seg5_load_mask:
12505 case Intrinsic::riscv_seg6_load_mask:
12506 case Intrinsic::riscv_seg7_load_mask:
12507 case Intrinsic::riscv_seg8_load_mask:
12508 case Intrinsic::riscv_sseg2_load_mask:
12509 case Intrinsic::riscv_sseg3_load_mask:
12510 case Intrinsic::riscv_sseg4_load_mask:
12511 case Intrinsic::riscv_sseg5_load_mask:
12512 case Intrinsic::riscv_sseg6_load_mask:
12513 case Intrinsic::riscv_sseg7_load_mask:
12514 case Intrinsic::riscv_sseg8_load_mask:
12515 return lowerFixedVectorSegLoadIntrinsics(IntNo, Op, Subtarget, DAG);
12516
12517 case Intrinsic::riscv_sf_vc_v_x_se:
12518 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_X_SE);
12519 case Intrinsic::riscv_sf_vc_v_i_se:
12520 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_I_SE);
12521 case Intrinsic::riscv_sf_vc_v_xv_se:
12522 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XV_SE);
12523 case Intrinsic::riscv_sf_vc_v_iv_se:
12524 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IV_SE);
12525 case Intrinsic::riscv_sf_vc_v_vv_se:
12526 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VV_SE);
12527 case Intrinsic::riscv_sf_vc_v_fv_se:
12528 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FV_SE);
12529 case Intrinsic::riscv_sf_vc_v_xvv_se:
12530 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XVV_SE);
12531 case Intrinsic::riscv_sf_vc_v_ivv_se:
12532 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IVV_SE);
12533 case Intrinsic::riscv_sf_vc_v_vvv_se:
12534 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VVV_SE);
12535 case Intrinsic::riscv_sf_vc_v_fvv_se:
12536 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FVV_SE);
12537 case Intrinsic::riscv_sf_vc_v_xvw_se:
12538 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XVW_SE);
12539 case Intrinsic::riscv_sf_vc_v_ivw_se:
12540 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IVW_SE);
12541 case Intrinsic::riscv_sf_vc_v_vvw_se:
12542 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VVW_SE);
12543 case Intrinsic::riscv_sf_vc_v_fvw_se:
12544 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FVW_SE);
12545 }
12546
12547 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12548}
12549
12550static SDValue
12551lowerFixedVectorSegStoreIntrinsics(unsigned IntNo, SDValue Op,
12552 const RISCVSubtarget &Subtarget,
12553 SelectionDAG &DAG) {
12554 bool IsStrided;
12555 switch (IntNo) {
12556 case Intrinsic::riscv_seg2_store_mask:
12557 case Intrinsic::riscv_seg3_store_mask:
12558 case Intrinsic::riscv_seg4_store_mask:
12559 case Intrinsic::riscv_seg5_store_mask:
12560 case Intrinsic::riscv_seg6_store_mask:
12561 case Intrinsic::riscv_seg7_store_mask:
12562 case Intrinsic::riscv_seg8_store_mask:
12563 IsStrided = false;
12564 break;
12565 case Intrinsic::riscv_sseg2_store_mask:
12566 case Intrinsic::riscv_sseg3_store_mask:
12567 case Intrinsic::riscv_sseg4_store_mask:
12568 case Intrinsic::riscv_sseg5_store_mask:
12569 case Intrinsic::riscv_sseg6_store_mask:
12570 case Intrinsic::riscv_sseg7_store_mask:
12571 case Intrinsic::riscv_sseg8_store_mask:
12572 IsStrided = true;
12573 break;
12574 default:
12575 llvm_unreachable("unexpected intrinsic ID");
12576 }
12577
12578 SDLoc DL(Op);
12579 static const Intrinsic::ID VssegInts[] = {
12580 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
12581 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
12582 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
12583 Intrinsic::riscv_vsseg8_mask};
12584 static const Intrinsic::ID VsssegInts[] = {
12585 Intrinsic::riscv_vssseg2_mask, Intrinsic::riscv_vssseg3_mask,
12586 Intrinsic::riscv_vssseg4_mask, Intrinsic::riscv_vssseg5_mask,
12587 Intrinsic::riscv_vssseg6_mask, Intrinsic::riscv_vssseg7_mask,
12588 Intrinsic::riscv_vssseg8_mask};
12589
12590 // Operands: (chain, int_id, vec*, ptr, mask, vl) or
12591 // (chain, int_id, vec*, ptr, stride, mask, vl)
12592 unsigned NF = Op->getNumOperands() - (IsStrided ? 6 : 5);
12593 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
12594 MVT XLenVT = Subtarget.getXLenVT();
12595 MVT VT = Op->getOperand(Num: 2).getSimpleValueType();
12596 MVT ContainerVT = ::getContainerForFixedLengthVector(VT, Subtarget);
12597 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
12598 ContainerVT.getScalarSizeInBits();
12599 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: NF);
12600
12601 SDValue VL = Op.getOperand(i: Op.getNumOperands() - 1);
12602 SDValue Mask = Op.getOperand(i: Op.getNumOperands() - 2);
12603 MVT MaskVT = Mask.getSimpleValueType();
12604 MVT MaskContainerVT = ::getContainerForFixedLengthVector(VT: MaskVT, Subtarget);
12605 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
12606
12607 SDValue IntID = DAG.getTargetConstant(
12608 Val: IsStrided ? VsssegInts[NF - 2] : VssegInts[NF - 2], DL, VT: XLenVT);
12609 SDValue Ptr = Op->getOperand(Num: NF + 2);
12610
12611 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Val&: Op);
12612
12613 SDValue StoredVal = DAG.getUNDEF(VT: VecTupTy);
12614 for (unsigned i = 0; i < NF; i++)
12615 StoredVal = DAG.getNode(
12616 Opcode: RISCVISD::TUPLE_INSERT, DL, VT: VecTupTy, N1: StoredVal,
12617 N2: convertToScalableVector(VT: ContainerVT, V: FixedIntrinsic->getOperand(Num: 2 + i),
12618 DAG, Subtarget),
12619 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
12620
12621 SmallVector<SDValue, 10> Ops = {
12622 FixedIntrinsic->getChain(),
12623 IntID,
12624 StoredVal,
12625 Ptr,
12626 Mask,
12627 VL,
12628 DAG.getTargetConstant(Val: Log2_64(Value: VT.getScalarSizeInBits()), DL, VT: XLenVT)};
12629 // Insert the stride operand.
12630 if (IsStrided)
12631 Ops.insert(I: std::next(x: Ops.begin(), n: 4),
12632 Elt: Op.getOperand(i: Op.getNumOperands() - 3));
12633
12634 return DAG.getMemIntrinsicNode(
12635 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
12636 MemVT: FixedIntrinsic->getMemoryVT(), MMO: FixedIntrinsic->getMemOperand());
12637}
12638
12639SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12640 SelectionDAG &DAG) const {
12641 unsigned IntNo = Op.getConstantOperandVal(i: 1);
12642 switch (IntNo) {
12643 default:
12644 break;
12645 case Intrinsic::riscv_seg2_store_mask:
12646 case Intrinsic::riscv_seg3_store_mask:
12647 case Intrinsic::riscv_seg4_store_mask:
12648 case Intrinsic::riscv_seg5_store_mask:
12649 case Intrinsic::riscv_seg6_store_mask:
12650 case Intrinsic::riscv_seg7_store_mask:
12651 case Intrinsic::riscv_seg8_store_mask:
12652 case Intrinsic::riscv_sseg2_store_mask:
12653 case Intrinsic::riscv_sseg3_store_mask:
12654 case Intrinsic::riscv_sseg4_store_mask:
12655 case Intrinsic::riscv_sseg5_store_mask:
12656 case Intrinsic::riscv_sseg6_store_mask:
12657 case Intrinsic::riscv_sseg7_store_mask:
12658 case Intrinsic::riscv_sseg8_store_mask:
12659 return lowerFixedVectorSegStoreIntrinsics(IntNo, Op, Subtarget, DAG);
12660
12661 case Intrinsic::riscv_sf_vc_xv_se:
12662 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XV_SE);
12663 case Intrinsic::riscv_sf_vc_iv_se:
12664 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IV_SE);
12665 case Intrinsic::riscv_sf_vc_vv_se:
12666 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VV_SE);
12667 case Intrinsic::riscv_sf_vc_fv_se:
12668 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FV_SE);
12669 case Intrinsic::riscv_sf_vc_xvv_se:
12670 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XVV_SE);
12671 case Intrinsic::riscv_sf_vc_ivv_se:
12672 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IVV_SE);
12673 case Intrinsic::riscv_sf_vc_vvv_se:
12674 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VVV_SE);
12675 case Intrinsic::riscv_sf_vc_fvv_se:
12676 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FVV_SE);
12677 case Intrinsic::riscv_sf_vc_xvw_se:
12678 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XVW_SE);
12679 case Intrinsic::riscv_sf_vc_ivw_se:
12680 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IVW_SE);
12681 case Intrinsic::riscv_sf_vc_vvw_se:
12682 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VVW_SE);
12683 case Intrinsic::riscv_sf_vc_fvw_se:
12684 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FVW_SE);
12685 }
12686
12687 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12688}
12689
12690static unsigned getRVVReductionOp(unsigned ISDOpcode) {
12691 switch (ISDOpcode) {
12692 default:
12693 llvm_unreachable("Unhandled reduction");
12694 case ISD::VP_REDUCE_ADD:
12695 case ISD::VECREDUCE_ADD:
12696 return RISCVISD::VECREDUCE_ADD_VL;
12697 case ISD::VP_REDUCE_UMAX:
12698 case ISD::VECREDUCE_UMAX:
12699 return RISCVISD::VECREDUCE_UMAX_VL;
12700 case ISD::VP_REDUCE_SMAX:
12701 case ISD::VECREDUCE_SMAX:
12702 return RISCVISD::VECREDUCE_SMAX_VL;
12703 case ISD::VP_REDUCE_UMIN:
12704 case ISD::VECREDUCE_UMIN:
12705 return RISCVISD::VECREDUCE_UMIN_VL;
12706 case ISD::VP_REDUCE_SMIN:
12707 case ISD::VECREDUCE_SMIN:
12708 return RISCVISD::VECREDUCE_SMIN_VL;
12709 case ISD::VP_REDUCE_AND:
12710 case ISD::VECREDUCE_AND:
12711 return RISCVISD::VECREDUCE_AND_VL;
12712 case ISD::VP_REDUCE_OR:
12713 case ISD::VECREDUCE_OR:
12714 return RISCVISD::VECREDUCE_OR_VL;
12715 case ISD::VP_REDUCE_XOR:
12716 case ISD::VECREDUCE_XOR:
12717 return RISCVISD::VECREDUCE_XOR_VL;
12718 case ISD::VP_REDUCE_FADD:
12719 return RISCVISD::VECREDUCE_FADD_VL;
12720 case ISD::VP_REDUCE_SEQ_FADD:
12721 return RISCVISD::VECREDUCE_SEQ_FADD_VL;
12722 case ISD::VP_REDUCE_FMAX:
12723 case ISD::VP_REDUCE_FMAXIMUM:
12724 return RISCVISD::VECREDUCE_FMAX_VL;
12725 case ISD::VP_REDUCE_FMIN:
12726 case ISD::VP_REDUCE_FMINIMUM:
12727 return RISCVISD::VECREDUCE_FMIN_VL;
12728 }
12729
12730}
12731
12732SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
12733 SelectionDAG &DAG,
12734 bool IsVP) const {
12735 SDLoc DL(Op);
12736 SDValue Vec = Op.getOperand(i: IsVP ? 1 : 0);
12737 MVT VecVT = Vec.getSimpleValueType();
12738 assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
12739 Op.getOpcode() == ISD::VECREDUCE_OR ||
12740 Op.getOpcode() == ISD::VECREDUCE_XOR ||
12741 Op.getOpcode() == ISD::VP_REDUCE_AND ||
12742 Op.getOpcode() == ISD::VP_REDUCE_OR ||
12743 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
12744 "Unexpected reduction lowering");
12745
12746 MVT XLenVT = Subtarget.getXLenVT();
12747
12748 MVT ContainerVT = VecVT;
12749 if (VecVT.isFixedLengthVector()) {
12750 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12751 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12752 }
12753
12754 SDValue Mask, VL;
12755 if (IsVP) {
12756 Mask = Op.getOperand(i: 2);
12757 VL = Op.getOperand(i: 3);
12758 } else {
12759 std::tie(args&: Mask, args&: VL) =
12760 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12761 }
12762
12763 ISD::CondCode CC;
12764 switch (Op.getOpcode()) {
12765 default:
12766 llvm_unreachable("Unhandled reduction");
12767 case ISD::VECREDUCE_AND:
12768 case ISD::VP_REDUCE_AND: {
12769 // vcpop ~x == 0
12770 SDValue TrueMask = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
12771 if (IsVP || VecVT.isFixedLengthVector())
12772 Vec = DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Vec, N2: TrueMask, N3: VL);
12773 else
12774 Vec = DAG.getNode(Opcode: ISD::XOR, DL, VT: ContainerVT, N1: Vec, N2: TrueMask);
12775 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
12776 CC = ISD::SETEQ;
12777 break;
12778 }
12779 case ISD::VECREDUCE_OR:
12780 case ISD::VP_REDUCE_OR:
12781 // vcpop x != 0
12782 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
12783 CC = ISD::SETNE;
12784 break;
12785 case ISD::VECREDUCE_XOR:
12786 case ISD::VP_REDUCE_XOR: {
12787 // ((vcpop x) & 1) != 0
12788 SDValue One = DAG.getConstant(Val: 1, DL, VT: XLenVT);
12789 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
12790 Vec = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Vec, N2: One);
12791 CC = ISD::SETNE;
12792 break;
12793 }
12794 }
12795
12796 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
12797 SDValue SetCC = DAG.getSetCC(DL, VT: XLenVT, LHS: Vec, RHS: Zero, Cond: CC);
12798 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: SetCC);
12799
12800 if (!IsVP)
12801 return SetCC;
12802
12803 // Now include the start value in the operation.
12804 // Note that we must return the start value when no elements are operated
12805 // upon. The vcpop instructions we've emitted in each case above will return
12806 // 0 for an inactive vector, and so we've already received the neutral value:
12807 // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
12808 // can simply include the start value.
12809 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
12810 return DAG.getNode(Opcode: BaseOpc, DL, VT: Op.getValueType(), N1: SetCC, N2: Op.getOperand(i: 0));
12811}
12812
12813static bool isNonZeroAVL(SDValue AVL) {
12814 auto *RegisterAVL = dyn_cast<RegisterSDNode>(Val&: AVL);
12815 auto *ImmAVL = dyn_cast<ConstantSDNode>(Val&: AVL);
12816 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
12817 (ImmAVL && ImmAVL->getZExtValue() >= 1);
12818}
12819
12820/// Helper to lower a reduction sequence of the form:
12821/// scalar = reduce_op vec, scalar_start
12822static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT,
12823 SDValue StartValue, SDValue Vec, SDValue Mask,
12824 SDValue VL, const SDLoc &DL, SelectionDAG &DAG,
12825 const RISCVSubtarget &Subtarget) {
12826 const MVT VecVT = Vec.getSimpleValueType();
12827 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: VecVT);
12828 const MVT XLenVT = Subtarget.getXLenVT();
12829 const bool NonZeroAVL = isNonZeroAVL(AVL: VL);
12830
12831 // The reduction needs an LMUL1 input; do the splat at either LMUL1
12832 // or the original VT if fractional.
12833 auto InnerVT = VecVT.bitsLE(VT: M1VT) ? VecVT : M1VT;
12834 // We reuse the VL of the reduction to reduce vsetvli toggles if we can
12835 // prove it is non-zero. For the AVL=0 case, we need the scalar to
12836 // be the result of the reduction operation.
12837 auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(Val: 1, DL, VT: XLenVT);
12838 SDValue InitialValue =
12839 lowerScalarInsert(Scalar: StartValue, VL: InnerVL, VT: InnerVT, DL, DAG, Subtarget);
12840 if (M1VT != InnerVT)
12841 InitialValue =
12842 DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: M1VT), SubVec: InitialValue, Idx: 0);
12843 SDValue PassThru = NonZeroAVL ? DAG.getUNDEF(VT: M1VT) : InitialValue;
12844 SDValue Policy = DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
12845 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
12846 SDValue Reduction = DAG.getNode(Opcode: RVVOpcode, DL, VT: M1VT, Ops);
12847 return DAG.getExtractVectorElt(DL, VT: ResVT, Vec: Reduction, Idx: 0);
12848}
12849
12850SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
12851 SelectionDAG &DAG) const {
12852 SDLoc DL(Op);
12853 SDValue Vec = Op.getOperand(i: 0);
12854 EVT VecEVT = Vec.getValueType();
12855
12856 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
12857
12858 // Due to ordering in legalize types we may have a vector type that needs to
12859 // be split. Do that manually so we can get down to a legal type.
12860 while (getTypeAction(Context&: *DAG.getContext(), VT: VecEVT) ==
12861 TargetLowering::TypeSplitVector) {
12862 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
12863 VecEVT = Lo.getValueType();
12864 Vec = DAG.getNode(Opcode: BaseOpc, DL, VT: VecEVT, N1: Lo, N2: Hi);
12865 }
12866
12867 // TODO: The type may need to be widened rather than split. Or widened before
12868 // it can be split.
12869 if (!isTypeLegal(VT: VecEVT))
12870 return SDValue();
12871
12872 MVT VecVT = VecEVT.getSimpleVT();
12873 MVT VecEltVT = VecVT.getVectorElementType();
12874 unsigned RVVOpcode = getRVVReductionOp(ISDOpcode: Op.getOpcode());
12875
12876 MVT ContainerVT = VecVT;
12877 if (VecVT.isFixedLengthVector()) {
12878 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12879 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12880 }
12881
12882 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12883
12884 SDValue StartV;
12885 switch (BaseOpc) {
12886 default:
12887 StartV = DAG.getIdentityElement(Opcode: BaseOpc, DL, VT: VecEltVT, Flags: SDNodeFlags());
12888 break;
12889 case ISD::AND:
12890 case ISD::OR:
12891 case ISD::UMAX:
12892 case ISD::UMIN:
12893 case ISD::SMAX:
12894 case ISD::SMIN:
12895 StartV = DAG.getExtractVectorElt(DL, VT: VecEltVT, Vec, Idx: 0);
12896 break;
12897 }
12898 return lowerReductionSeq(RVVOpcode, ResVT: Op.getSimpleValueType(), StartValue: StartV, Vec,
12899 Mask, VL, DL, DAG, Subtarget);
12900}
12901
12902// Given a reduction op, this function returns the matching reduction opcode,
12903// the vector SDValue and the scalar SDValue required to lower this to a
12904// RISCVISD node.
12905static std::tuple<unsigned, SDValue, SDValue>
12906getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT,
12907 const RISCVSubtarget &Subtarget) {
12908 SDLoc DL(Op);
12909 auto Flags = Op->getFlags();
12910 unsigned Opcode = Op.getOpcode();
12911 switch (Opcode) {
12912 default:
12913 llvm_unreachable("Unhandled reduction");
12914 case ISD::VECREDUCE_FADD: {
12915 // Use positive zero if we can. It is cheaper to materialize.
12916 SDValue Zero =
12917 DAG.getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT: EltVT);
12918 return std::make_tuple(args: RISCVISD::VECREDUCE_FADD_VL, args: Op.getOperand(i: 0), args&: Zero);
12919 }
12920 case ISD::VECREDUCE_SEQ_FADD:
12921 return std::make_tuple(args: RISCVISD::VECREDUCE_SEQ_FADD_VL, args: Op.getOperand(i: 1),
12922 args: Op.getOperand(i: 0));
12923 case ISD::VECREDUCE_FMINIMUM:
12924 case ISD::VECREDUCE_FMAXIMUM:
12925 case ISD::VECREDUCE_FMIN:
12926 case ISD::VECREDUCE_FMAX: {
12927 SDValue Front = DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op.getOperand(i: 0), Idx: 0);
12928 unsigned RVVOpc =
12929 (Opcode == ISD::VECREDUCE_FMIN || Opcode == ISD::VECREDUCE_FMINIMUM)
12930 ? RISCVISD::VECREDUCE_FMIN_VL
12931 : RISCVISD::VECREDUCE_FMAX_VL;
12932 return std::make_tuple(args&: RVVOpc, args: Op.getOperand(i: 0), args&: Front);
12933 }
12934 }
12935}
12936
12937SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
12938 SelectionDAG &DAG) const {
12939 SDLoc DL(Op);
12940 MVT VecEltVT = Op.getSimpleValueType();
12941
12942 unsigned RVVOpcode;
12943 SDValue VectorVal, ScalarVal;
12944 std::tie(args&: RVVOpcode, args&: VectorVal, args&: ScalarVal) =
12945 getRVVFPReductionOpAndOperands(Op, DAG, EltVT: VecEltVT, Subtarget);
12946 MVT VecVT = VectorVal.getSimpleValueType();
12947
12948 MVT ContainerVT = VecVT;
12949 if (VecVT.isFixedLengthVector()) {
12950 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12951 VectorVal = convertToScalableVector(VT: ContainerVT, V: VectorVal, DAG, Subtarget);
12952 }
12953
12954 MVT ResVT = Op.getSimpleValueType();
12955 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12956 SDValue Res = lowerReductionSeq(RVVOpcode, ResVT, StartValue: ScalarVal, Vec: VectorVal, Mask,
12957 VL, DL, DAG, Subtarget);
12958 if (Op.getOpcode() != ISD::VECREDUCE_FMINIMUM &&
12959 Op.getOpcode() != ISD::VECREDUCE_FMAXIMUM)
12960 return Res;
12961
12962 if (Op->getFlags().hasNoNaNs())
12963 return Res;
12964
12965 // Force output to NaN if any element is Nan.
12966 SDValue IsNan =
12967 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
12968 Ops: {VectorVal, VectorVal, DAG.getCondCode(Cond: ISD::SETNE),
12969 DAG.getUNDEF(VT: Mask.getValueType()), Mask, VL});
12970 MVT XLenVT = Subtarget.getXLenVT();
12971 SDValue CPop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: IsNan, N2: Mask, N3: VL);
12972 SDValue NoNaNs = DAG.getSetCC(DL, VT: XLenVT, LHS: CPop,
12973 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
12974 return DAG.getSelect(
12975 DL, VT: ResVT, Cond: NoNaNs, LHS: Res,
12976 RHS: DAG.getConstantFP(Val: APFloat::getNaN(Sem: ResVT.getFltSemantics()), DL, VT: ResVT));
12977}
12978
12979SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
12980 SelectionDAG &DAG) const {
12981 SDLoc DL(Op);
12982 unsigned Opc = Op.getOpcode();
12983 SDValue Start = Op.getOperand(i: 0);
12984 SDValue Vec = Op.getOperand(i: 1);
12985 EVT VecEVT = Vec.getValueType();
12986 MVT XLenVT = Subtarget.getXLenVT();
12987
12988 // TODO: The type may need to be widened rather than split. Or widened before
12989 // it can be split.
12990 if (!isTypeLegal(VT: VecEVT))
12991 return SDValue();
12992
12993 MVT VecVT = VecEVT.getSimpleVT();
12994 unsigned RVVOpcode = getRVVReductionOp(ISDOpcode: Opc);
12995
12996 if (VecVT.isFixedLengthVector()) {
12997 auto ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12998 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12999 }
13000
13001 SDValue VL = Op.getOperand(i: 3);
13002 SDValue Mask = Op.getOperand(i: 2);
13003 SDValue Res =
13004 lowerReductionSeq(RVVOpcode, ResVT: Op.getSimpleValueType(), StartValue: Op.getOperand(i: 0),
13005 Vec, Mask, VL, DL, DAG, Subtarget);
13006 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
13007 Op->getFlags().hasNoNaNs())
13008 return Res;
13009
13010 // Propagate NaNs.
13011 MVT PredVT = getMaskTypeFor(VecVT: Vec.getSimpleValueType());
13012 // Check if any of the elements in Vec is NaN.
13013 SDValue IsNaN = DAG.getNode(
13014 Opcode: RISCVISD::SETCC_VL, DL, VT: PredVT,
13015 Ops: {Vec, Vec, DAG.getCondCode(Cond: ISD::SETNE), DAG.getUNDEF(VT: PredVT), Mask, VL});
13016 SDValue VCPop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: IsNaN, N2: Mask, N3: VL);
13017 // Check if the start value is NaN.
13018 SDValue StartIsNaN = DAG.getSetCC(DL, VT: XLenVT, LHS: Start, RHS: Start, Cond: ISD::SETUO);
13019 VCPop = DAG.getNode(Opcode: ISD::OR, DL, VT: XLenVT, N1: VCPop, N2: StartIsNaN);
13020 SDValue NoNaNs = DAG.getSetCC(DL, VT: XLenVT, LHS: VCPop,
13021 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
13022 MVT ResVT = Res.getSimpleValueType();
13023 return DAG.getSelect(
13024 DL, VT: ResVT, Cond: NoNaNs, LHS: Res,
13025 RHS: DAG.getConstantFP(Val: APFloat::getNaN(Sem: ResVT.getFltSemantics()), DL, VT: ResVT));
13026}
13027
13028static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL,
13029 SDValue V, MVT WideVT);
13030
13031SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
13032 SelectionDAG &DAG) const {
13033 SDValue Vec = Op.getOperand(i: 0);
13034 SDValue SubVec = Op.getOperand(i: 1);
13035 MVT VecVT = Vec.getSimpleValueType();
13036 MVT SubVecVT = SubVec.getSimpleValueType();
13037
13038 SDLoc DL(Op);
13039 MVT XLenVT = Subtarget.getXLenVT();
13040 unsigned OrigIdx = Op.getConstantOperandVal(i: 2);
13041 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
13042
13043 bool IsPExtInsert =
13044 Subtarget.hasStdExtP() &&
13045 ((Subtarget.is64Bit() &&
13046 (SubVecVT == MVT::v2i16 || SubVecVT == MVT::v4i8)) ||
13047 (!Subtarget.is64Bit() && (VecVT == MVT::v4i16 || VecVT == MVT::v8i8)));
13048
13049 // Fold insert of a 32-bit packed type into a zero-filled 64-bit packed vector
13050 // at index 0 (a zero-extend) to avoid scalarizing it into a byte-wise repack.
13051 if (IsPExtInsert) {
13052 if ((VecVT != MVT::v4i16 && VecVT != MVT::v8i8) ||
13053 SubVecVT.getSizeInBits() != 32 || OrigIdx != 0 ||
13054 !ISD::isConstantSplatVectorAllZeros(N: Vec.getNode()))
13055 return SDValue();
13056
13057 if (!Subtarget.is64Bit()) {
13058 SDValue Zero = DAG.getBitcast(VT: SubVecVT, V: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
13059 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: SubVec, N2: Zero);
13060 }
13061 return widenPackedVectorWithZeros(DAG, DL, V: SubVec, WideVT: VecVT);
13062 }
13063
13064 if (OrigIdx == 0 && Vec.isUndef())
13065 return Op;
13066
13067 // We don't have the ability to slide mask vectors up indexed by their i1
13068 // elements; the smallest we can do is i8. Often we are able to bitcast to
13069 // equivalent i8 vectors. Note that when inserting a fixed-length vector
13070 // into a scalable one, we might not necessarily have enough scalable
13071 // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
13072 if (SubVecVT.getVectorElementType() == MVT::i1) {
13073 if (VecVT.getVectorMinNumElements() >= 8 &&
13074 SubVecVT.getVectorMinNumElements() >= 8) {
13075 assert(OrigIdx % 8 == 0 && "Invalid index");
13076 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
13077 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
13078 "Unexpected mask vector lowering");
13079 OrigIdx /= 8;
13080 SubVecVT =
13081 MVT::getVectorVT(VT: MVT::i8, NumElements: SubVecVT.getVectorMinNumElements() / 8,
13082 IsScalable: SubVecVT.isScalableVector());
13083 VecVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VecVT.getVectorMinNumElements() / 8,
13084 IsScalable: VecVT.isScalableVector());
13085 Vec = DAG.getBitcast(VT: VecVT, V: Vec);
13086 SubVec = DAG.getBitcast(VT: SubVecVT, V: SubVec);
13087 } else {
13088 // We can't slide this mask vector up indexed by its i1 elements.
13089 // This poses a problem when we wish to insert a scalable vector which
13090 // can't be re-expressed as a larger type. Just choose the slow path and
13091 // extend to a larger type, then truncate back down.
13092 MVT ExtVecVT = VecVT.changeVectorElementType(EltVT: MVT::i8);
13093 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(EltVT: MVT::i8);
13094 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVecVT, Operand: Vec);
13095 SubVec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtSubVecVT, Operand: SubVec);
13096 Vec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: ExtVecVT, N1: Vec, N2: SubVec,
13097 N3: Op.getOperand(i: 2));
13098 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: ExtVecVT);
13099 return DAG.getSetCC(DL, VT: VecVT, LHS: Vec, RHS: SplatZero, Cond: ISD::SETNE);
13100 }
13101 }
13102
13103 // If the subvector vector is a fixed-length type and we don't know VLEN
13104 // exactly, we cannot use subregister manipulation to simplify the codegen; we
13105 // don't know which register of a LMUL group contains the specific subvector
13106 // as we only know the minimum register size. Therefore we must slide the
13107 // vector group up the full amount.
13108 const auto VLen = Subtarget.getRealVLen();
13109 if (SubVecVT.isFixedLengthVector() && !VLen) {
13110 MVT ContainerVT = VecVT;
13111 if (VecVT.isFixedLengthVector()) {
13112 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13113 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
13114 }
13115
13116 SubVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec, Idx: 0);
13117
13118 SDValue Mask =
13119 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
13120 // Set the vector length to only the number of elements we care about. Note
13121 // that for slideup this includes the offset.
13122 unsigned EndIndex = OrigIdx + SubVecVT.getVectorNumElements();
13123 SDValue VL = DAG.getConstant(Val: EndIndex, DL, VT: XLenVT);
13124
13125 // Use tail agnostic policy if we're inserting over Vec's tail.
13126 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
13127 if (VecVT.isFixedLengthVector() && EndIndex == VecVT.getVectorNumElements())
13128 Policy = RISCVVType::TAIL_AGNOSTIC;
13129
13130 // If we're inserting into the lowest elements, use a tail undisturbed
13131 // vmv.v.v.
13132 if (OrigIdx == 0) {
13133 SubVec =
13134 DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: ContainerVT, N1: Vec, N2: SubVec, N3: VL);
13135 } else {
13136 SDValue SlideupAmt = DAG.getConstant(Val: OrigIdx, DL, VT: XLenVT);
13137 SubVec = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Vec, Op: SubVec,
13138 Offset: SlideupAmt, Mask, VL, Policy);
13139 }
13140
13141 if (VecVT.isFixedLengthVector())
13142 SubVec = convertFromScalableVector(VT: VecVT, V: SubVec, DAG, Subtarget);
13143 return DAG.getBitcast(VT: Op.getValueType(), V: SubVec);
13144 }
13145
13146 MVT ContainerVecVT = VecVT;
13147 if (VecVT.isFixedLengthVector()) {
13148 ContainerVecVT = getContainerForFixedLengthVector(VT: VecVT);
13149 Vec = convertToScalableVector(VT: ContainerVecVT, V: Vec, DAG, Subtarget);
13150 }
13151
13152 MVT ContainerSubVecVT = SubVecVT;
13153 if (SubVecVT.isFixedLengthVector()) {
13154 ContainerSubVecVT = getContainerForFixedLengthVector(VT: SubVecVT);
13155 SubVec = convertToScalableVector(VT: ContainerSubVecVT, V: SubVec, DAG, Subtarget);
13156 }
13157
13158 unsigned SubRegIdx;
13159 ElementCount RemIdx;
13160 // insert_subvector scales the index by vscale if the subvector is scalable,
13161 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
13162 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
13163 if (SubVecVT.isFixedLengthVector()) {
13164 assert(VLen);
13165 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
13166 auto Decompose =
13167 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
13168 VecVT: ContainerVecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx / Vscale, TRI);
13169 SubRegIdx = Decompose.first;
13170 RemIdx = ElementCount::getFixed(MinVal: (Decompose.second * Vscale) +
13171 (OrigIdx % Vscale));
13172 } else {
13173 auto Decompose =
13174 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
13175 VecVT: ContainerVecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx, TRI);
13176 SubRegIdx = Decompose.first;
13177 RemIdx = ElementCount::getScalable(MinVal: Decompose.second);
13178 }
13179
13180 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
13181 assert(isPowerOf2_64(
13182 Subtarget.expandVScale(SubVecVT.getSizeInBits()).getKnownMinValue()));
13183 bool ExactlyVecRegSized =
13184 Subtarget.expandVScale(X: SubVecVT.getSizeInBits())
13185 .isKnownMultipleOf(RHS: Subtarget.expandVScale(X: VecRegSize));
13186
13187 // 1. If the Idx has been completely eliminated and this subvector's size is
13188 // a vector register or a multiple thereof, or the surrounding elements are
13189 // undef, then this is a subvector insert which naturally aligns to a vector
13190 // register. These can easily be handled using subregister manipulation.
13191 // 2. If the subvector isn't an exact multiple of a valid register group size,
13192 // then the insertion must preserve the undisturbed elements of the register.
13193 // We do this by lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1
13194 // vector type (which resolves to a subregister copy), performing a VSLIDEUP
13195 // to place the subvector within the vector register, and an INSERT_SUBVECTOR
13196 // of that LMUL=1 type back into the larger vector (resolving to another
13197 // subregister operation). See below for how our VSLIDEUP works. We go via a
13198 // LMUL=1 type to avoid allocating a large register group to hold our
13199 // subvector.
13200 if (RemIdx.isZero() && (ExactlyVecRegSized || Vec.isUndef())) {
13201 if (SubVecVT.isFixedLengthVector()) {
13202 // We may get NoSubRegister if inserting at index 0 and the subvec
13203 // container is the same as the vector, e.g. vec=v4i32,subvec=v4i32,idx=0
13204 if (SubRegIdx == RISCV::NoSubRegister) {
13205 assert(OrigIdx == 0);
13206 return Op;
13207 }
13208
13209 // Use a insert_subvector that will resolve to an insert subreg.
13210 assert(VLen);
13211 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
13212 SDValue Insert =
13213 DAG.getInsertSubvector(DL, Vec, SubVec, Idx: OrigIdx / Vscale);
13214 if (VecVT.isFixedLengthVector())
13215 Insert = convertFromScalableVector(VT: VecVT, V: Insert, DAG, Subtarget);
13216 return Insert;
13217 }
13218 return Op;
13219 }
13220
13221 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
13222 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
13223 // (in our case undisturbed). This means we can set up a subvector insertion
13224 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
13225 // size of the subvector.
13226 MVT InterSubVT = ContainerVecVT;
13227 SDValue AlignedExtract = Vec;
13228 unsigned AlignedIdx = OrigIdx - RemIdx.getKnownMinValue();
13229 if (SubVecVT.isFixedLengthVector()) {
13230 assert(VLen);
13231 AlignedIdx /= *VLen / RISCV::RVVBitsPerBlock;
13232 }
13233 if (ContainerVecVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: ContainerVecVT))) {
13234 InterSubVT = RISCVTargetLowering::getM1VT(VT: ContainerVecVT);
13235 // Extract a subvector equal to the nearest full vector register type. This
13236 // should resolve to a EXTRACT_SUBREG instruction.
13237 AlignedExtract = DAG.getExtractSubvector(DL, VT: InterSubVT, Vec, Idx: AlignedIdx);
13238 }
13239
13240 SubVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: InterSubVT), SubVec, Idx: 0);
13241
13242 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT: ContainerVecVT, DL, DAG, Subtarget);
13243
13244 ElementCount EndIndex = RemIdx + SubVecVT.getVectorElementCount();
13245 VL = DAG.getElementCount(DL, VT: XLenVT, EC: SubVecVT.getVectorElementCount());
13246
13247 // Use tail agnostic policy if we're inserting over InterSubVT's tail.
13248 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
13249 if (Subtarget.expandVScale(X: EndIndex) ==
13250 Subtarget.expandVScale(X: InterSubVT.getVectorElementCount()))
13251 Policy = RISCVVType::TAIL_AGNOSTIC;
13252
13253 // If we're inserting into the lowest elements, use a tail undisturbed
13254 // vmv.v.v.
13255 if (RemIdx.isZero()) {
13256 SubVec = DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: InterSubVT, N1: AlignedExtract,
13257 N2: SubVec, N3: VL);
13258 } else {
13259 SDValue SlideupAmt = DAG.getElementCount(DL, VT: XLenVT, EC: RemIdx);
13260
13261 // Construct the vector length corresponding to RemIdx + length(SubVecVT).
13262 VL = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: SlideupAmt, N2: VL);
13263
13264 SubVec = getVSlideup(DAG, Subtarget, DL, VT: InterSubVT, Passthru: AlignedExtract, Op: SubVec,
13265 Offset: SlideupAmt, Mask, VL, Policy);
13266 }
13267
13268 // If required, insert this subvector back into the correct vector register.
13269 // This should resolve to an INSERT_SUBREG instruction.
13270 if (ContainerVecVT.bitsGT(VT: InterSubVT))
13271 SubVec = DAG.getInsertSubvector(DL, Vec, SubVec, Idx: AlignedIdx);
13272
13273 if (VecVT.isFixedLengthVector())
13274 SubVec = convertFromScalableVector(VT: VecVT, V: SubVec, DAG, Subtarget);
13275
13276 // We might have bitcast from a mask type: cast back to the original type if
13277 // required.
13278 return DAG.getBitcast(VT: Op.getSimpleValueType(), V: SubVec);
13279}
13280
13281SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
13282 SelectionDAG &DAG) const {
13283 SDValue Vec = Op.getOperand(i: 0);
13284 MVT SubVecVT = Op.getSimpleValueType();
13285 MVT VecVT = Vec.getSimpleValueType();
13286
13287 SDLoc DL(Op);
13288 MVT XLenVT = Subtarget.getXLenVT();
13289 unsigned OrigIdx = Op.getConstantOperandVal(i: 1);
13290 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
13291
13292 // With an index of 0 this is a cast-like subvector, which can be performed
13293 // with subregister operations.
13294 if (OrigIdx == 0)
13295 return Op;
13296
13297 // We don't have the ability to slide mask vectors down indexed by their i1
13298 // elements; the smallest we can do is i8. Often we are able to bitcast to
13299 // equivalent i8 vectors. Note that when extracting a fixed-length vector
13300 // from a scalable one, we might not necessarily have enough scalable
13301 // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
13302 if (SubVecVT.getVectorElementType() == MVT::i1) {
13303 if (VecVT.getVectorMinNumElements() >= 8 &&
13304 SubVecVT.getVectorMinNumElements() >= 8) {
13305 assert(OrigIdx % 8 == 0 && "Invalid index");
13306 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
13307 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
13308 "Unexpected mask vector lowering");
13309 OrigIdx /= 8;
13310 SubVecVT =
13311 MVT::getVectorVT(VT: MVT::i8, NumElements: SubVecVT.getVectorMinNumElements() / 8,
13312 IsScalable: SubVecVT.isScalableVector());
13313 VecVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VecVT.getVectorMinNumElements() / 8,
13314 IsScalable: VecVT.isScalableVector());
13315 Vec = DAG.getBitcast(VT: VecVT, V: Vec);
13316 } else {
13317 // We can't slide this mask vector down, indexed by its i1 elements.
13318 // This poses a problem when we wish to extract a scalable vector which
13319 // can't be re-expressed as a larger type. Just choose the slow path and
13320 // extend to a larger type, then truncate back down.
13321 // TODO: We could probably improve this when extracting certain fixed
13322 // from fixed, where we can extract as i8 and shift the correct element
13323 // right to reach the desired subvector?
13324 MVT ExtVecVT = VecVT.changeVectorElementType(EltVT: MVT::i8);
13325 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(EltVT: MVT::i8);
13326 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVecVT, Operand: Vec);
13327 Vec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ExtSubVecVT, N1: Vec,
13328 N2: Op.getOperand(i: 1));
13329 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: ExtSubVecVT);
13330 return DAG.getSetCC(DL, VT: SubVecVT, LHS: Vec, RHS: SplatZero, Cond: ISD::SETNE);
13331 }
13332 }
13333
13334 const auto VLen = Subtarget.getRealVLen();
13335
13336 // If the subvector vector is a fixed-length type and we don't know VLEN
13337 // exactly, we cannot use subregister manipulation to simplify the codegen; we
13338 // don't know which register of a LMUL group contains the specific subvector
13339 // as we only know the minimum register size. Therefore we must slide the
13340 // vector group down the full amount.
13341 if (SubVecVT.isFixedLengthVector() && !VLen) {
13342 MVT ContainerVT = VecVT;
13343 if (VecVT.isFixedLengthVector()) {
13344 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13345 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
13346 }
13347
13348 // Shrink down Vec so we're performing the slidedown on a smaller LMUL.
13349 unsigned LastIdx = OrigIdx + SubVecVT.getVectorNumElements() - 1;
13350 if (auto ShrunkVT =
13351 getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: LastIdx, DL, DAG, Subtarget)) {
13352 ContainerVT = *ShrunkVT;
13353 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: 0);
13354 }
13355
13356 SDValue Mask =
13357 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
13358 // Set the vector length to only the number of elements we care about. This
13359 // avoids sliding down elements we're going to discard straight away.
13360 SDValue VL = DAG.getConstant(Val: SubVecVT.getVectorNumElements(), DL, VT: XLenVT);
13361 SDValue SlidedownAmt = DAG.getConstant(Val: OrigIdx, DL, VT: XLenVT);
13362 SDValue Slidedown =
13363 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
13364 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: SlidedownAmt, Mask, VL);
13365 // Now we can use a cast-like subvector extract to get the result.
13366 Slidedown = DAG.getExtractSubvector(DL, VT: SubVecVT, Vec: Slidedown, Idx: 0);
13367 return DAG.getBitcast(VT: Op.getValueType(), V: Slidedown);
13368 }
13369
13370 if (VecVT.isFixedLengthVector()) {
13371 VecVT = getContainerForFixedLengthVector(VT: VecVT);
13372 Vec = convertToScalableVector(VT: VecVT, V: Vec, DAG, Subtarget);
13373 }
13374
13375 MVT ContainerSubVecVT = SubVecVT;
13376 if (SubVecVT.isFixedLengthVector())
13377 ContainerSubVecVT = getContainerForFixedLengthVector(VT: SubVecVT);
13378
13379 unsigned SubRegIdx;
13380 ElementCount RemIdx;
13381 // extract_subvector scales the index by vscale if the subvector is scalable,
13382 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
13383 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
13384 if (SubVecVT.isFixedLengthVector()) {
13385 assert(VLen);
13386 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
13387 auto Decompose =
13388 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
13389 VecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx / Vscale, TRI);
13390 SubRegIdx = Decompose.first;
13391 RemIdx = ElementCount::getFixed(MinVal: (Decompose.second * Vscale) +
13392 (OrigIdx % Vscale));
13393 } else {
13394 auto Decompose =
13395 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
13396 VecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx, TRI);
13397 SubRegIdx = Decompose.first;
13398 RemIdx = ElementCount::getScalable(MinVal: Decompose.second);
13399 }
13400
13401 // If the Idx has been completely eliminated then this is a subvector extract
13402 // which naturally aligns to a vector register. These can easily be handled
13403 // using subregister manipulation. We use an extract_subvector that will
13404 // resolve to an extract subreg.
13405 if (RemIdx.isZero()) {
13406 if (SubVecVT.isFixedLengthVector()) {
13407 assert(VLen);
13408 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
13409 Vec =
13410 DAG.getExtractSubvector(DL, VT: ContainerSubVecVT, Vec, Idx: OrigIdx / Vscale);
13411 return convertFromScalableVector(VT: SubVecVT, V: Vec, DAG, Subtarget);
13412 }
13413 return Op;
13414 }
13415
13416 // Else SubVecVT is M1 or smaller and may need to be slid down: if SubVecVT
13417 // was > M1 then the index would need to be a multiple of VLMAX, and so would
13418 // divide exactly.
13419 assert(RISCVVType::decodeVLMUL(getLMUL(ContainerSubVecVT)).second ||
13420 getLMUL(ContainerSubVecVT) == RISCVVType::LMUL_1);
13421
13422 // If the vector type is an LMUL-group type, extract a subvector equal to the
13423 // nearest full vector register type.
13424 MVT InterSubVT = VecVT;
13425 if (VecVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: VecVT))) {
13426 // If VecVT has an LMUL > 1, then SubVecVT should have a smaller LMUL, and
13427 // we should have successfully decomposed the extract into a subregister.
13428 // We use an extract_subvector that will resolve to a subreg extract.
13429 assert(SubRegIdx != RISCV::NoSubRegister);
13430 (void)SubRegIdx;
13431 unsigned Idx = OrigIdx - RemIdx.getKnownMinValue();
13432 if (SubVecVT.isFixedLengthVector()) {
13433 assert(VLen);
13434 Idx /= *VLen / RISCV::RVVBitsPerBlock;
13435 }
13436 InterSubVT = RISCVTargetLowering::getM1VT(VT: VecVT);
13437 Vec = DAG.getExtractSubvector(DL, VT: InterSubVT, Vec, Idx);
13438 }
13439
13440 // Slide this vector register down by the desired number of elements in order
13441 // to place the desired subvector starting at element 0.
13442 SDValue SlidedownAmt = DAG.getElementCount(DL, VT: XLenVT, EC: RemIdx);
13443 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: InterSubVT, DL, DAG, Subtarget);
13444 if (SubVecVT.isFixedLengthVector())
13445 VL = DAG.getConstant(Val: SubVecVT.getVectorNumElements(), DL, VT: XLenVT);
13446 SDValue Slidedown =
13447 getVSlidedown(DAG, Subtarget, DL, VT: InterSubVT, Passthru: DAG.getUNDEF(VT: InterSubVT),
13448 Op: Vec, Offset: SlidedownAmt, Mask, VL);
13449
13450 // Now the vector is in the right position, extract our final subvector. This
13451 // should resolve to a COPY.
13452 Slidedown = DAG.getExtractSubvector(DL, VT: SubVecVT, Vec: Slidedown, Idx: 0);
13453
13454 // We might have bitcast from a mask type: cast back to the original type if
13455 // required.
13456 return DAG.getBitcast(VT: Op.getSimpleValueType(), V: Slidedown);
13457}
13458
13459// Widen a vector's operands to i8, then truncate its results back to the
13460// original type, typically i1. All operand and result types must be the same.
13461static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL,
13462 SelectionDAG &DAG) {
13463 MVT VT = N.getSimpleValueType();
13464 MVT WideVT = VT.changeVectorElementType(EltVT: MVT::i8);
13465 SmallVector<SDValue, 4> WideOps;
13466 for (SDValue Op : N->ops()) {
13467 assert(Op.getSimpleValueType() == VT &&
13468 "Operands and result must be same type");
13469 WideOps.push_back(Elt: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Op));
13470 }
13471
13472 unsigned NumVals = N->getNumValues();
13473
13474 SDVTList VTs = DAG.getVTList(VTs: SmallVector<EVT, 4>(
13475 NumVals,
13476 N.getValueType().changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i8)));
13477 SDValue WideN = DAG.getNode(Opcode: N.getOpcode(), DL, VTList: VTs, Ops: WideOps);
13478 SmallVector<SDValue, 4> TruncVals;
13479 for (unsigned I = 0; I < NumVals; I++) {
13480 TruncVals.push_back(
13481 Elt: DAG.getSetCC(DL, VT: N->getSimpleValueType(ResNo: I), LHS: WideN.getValue(R: I),
13482 RHS: DAG.getConstant(Val: 0, DL, VT: WideVT), Cond: ISD::SETNE));
13483 }
13484
13485 if (TruncVals.size() > 1)
13486 return DAG.getMergeValues(Ops: TruncVals, dl: DL);
13487 return TruncVals.front();
13488}
13489
13490SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
13491 SelectionDAG &DAG) const {
13492 SDLoc DL(Op);
13493 MVT VecVT = Op.getSimpleValueType();
13494
13495 const unsigned Factor = Op->getNumValues();
13496 assert(Factor <= 8);
13497
13498 // 1 bit element vectors need to be widened to e8
13499 if (VecVT.getVectorElementType() == MVT::i1)
13500 return widenVectorOpsToi8(N: Op, DL, DAG);
13501
13502 bool IsFixedVector = VecVT.isFixedLengthVector();
13503
13504 MVT ContainerVecVT = VecVT;
13505 if (IsFixedVector)
13506 ContainerVecVT = getContainerForFixedLengthVector(VT: VecVT);
13507
13508 // If concatenating would exceed LMUL=8, we need to split.
13509 if ((ContainerVecVT.getSizeInBits().getKnownMinValue() * Factor) >
13510 (8 * RISCV::RVVBitsPerBlock)) {
13511 SmallVector<SDValue, 8> Ops(Factor * 2);
13512 for (unsigned i = 0; i != Factor; ++i) {
13513 auto [OpLo, OpHi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: i);
13514 Ops[i * 2] = OpLo;
13515 Ops[i * 2 + 1] = OpHi;
13516 }
13517
13518 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
13519
13520 SDValue Lo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
13521 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
13522 SDValue Hi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
13523 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
13524
13525 SmallVector<SDValue, 8> Res(Factor);
13526 for (unsigned i = 0; i != Factor; ++i)
13527 Res[i] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo.getValue(R: i),
13528 N2: Hi.getValue(R: i));
13529
13530 return DAG.getMergeValues(Ops: Res, dl: DL);
13531 }
13532
13533 if (Subtarget.hasStdExtZvzip() && Factor == 2 && !IsFixedVector) {
13534 MVT VT = Op->getSimpleValueType(ResNo: 0);
13535 MVT NewVT = VT.getDoubleNumVectorElementsVT();
13536 if (isTypeLegal(VT: NewVT) && isLegalVTForZvzipOperand(VT, Subtarget)) {
13537 SDValue V1 = Op->getOperand(Num: 0);
13538 SDValue V2 = Op->getOperand(Num: 1);
13539 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, N1: V1, N2: V2);
13540 SDValue Even =
13541 lowerZvzipVUNZIP(Opc: RISCVISD::VUNZIPE_VL, Op: V, DL, DAG, Subtarget);
13542 SDValue Odd =
13543 lowerZvzipVUNZIP(Opc: RISCVISD::VUNZIPO_VL, Op: V, DL, DAG, Subtarget);
13544 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
13545 }
13546 }
13547
13548 SmallVector<SDValue, 8> Ops(Op->op_values());
13549
13550 // Concatenate the vectors as one vector to deinterleave
13551 MVT ConcatVT =
13552 MVT::getVectorVT(VT: VecVT.getVectorElementType(),
13553 EC: VecVT.getVectorElementCount().multiplyCoefficientBy(
13554 RHS: PowerOf2Ceil(A: Factor)));
13555 if (Ops.size() < PowerOf2Ceil(A: Factor))
13556 Ops.append(NumInputs: PowerOf2Ceil(A: Factor) - Factor, Elt: DAG.getUNDEF(VT: VecVT));
13557 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops);
13558
13559 if (Factor == 2 && !IsFixedVector) {
13560 // We can deinterleave through vnsrl.wi if the element type is smaller than
13561 // ELEN
13562 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
13563 SDValue Even = getDeinterleaveShiftAndTrunc(DL, VT: VecVT, Src: Concat, Factor: 2, Index: 0, DAG);
13564 SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VT: VecVT, Src: Concat, Factor: 2, Index: 1, DAG);
13565 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
13566 }
13567
13568 // For the indices, use the vmv.v.x of an i8 constant to fill the largest
13569 // possibly mask vector, then extract the required subvector. Doing this
13570 // (instead of a vid, vmsne sequence) reduces LMUL, and allows the mask
13571 // creation to be rematerialized during register allocation to reduce
13572 // register pressure if needed.
13573
13574 MVT MaskVT = ConcatVT.changeVectorElementType(EltVT: MVT::i1);
13575
13576 SDValue EvenSplat = DAG.getConstant(Val: 0b01010101, DL, VT: MVT::nxv8i8);
13577 EvenSplat = DAG.getBitcast(VT: MVT::nxv64i1, V: EvenSplat);
13578 SDValue EvenMask = DAG.getExtractSubvector(DL, VT: MaskVT, Vec: EvenSplat, Idx: 0);
13579
13580 SDValue OddSplat = DAG.getConstant(Val: 0b10101010, DL, VT: MVT::nxv8i8);
13581 OddSplat = DAG.getBitcast(VT: MVT::nxv64i1, V: OddSplat);
13582 SDValue OddMask = DAG.getExtractSubvector(DL, VT: MaskVT, Vec: OddSplat, Idx: 0);
13583
13584 // vcompress the even and odd elements into two separate vectors
13585 SDValue EvenWide = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: ConcatVT, N1: Concat,
13586 N2: EvenMask, N3: DAG.getUNDEF(VT: ConcatVT));
13587 SDValue OddWide = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: ConcatVT, N1: Concat,
13588 N2: OddMask, N3: DAG.getUNDEF(VT: ConcatVT));
13589
13590 // Extract the result half of the gather for even and odd
13591 SDValue Even = DAG.getExtractSubvector(DL, VT: VecVT, Vec: EvenWide, Idx: 0);
13592 SDValue Odd = DAG.getExtractSubvector(DL, VT: VecVT, Vec: OddWide, Idx: 0);
13593
13594 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
13595 }
13596
13597 // Store with unit-stride store and load it back with segmented load.
13598 SDValue Mask, VL;
13599 MVT XLenVT = Subtarget.getXLenVT();
13600 auto &MF = DAG.getMachineFunction();
13601 SDValue Chain = DAG.getEntryNode();
13602 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
13603 SDValue StackPtr;
13604 MachinePointerInfo PtrInfo;
13605 if (IsFixedVector) {
13606 // Calculating the stack size.
13607 ElementCount ActualConcatEC =
13608 VecVT.getVectorElementCount().multiplyCoefficientBy(RHS: Factor);
13609 EVT ConcatEVT = EVT::getVectorVT(
13610 Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(), EC: ActualConcatEC);
13611 StackPtr = DAG.CreateStackTemporary(Bytes: ConcatEVT.getStoreSize(), Alignment);
13612 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13613 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13614
13615 // If this is a fixed vector, instead of using the concat vector, we simply
13616 // store each fixed vector operand directly onto the stack, individually.
13617 // The reason being that if the fixed vector is (much) smaller than the
13618 // container vector, we will be wasting space on stack.
13619 TypeSize VecSize = VecVT.getStoreSize();
13620 SDValue BasePtr = StackPtr;
13621 MachinePointerInfo PI = PtrInfo;
13622 SmallVector<SDValue, 8> Tokens(Factor);
13623 for (auto [Idx, FieldOp] : enumerate(First: Op->op_values())) {
13624 if (Idx) {
13625 // Advance the pointer.
13626 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: VecSize);
13627 PI = PI.getWithOffset(O: VecSize);
13628 }
13629 Tokens[Idx] = DAG.getStore(Chain, dl: DL, Val: FieldOp, Ptr: BasePtr, PtrInfo: PI, Alignment);
13630 }
13631 Chain = DAG.getTokenFactor(DL, Vals&: Tokens);
13632
13633 // Calculating Mask and VL for later usages.
13634 std::tie(args&: Mask, args&: VL) =
13635 getDefaultVLOps(VecVT, ContainerVT: ContainerVecVT, DL, DAG, Subtarget);
13636 ConcatVT = getContainerForFixedLengthVector(VT: ConcatVT);
13637 } else {
13638 std::tie(args&: Mask, args&: VL) = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
13639 StackPtr = DAG.CreateStackTemporary(Bytes: ConcatVT.getStoreSize(), Alignment);
13640 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13641 PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13642
13643 SDValue StoreOps[] = {
13644 Chain, DAG.getTargetConstant(Val: Intrinsic::riscv_vse, DL, VT: XLenVT), Concat,
13645 StackPtr, VL};
13646
13647 Chain = DAG.getMemIntrinsicNode(
13648 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps,
13649 MemVT: ConcatVT.getVectorElementType(), PtrInfo, Alignment,
13650 Flags: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer());
13651 }
13652
13653 // Load it back with segmented load.
13654 SDValue Passthru = DAG.getUNDEF(VT: ConcatVT);
13655 static const Intrinsic::ID VlsegIntrinsicsIds[] = {
13656 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
13657 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
13658 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
13659 Intrinsic::riscv_vlseg8_mask};
13660
13661 SDValue LoadOps[] = {
13662 Chain,
13663 DAG.getTargetConstant(Val: VlsegIntrinsicsIds[Factor - 2], DL, VT: XLenVT),
13664 Passthru,
13665 StackPtr,
13666 Mask,
13667 VL,
13668 DAG.getTargetConstant(
13669 Val: RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, VT: XLenVT),
13670 DAG.getTargetConstant(Val: Log2_64(Value: VecVT.getScalarSizeInBits()), DL, VT: XLenVT)};
13671
13672 unsigned Sz = Factor * ContainerVecVT.getVectorMinNumElements() *
13673 ContainerVecVT.getScalarSizeInBits();
13674 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: Factor);
13675
13676 SDValue Load = DAG.getMemIntrinsicNode(
13677 Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: DAG.getVTList(VTs: {VecTupTy, MVT::Other}),
13678 Ops: LoadOps, MemVT: ConcatVT.getVectorElementType(), PtrInfo, Alignment,
13679 Flags: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer());
13680
13681 SmallVector<SDValue, 8> Res(Factor);
13682
13683 for (unsigned i = 0U; i < Factor; ++i) {
13684 SDValue FieldRes =
13685 DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: ContainerVecVT, N1: Load,
13686 N2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
13687 if (IsFixedVector)
13688 FieldRes = convertFromScalableVector(VT: VecVT, V: FieldRes, DAG, Subtarget);
13689 Res[i] = FieldRes;
13690 }
13691
13692 return DAG.getMergeValues(Ops: Res, dl: DL);
13693}
13694
13695SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
13696 SelectionDAG &DAG) const {
13697 SDLoc DL(Op);
13698 MVT VecVT = Op.getSimpleValueType();
13699
13700 const unsigned Factor = Op.getNumOperands();
13701 assert(Factor <= 8);
13702
13703 // i1 vectors need to be widened to i8
13704 if (VecVT.getVectorElementType() == MVT::i1)
13705 return widenVectorOpsToi8(N: Op, DL, DAG);
13706
13707 // Convert to scalable vectors first.
13708 if (VecVT.isFixedLengthVector()) {
13709 MVT ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13710 SmallVector<SDValue, 8> Ops(Factor);
13711 for (unsigned i = 0U; i < Factor; ++i)
13712 Ops[i] = convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i), DAG,
13713 Subtarget);
13714
13715 SmallVector<EVT, 8> VTs(Factor, ContainerVT);
13716 SDValue NewInterleave = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs, Ops);
13717
13718 SmallVector<SDValue, 8> Res(Factor);
13719 for (unsigned i = 0U; i < Factor; ++i)
13720 Res[i] = convertFromScalableVector(VT: VecVT, V: NewInterleave.getValue(R: i), DAG,
13721 Subtarget);
13722 return DAG.getMergeValues(Ops: Res, dl: DL);
13723 }
13724
13725 MVT XLenVT = Subtarget.getXLenVT();
13726 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
13727
13728 // If the VT is larger than LMUL=8, we need to split and reassemble.
13729 if ((VecVT.getSizeInBits().getKnownMinValue() * Factor) >
13730 (8 * RISCV::RVVBitsPerBlock)) {
13731 SmallVector<SDValue, 8> Ops(Factor * 2);
13732 for (unsigned i = 0; i != Factor; ++i) {
13733 auto [OpLo, OpHi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: i);
13734 Ops[i] = OpLo;
13735 Ops[i + Factor] = OpHi;
13736 }
13737
13738 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
13739
13740 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
13741 Ops: ArrayRef(Ops).take_front(N: Factor)),
13742 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
13743 Ops: ArrayRef(Ops).drop_front(N: Factor))};
13744
13745 SmallVector<SDValue, 8> Concats(Factor);
13746 for (unsigned i = 0; i != Factor; ++i) {
13747 unsigned IdxLo = 2 * i;
13748 unsigned IdxHi = 2 * i + 1;
13749 Concats[i] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT,
13750 N1: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
13751 N2: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
13752 }
13753
13754 return DAG.getMergeValues(Ops: Concats, dl: DL);
13755 }
13756
13757 SDValue Interleaved;
13758
13759 // Spill to the stack using a segment store for simplicity.
13760 if (Factor != 2) {
13761 EVT MemVT =
13762 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(),
13763 EC: VecVT.getVectorElementCount() * Factor);
13764
13765 // Allocate a stack slot.
13766 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
13767 SDValue StackPtr =
13768 DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
13769 EVT PtrVT = StackPtr.getValueType();
13770 auto &MF = DAG.getMachineFunction();
13771 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13772 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13773
13774 static const Intrinsic::ID IntrIds[] = {
13775 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
13776 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
13777 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
13778 Intrinsic::riscv_vsseg8_mask,
13779 };
13780
13781 unsigned Sz =
13782 Factor * VecVT.getVectorMinNumElements() * VecVT.getScalarSizeInBits();
13783 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: Factor);
13784
13785 SDValue StoredVal = DAG.getUNDEF(VT: VecTupTy);
13786 for (unsigned i = 0; i < Factor; i++)
13787 StoredVal =
13788 DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: VecTupTy, N1: StoredVal,
13789 N2: Op.getOperand(i), N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
13790
13791 SDValue Ops[] = {DAG.getEntryNode(),
13792 DAG.getTargetConstant(Val: IntrIds[Factor - 2], DL, VT: XLenVT),
13793 StoredVal,
13794 StackPtr,
13795 Mask,
13796 VL,
13797 DAG.getTargetConstant(Val: Log2_64(Value: VecVT.getScalarSizeInBits()),
13798 DL, VT: XLenVT)};
13799
13800 SDValue Chain = DAG.getMemIntrinsicNode(
13801 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
13802 MemVT: VecVT.getVectorElementType(), PtrInfo, Alignment,
13803 Flags: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer());
13804
13805 SmallVector<SDValue, 8> Loads(Factor);
13806
13807 SDValue Increment = DAG.getTypeSize(DL, VT: PtrVT, TS: VecVT.getStoreSize());
13808 for (unsigned i = 0; i != Factor; ++i) {
13809 if (i != 0)
13810 StackPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: Increment);
13811
13812 Loads[i] = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
13813 }
13814
13815 return DAG.getMergeValues(Ops: Loads, dl: DL);
13816 }
13817
13818 if (Subtarget.hasStdExtZvzip() && !Op.getOperand(i: 0).isUndef() &&
13819 !Op.getOperand(i: 1).isUndef()) {
13820 MVT VT = Op->getSimpleValueType(ResNo: 0);
13821 if (isLegalVTForZvzipOperand(VT, Subtarget)) {
13822 // Freeze the sources so we can increase their use count.
13823 SDValue V1 = DAG.getFreeze(V: Op->getOperand(Num: 0));
13824 SDValue V2 = DAG.getFreeze(V: Op->getOperand(Num: 1));
13825 SDValue Interleaved = lowerZvzipVZIP(Op0: V1, Op1: V2, DL, DAG, Subtarget);
13826 SDValue Lo = DAG.getExtractSubvector(DL, VT, Vec: Interleaved, Idx: 0);
13827 SDValue Hi = DAG.getExtractSubvector(DL, VT, Vec: Interleaved,
13828 Idx: VT.getVectorMinNumElements());
13829 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
13830 }
13831 }
13832
13833 // If the element type is smaller than ELEN, then we can interleave with
13834 // vwaddu.vv and vwmaccu.vx
13835 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
13836 Interleaved = getWideningInterleave(EvenV: Op.getOperand(i: 0), OddV: Op.getOperand(i: 1), DL,
13837 DAG, Subtarget);
13838 } else {
13839 // Otherwise, fallback to using vrgathere16.vv
13840 MVT ConcatVT =
13841 MVT::getVectorVT(VT: VecVT.getVectorElementType(),
13842 EC: VecVT.getVectorElementCount().multiplyCoefficientBy(RHS: 2));
13843 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT,
13844 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
13845
13846 MVT IdxVT = ConcatVT.changeVectorElementType(EltVT: MVT::i16);
13847
13848 // 0 1 2 3 4 5 6 7 ...
13849 SDValue StepVec = DAG.getStepVector(DL, ResVT: IdxVT);
13850
13851 // 1 1 1 1 1 1 1 1 ...
13852 SDValue Ones = DAG.getSplatVector(VT: IdxVT, DL, Op: DAG.getConstant(Val: 1, DL, VT: XLenVT));
13853
13854 // 1 0 1 0 1 0 1 0 ...
13855 SDValue OddMask = DAG.getNode(Opcode: ISD::AND, DL, VT: IdxVT, N1: StepVec, N2: Ones);
13856 OddMask = DAG.getSetCC(
13857 DL, VT: IdxVT.changeVectorElementType(EltVT: MVT::i1), LHS: OddMask,
13858 RHS: DAG.getSplatVector(VT: IdxVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: XLenVT)),
13859 Cond: ISD::CondCode::SETNE);
13860
13861 SDValue VLMax = DAG.getSplatVector(VT: IdxVT, DL, Op: computeVLMax(VecVT, DL, DAG));
13862
13863 // Build up the index vector for interleaving the concatenated vector
13864 // 0 0 1 1 2 2 3 3 ...
13865 SDValue Idx = DAG.getNode(Opcode: ISD::SRL, DL, VT: IdxVT, N1: StepVec, N2: Ones);
13866 // 0 n 1 n+1 2 n+2 3 n+3 ...
13867 Idx =
13868 DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT: IdxVT, N1: Idx, N2: VLMax, N3: Idx, N4: OddMask, N5: VL);
13869
13870 // Then perform the interleave
13871 // v[0] v[n] v[1] v[n+1] v[2] v[n+2] v[3] v[n+3] ...
13872 SDValue TrueMask = getAllOnesMask(VecVT: IdxVT, VL, DL, DAG);
13873 Interleaved = DAG.getNode(Opcode: RISCVISD::VRGATHEREI16_VV_VL, DL, VT: ConcatVT,
13874 N1: Concat, N2: Idx, N3: DAG.getUNDEF(VT: ConcatVT), N4: TrueMask, N5: VL);
13875 }
13876
13877 // Extract the two halves from the interleaved result
13878 SDValue Lo = DAG.getExtractSubvector(DL, VT: VecVT, Vec: Interleaved, Idx: 0);
13879 SDValue Hi = DAG.getExtractSubvector(DL, VT: VecVT, Vec: Interleaved,
13880 Idx: VecVT.getVectorMinNumElements());
13881
13882 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
13883}
13884
13885// Lower step_vector to the vid instruction. Any non-identity step value must
13886// be accounted for my manual expansion.
13887SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
13888 SelectionDAG &DAG) const {
13889 SDLoc DL(Op);
13890 MVT VT = Op.getSimpleValueType();
13891 assert(VT.isScalableVector() && "Expected scalable vector");
13892 MVT XLenVT = Subtarget.getXLenVT();
13893 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
13894 SDValue StepVec = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT, N1: Mask, N2: VL);
13895 uint64_t StepValImm = Op.getConstantOperandVal(i: 0);
13896 if (StepValImm != 1) {
13897 if (isPowerOf2_64(Value: StepValImm)) {
13898 SDValue StepVal =
13899 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
13900 N2: DAG.getConstant(Val: Log2_64(Value: StepValImm), DL, VT: XLenVT), N3: VL);
13901 StepVec = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: StepVec, N2: StepVal);
13902 } else {
13903 SDValue StepVal = lowerScalarSplat(
13904 Passthru: SDValue(), Scalar: DAG.getConstant(Val: StepValImm, DL, VT: VT.getVectorElementType()),
13905 VL, VT, DL, DAG, Subtarget);
13906 StepVec = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: StepVec, N2: StepVal);
13907 }
13908 }
13909 return StepVec;
13910}
13911
13912// Implement vector_reverse using vrgather.vv with indices determined by
13913// subtracting the id of each element from (VLMAX-1). This will convert
13914// the indices like so:
13915// (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
13916// TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
13917SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
13918 SelectionDAG &DAG) const {
13919 SDLoc DL(Op);
13920 MVT VecVT = Op.getSimpleValueType();
13921
13922 // Reverse a 64-bit packed vector on RV32 by reversing each 32-bit half and
13923 // swapping them.
13924 if (Subtarget.hasStdExtP() && !Subtarget.hasVInstructions()) {
13925 assert(!Subtarget.is64Bit() && VecVT.getSizeInBits() == 64 &&
13926 "Unexpected packed VECTOR_REVERSE type");
13927 SDValue V = Op.getOperand(i: 0);
13928 if (VecVT == MVT::v2i32) {
13929 // A 2-element reverse is just an element swap.
13930 SDValue Lo = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: V, Idx: 0);
13931 SDValue Hi = DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec: V, Idx: 1);
13932 return DAG.getBuildVector(VT: VecVT, DL, Ops: {Hi, Lo});
13933 }
13934 auto [Lo, Hi] = DAG.SplitVector(N: V, DL);
13935 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Lo.getSimpleValueType(), Operand: Lo);
13936 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Hi.getSimpleValueType(), Operand: Hi);
13937 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Hi, N2: Lo);
13938 }
13939
13940 if (VecVT.getVectorElementType() == MVT::i1) {
13941 MVT WidenVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
13942 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: Op.getOperand(i: 0));
13943 SDValue Op2 = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: WidenVT, Operand: Op1);
13944 return DAG.getSetCC(DL, VT: VecVT, LHS: Op2,
13945 RHS: DAG.getConstant(Val: 0, DL, VT: Op2.getValueType()), Cond: ISD::SETNE);
13946 }
13947
13948 MVT ContainerVT = VecVT;
13949 SDValue Vec = Op.getOperand(i: 0);
13950 if (VecVT.isFixedLengthVector()) {
13951 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13952 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
13953 }
13954
13955 MVT XLenVT = Subtarget.getXLenVT();
13956 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
13957
13958 // On some uarchs vrgather.vv will read from every input register for each
13959 // output register, regardless of the indices. However to reverse a vector
13960 // each output register only needs to read from one register. So decompose it
13961 // into LMUL * M1 vrgather.vvs, so we get O(LMUL) performance instead of
13962 // O(LMUL^2).
13963 //
13964 // vsetvli a1, zero, e64, m4, ta, ma
13965 // vrgatherei16.vv v12, v8, v16
13966 // ->
13967 // vsetvli a1, zero, e64, m1, ta, ma
13968 // vrgather.vv v15, v8, v16
13969 // vrgather.vv v14, v9, v16
13970 // vrgather.vv v13, v10, v16
13971 // vrgather.vv v12, v11, v16
13972 if (ContainerVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT)) &&
13973 ContainerVT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
13974 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
13975 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Lo.getValueType(), Operand: Lo);
13976 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Hi.getValueType(), Operand: Hi);
13977 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ContainerVT, N1: Hi, N2: Lo);
13978
13979 // Fixed length vectors might not fit exactly into their container, and so
13980 // leave a gap in the front of the vector after being reversed. Slide this
13981 // away.
13982 //
13983 // x x x x 3 2 1 0 <- v4i16 @ vlen=128
13984 // 0 1 2 3 x x x x <- reverse
13985 // x x x x 0 1 2 3 <- vslidedown.vx
13986 if (VecVT.isFixedLengthVector()) {
13987 SDValue Offset = DAG.getNode(
13988 Opcode: ISD::SUB, DL, VT: XLenVT,
13989 N1: DAG.getElementCount(DL, VT: XLenVT, EC: ContainerVT.getVectorElementCount()),
13990 N2: DAG.getElementCount(DL, VT: XLenVT, EC: VecVT.getVectorElementCount()));
13991 Concat =
13992 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
13993 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Concat, Offset, Mask, VL);
13994 Concat = convertFromScalableVector(VT: VecVT, V: Concat, DAG, Subtarget);
13995 }
13996 return Concat;
13997 }
13998
13999 unsigned EltSize = ContainerVT.getScalarSizeInBits();
14000 unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
14001 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
14002 unsigned MaxVLMAX =
14003 VecVT.isFixedLengthVector()
14004 ? VecVT.getVectorNumElements()
14005 : RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
14006
14007 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
14008 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
14009
14010 // If this is SEW=8 and VLMAX is potentially more than 256, we need
14011 // to use vrgatherei16.vv.
14012 if (MaxVLMAX > 256 && EltSize == 8) {
14013 // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
14014 // Reverse each half, then reassemble them in reverse order.
14015 // NOTE: It's also possible that after splitting that VLMAX no longer
14016 // requires vrgatherei16.vv.
14017 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
14018 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
14019 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
14020 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: LoVT, Operand: Lo);
14021 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: HiVT, Operand: Hi);
14022 // Reassemble the low and high pieces reversed.
14023 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Hi, N2: Lo);
14024 }
14025
14026 // Just promote the int type to i16 which will double the LMUL.
14027 IntVT = MVT::getVectorVT(VT: MVT::i16, EC: ContainerVT.getVectorElementCount());
14028 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
14029 }
14030
14031 // At LMUL > 1, do the index computation in 16 bits to reduce register
14032 // pressure.
14033 if (IntVT.getScalarType().bitsGT(VT: MVT::i16) &&
14034 IntVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: IntVT))) {
14035 assert(isUInt<16>(MaxVLMAX - 1)); // Largest VLMAX is 65536 @ zvl65536b
14036 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
14037 IntVT = IntVT.changeVectorElementType(EltVT: MVT::i16);
14038 }
14039
14040 // Calculate VLMAX-1 for the desired SEW.
14041 SDValue VLMinus1 = DAG.getNode(
14042 Opcode: ISD::SUB, DL, VT: XLenVT,
14043 N1: DAG.getElementCount(DL, VT: XLenVT, EC: VecVT.getVectorElementCount()),
14044 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
14045
14046 // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
14047 bool IsRV32E64 =
14048 !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
14049 SDValue SplatVL;
14050 if (!IsRV32E64)
14051 SplatVL = DAG.getSplatVector(VT: IntVT, DL, Op: VLMinus1);
14052 else
14053 SplatVL = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IntVT, N1: DAG.getUNDEF(VT: IntVT),
14054 N2: VLMinus1, N3: DAG.getRegister(Reg: RISCV::X0, VT: XLenVT));
14055
14056 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: IntVT, N1: Mask, N2: VL);
14057 SDValue Indices = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: IntVT, N1: SplatVL, N2: VID,
14058 N3: DAG.getUNDEF(VT: IntVT), N4: Mask, N5: VL);
14059
14060 SDValue Gather = DAG.getNode(Opcode: GatherOpc, DL, VT: ContainerVT, N1: Vec, N2: Indices,
14061 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
14062 if (VecVT.isFixedLengthVector())
14063 Gather = convertFromScalableVector(VT: VecVT, V: Gather, DAG, Subtarget);
14064 return Gather;
14065}
14066
14067SDValue RISCVTargetLowering::lowerVECTOR_SPLICE(SDValue Op,
14068 SelectionDAG &DAG) const {
14069 SDLoc DL(Op);
14070 SDValue V1 = Op.getOperand(i: 0);
14071 SDValue V2 = Op.getOperand(i: 1);
14072 SDValue Offset = Op.getOperand(i: 2);
14073 MVT XLenVT = Subtarget.getXLenVT();
14074 MVT VecVT = Op.getSimpleValueType();
14075
14076 SDValue VLMax = computeVLMax(VecVT, DL, DAG);
14077
14078 SDValue DownOffset, UpOffset;
14079 if (Op.getOpcode() == ISD::VECTOR_SPLICE_LEFT) {
14080 // The operand is a TargetConstant, we need to rebuild it as a regular
14081 // constant.
14082 DownOffset = Offset;
14083 UpOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: Offset);
14084 } else {
14085 // The operand is a TargetConstant, we need to rebuild it as a regular
14086 // constant rather than negating the original operand.
14087 UpOffset = Offset;
14088 DownOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: Offset);
14089 }
14090
14091 SDValue TrueMask = getAllOnesMask(VecVT, VL: VLMax, DL, DAG);
14092
14093 SDValue SlideDown = getVSlidedown(
14094 DAG, Subtarget, DL, VT: VecVT, Passthru: DAG.getUNDEF(VT: VecVT), Op: V1, Offset: DownOffset, Mask: TrueMask,
14095 VL: Subtarget.hasVLDependentLatency() ? UpOffset
14096 : DAG.getRegister(Reg: RISCV::X0, VT: XLenVT));
14097 return getVSlideup(DAG, Subtarget, DL, VT: VecVT, Passthru: SlideDown, Op: V2, Offset: UpOffset,
14098 Mask: TrueMask, VL: DAG.getRegister(Reg: RISCV::X0, VT: XLenVT),
14099 Policy: RISCVVType::TAIL_AGNOSTIC);
14100}
14101
14102SDValue
14103RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
14104 SelectionDAG &DAG) const {
14105 SDLoc DL(Op);
14106 auto *Load = cast<LoadSDNode>(Val&: Op);
14107
14108 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
14109 Load->getMemoryVT(),
14110 *Load->getMemOperand()) &&
14111 "Expecting a correctly-aligned load");
14112
14113 MVT VT = Op.getSimpleValueType();
14114 MVT XLenVT = Subtarget.getXLenVT();
14115 MVT ContainerVT = getContainerForFixedLengthVector(VT);
14116
14117 // If we know the exact VLEN and our fixed length vector completely fills
14118 // the container, use a whole register load instead.
14119 const auto [MinVLMAX, MaxVLMAX] =
14120 RISCVTargetLowering::computeVLMAXBounds(VecVT: ContainerVT, Subtarget);
14121 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
14122 RISCVTargetLowering::getM1VT(VT: ContainerVT).bitsLE(VT: ContainerVT)) {
14123 MachineMemOperand *MMO = Load->getMemOperand();
14124 SDValue NewLoad =
14125 DAG.getLoad(VT: ContainerVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
14126 PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getBaseAlign(), MMOFlags: MMO->getFlags(),
14127 AAInfo: MMO->getAAInfo(), Ranges: MMO->getRanges());
14128 SDValue Result = convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
14129 return DAG.getMergeValues(Ops: {Result, NewLoad.getValue(R: 1)}, dl: DL);
14130 }
14131
14132 SDValue VL = DAG.getConstant(Val: VT.getVectorNumElements(), DL, VT: XLenVT);
14133
14134 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
14135 SDValue IntID = DAG.getTargetConstant(
14136 Val: IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle, DL, VT: XLenVT);
14137 SmallVector<SDValue, 4> Ops{Load->getChain(), IntID};
14138 if (!IsMaskOp)
14139 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14140 Ops.push_back(Elt: Load->getBasePtr());
14141 Ops.push_back(Elt: VL);
14142 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
14143 SDValue NewLoad =
14144 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
14145 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
14146
14147 SDValue Result = convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
14148 return DAG.getMergeValues(Ops: {Result, NewLoad.getValue(R: 1)}, dl: DL);
14149}
14150
14151SDValue
14152RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
14153 SelectionDAG &DAG) const {
14154 SDLoc DL(Op);
14155 auto *Store = cast<StoreSDNode>(Val&: Op);
14156
14157 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
14158 Store->getMemoryVT(),
14159 *Store->getMemOperand()) &&
14160 "Expecting a correctly-aligned store");
14161
14162 SDValue StoreVal = Store->getValue();
14163 MVT VT = StoreVal.getSimpleValueType();
14164 MVT XLenVT = Subtarget.getXLenVT();
14165
14166 // If the size less than a byte, we need to pad with zeros to make a byte.
14167 if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
14168 VT = MVT::v8i1;
14169 StoreVal =
14170 DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT), SubVec: StoreVal, Idx: 0);
14171 }
14172
14173 MVT ContainerVT = getContainerForFixedLengthVector(VT);
14174
14175 SDValue NewValue =
14176 convertToScalableVector(VT: ContainerVT, V: StoreVal, DAG, Subtarget);
14177
14178 // If we know the exact VLEN and our fixed length vector completely fills
14179 // the container, use a whole register store instead.
14180 const auto [MinVLMAX, MaxVLMAX] =
14181 RISCVTargetLowering::computeVLMAXBounds(VecVT: ContainerVT, Subtarget);
14182 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
14183 RISCVTargetLowering::getM1VT(VT: ContainerVT).bitsLE(VT: ContainerVT)) {
14184 MachineMemOperand *MMO = Store->getMemOperand();
14185 return DAG.getStore(Chain: Store->getChain(), dl: DL, Val: NewValue, Ptr: Store->getBasePtr(),
14186 PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getBaseAlign(),
14187 MMOFlags: MMO->getFlags(), AAInfo: MMO->getAAInfo());
14188 }
14189
14190 SDValue VL = DAG.getConstant(Val: VT.getVectorNumElements(), DL, VT: XLenVT);
14191
14192 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
14193 SDValue IntID = DAG.getTargetConstant(
14194 Val: IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, VT: XLenVT);
14195 return DAG.getMemIntrinsicNode(
14196 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
14197 Ops: {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
14198 MemVT: Store->getMemoryVT(), MMO: Store->getMemOperand());
14199}
14200
14201SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
14202 SelectionDAG &DAG) const {
14203 SDLoc DL(Op);
14204 MVT VT = Op.getSimpleValueType();
14205
14206 const auto *MemSD = cast<MemSDNode>(Val&: Op);
14207 EVT MemVT = MemSD->getMemoryVT();
14208 MachineMemOperand *MMO = MemSD->getMemOperand();
14209 SDValue Chain = MemSD->getChain();
14210 SDValue BasePtr = MemSD->getBasePtr();
14211
14212 SDValue Mask, PassThru, VL;
14213 bool IsExpandingLoad = false;
14214 if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Val&: Op)) {
14215 Mask = VPLoad->getMask();
14216 PassThru = DAG.getUNDEF(VT);
14217 VL = VPLoad->getVectorLength();
14218 } else {
14219 const auto *MLoad = cast<MaskedLoadSDNode>(Val&: Op);
14220 Mask = MLoad->getMask();
14221 PassThru = MLoad->getPassThru();
14222 IsExpandingLoad = MLoad->isExpandingLoad();
14223 }
14224
14225 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
14226
14227 MVT XLenVT = Subtarget.getXLenVT();
14228
14229 MVT ContainerVT = VT;
14230 if (VT.isFixedLengthVector()) {
14231 ContainerVT = getContainerForFixedLengthVector(VT);
14232 PassThru = convertToScalableVector(VT: ContainerVT, V: PassThru, DAG, Subtarget);
14233 if (!IsUnmasked) {
14234 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14235 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14236 }
14237 }
14238
14239 if (!VL)
14240 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
14241
14242 SDValue ExpandingVL;
14243 if (!IsUnmasked && IsExpandingLoad) {
14244 ExpandingVL = VL;
14245 VL =
14246 DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Mask,
14247 N2: getAllOnesMask(VecVT: Mask.getSimpleValueType(), VL, DL, DAG), N3: VL);
14248 }
14249
14250 unsigned IntID = IsUnmasked || IsExpandingLoad ? Intrinsic::riscv_vle
14251 : Intrinsic::riscv_vle_mask;
14252 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
14253 if (IntID == Intrinsic::riscv_vle)
14254 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14255 else
14256 Ops.push_back(Elt: PassThru);
14257 Ops.push_back(Elt: BasePtr);
14258 if (IntID == Intrinsic::riscv_vle_mask)
14259 Ops.push_back(Elt: Mask);
14260 Ops.push_back(Elt: VL);
14261 if (IntID == Intrinsic::riscv_vle_mask)
14262 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT));
14263
14264 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
14265
14266 SDValue Result =
14267 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
14268 Chain = Result.getValue(R: 1);
14269 if (ExpandingVL) {
14270 MVT IndexVT = ContainerVT;
14271 if (ContainerVT.isFloatingPoint())
14272 IndexVT = ContainerVT.changeVectorElementTypeToInteger();
14273
14274 MVT IndexEltVT = IndexVT.getVectorElementType();
14275 bool UseVRGATHEREI16 = false;
14276 // If index vector is an i8 vector and the element count exceeds 256, we
14277 // should change the element type of index vector to i16 to avoid
14278 // overflow.
14279 if (IndexEltVT == MVT::i8 && VT.getVectorNumElements() > 256) {
14280 // FIXME: We need to do vector splitting manually for LMUL=8 cases.
14281 assert(getLMUL(IndexVT) != RISCVVType::LMUL_8);
14282 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
14283 UseVRGATHEREI16 = true;
14284 }
14285
14286 SDValue Iota =
14287 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: IndexVT,
14288 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_viota, DL, VT: XLenVT),
14289 N2: DAG.getUNDEF(VT: IndexVT), N3: Mask, N4: ExpandingVL);
14290 Result =
14291 DAG.getNode(Opcode: UseVRGATHEREI16 ? RISCVISD::VRGATHEREI16_VV_VL
14292 : RISCVISD::VRGATHER_VV_VL,
14293 DL, VT: ContainerVT, N1: Result, N2: Iota, N3: PassThru, N4: Mask, N5: ExpandingVL);
14294 }
14295
14296 if (VT.isFixedLengthVector())
14297 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14298
14299 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
14300}
14301
14302SDValue RISCVTargetLowering::lowerLoadFF(SDValue Op, SelectionDAG &DAG) const {
14303 SDLoc DL(Op);
14304 MVT VT = Op->getSimpleValueType(ResNo: 0);
14305
14306 const auto *VPLoadFF = cast<VPLoadFFSDNode>(Val&: Op);
14307 EVT MemVT = VPLoadFF->getMemoryVT();
14308 MachineMemOperand *MMO = VPLoadFF->getMemOperand();
14309 SDValue Chain = VPLoadFF->getChain();
14310 SDValue BasePtr = VPLoadFF->getBasePtr();
14311
14312 SDValue Mask = VPLoadFF->getMask();
14313 SDValue VL = VPLoadFF->getVectorLength();
14314
14315 MVT XLenVT = Subtarget.getXLenVT();
14316
14317 MVT ContainerVT = VT;
14318 if (VT.isFixedLengthVector()) {
14319 ContainerVT = getContainerForFixedLengthVector(VT);
14320 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14321 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14322 }
14323
14324 unsigned IntID = Intrinsic::riscv_vleff_mask;
14325 SDValue Ops[] = {
14326 Chain,
14327 DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT),
14328 DAG.getUNDEF(VT: ContainerVT),
14329 BasePtr,
14330 Mask,
14331 VL,
14332 DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT)};
14333
14334 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, Op->getValueType(ResNo: 1), MVT::Other});
14335
14336 SDValue Result =
14337 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
14338 SDValue OutVL = Result.getValue(R: 1);
14339 Chain = Result.getValue(R: 2);
14340
14341 if (VT.isFixedLengthVector())
14342 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14343
14344 return DAG.getMergeValues(Ops: {Result, OutVL, Chain}, dl: DL);
14345}
14346
14347SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
14348 SelectionDAG &DAG) const {
14349 SDLoc DL(Op);
14350
14351 const auto *MemSD = cast<MemSDNode>(Val&: Op);
14352 EVT MemVT = MemSD->getMemoryVT();
14353 MachineMemOperand *MMO = MemSD->getMemOperand();
14354 SDValue Chain = MemSD->getChain();
14355 SDValue BasePtr = MemSD->getBasePtr();
14356 SDValue Val, Mask, VL;
14357
14358 bool IsCompressingStore = false;
14359 if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Val&: Op)) {
14360 Val = VPStore->getValue();
14361 Mask = VPStore->getMask();
14362 VL = VPStore->getVectorLength();
14363 } else {
14364 const auto *MStore = cast<MaskedStoreSDNode>(Val&: Op);
14365 Val = MStore->getValue();
14366 Mask = MStore->getMask();
14367 IsCompressingStore = MStore->isCompressingStore();
14368 }
14369
14370 bool IsUnmasked =
14371 ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()) || IsCompressingStore;
14372
14373 MVT VT = Val.getSimpleValueType();
14374 MVT XLenVT = Subtarget.getXLenVT();
14375
14376 MVT ContainerVT = VT;
14377 if (VT.isFixedLengthVector()) {
14378 ContainerVT = getContainerForFixedLengthVector(VT);
14379
14380 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
14381 if (!IsUnmasked || IsCompressingStore) {
14382 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14383 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14384 }
14385 }
14386
14387 if (!VL)
14388 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
14389
14390 if (IsCompressingStore) {
14391 Val = DAG.getNode(
14392 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ContainerVT,
14393 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_vcompress, DL, VT: XLenVT),
14394 N2: DAG.getUNDEF(VT: ContainerVT), N3: Val, N4: Mask, N5: VL);
14395 VL =
14396 DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Mask,
14397 N2: getAllOnesMask(VecVT: Mask.getSimpleValueType(), VL, DL, DAG), N3: VL);
14398 }
14399
14400 unsigned IntID =
14401 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
14402 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
14403 Ops.push_back(Elt: Val);
14404 Ops.push_back(Elt: BasePtr);
14405 if (!IsUnmasked)
14406 Ops.push_back(Elt: Mask);
14407 Ops.push_back(Elt: VL);
14408
14409 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
14410 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
14411}
14412
14413SDValue RISCVTargetLowering::lowerVectorCompress(SDValue Op,
14414 SelectionDAG &DAG) const {
14415 SDLoc DL(Op);
14416 SDValue Val = Op.getOperand(i: 0);
14417 SDValue Mask = Op.getOperand(i: 1);
14418 SDValue Passthru = Op.getOperand(i: 2);
14419
14420 MVT VT = Val.getSimpleValueType();
14421 MVT XLenVT = Subtarget.getXLenVT();
14422 MVT ContainerVT = VT;
14423 if (VT.isFixedLengthVector()) {
14424 ContainerVT = getContainerForFixedLengthVector(VT);
14425 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14426 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
14427 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14428 Passthru = convertToScalableVector(VT: ContainerVT, V: Passthru, DAG, Subtarget);
14429 }
14430
14431 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
14432 SDValue Res =
14433 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ContainerVT,
14434 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_vcompress, DL, VT: XLenVT),
14435 N2: Passthru, N3: Val, N4: Mask, N5: VL);
14436
14437 if (VT.isFixedLengthVector())
14438 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
14439
14440 return Res;
14441}
14442
14443SDValue RISCVTargetLowering::lowerVectorStrictFSetcc(SDValue Op,
14444 SelectionDAG &DAG) const {
14445 unsigned Opc = Op.getOpcode();
14446 SDLoc DL(Op);
14447 SDValue Chain = Op.getOperand(i: 0);
14448 SDValue Op1 = Op.getOperand(i: 1);
14449 SDValue Op2 = Op.getOperand(i: 2);
14450 SDValue CC = Op.getOperand(i: 3);
14451 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
14452 MVT VT = Op.getSimpleValueType();
14453 MVT InVT = Op1.getSimpleValueType();
14454
14455 // RVV VMFEQ/VMFNE ignores qNan, so we expand strict_fsetccs with OEQ/UNE
14456 // condition code.
14457 if (Opc == ISD::STRICT_FSETCCS) {
14458 // Expand strict_fsetccs(x, oeq) to
14459 // (and strict_fsetccs(x, y, oge), strict_fsetccs(x, y, ole))
14460 SDVTList VTList = Op->getVTList();
14461 if (CCVal == ISD::SETEQ || CCVal == ISD::SETOEQ) {
14462 SDValue OLECCVal = DAG.getCondCode(Cond: ISD::SETOLE);
14463 SDValue Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op1,
14464 N3: Op2, N4: OLECCVal);
14465 SDValue Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op2,
14466 N3: Op1, N4: OLECCVal);
14467 SDValue OutChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
14468 N1: Tmp1.getValue(R: 1), N2: Tmp2.getValue(R: 1));
14469 // Tmp1 and Tmp2 might be the same node.
14470 if (Tmp1 != Tmp2)
14471 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp1, N2: Tmp2);
14472 return DAG.getMergeValues(Ops: {Tmp1, OutChain}, dl: DL);
14473 }
14474
14475 // Expand (strict_fsetccs x, y, une) to (not (strict_fsetccs x, y, oeq))
14476 if (CCVal == ISD::SETNE || CCVal == ISD::SETUNE) {
14477 SDValue OEQCCVal = DAG.getCondCode(Cond: ISD::SETOEQ);
14478 SDValue OEQ = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op1,
14479 N3: Op2, N4: OEQCCVal);
14480 SDValue Res = DAG.getNOT(DL, Val: OEQ, VT);
14481 return DAG.getMergeValues(Ops: {Res, OEQ.getValue(R: 1)}, dl: DL);
14482 }
14483 }
14484
14485 MVT ContainerInVT = InVT;
14486 if (InVT.isFixedLengthVector()) {
14487 ContainerInVT = getContainerForFixedLengthVector(VT: InVT);
14488 Op1 = convertToScalableVector(VT: ContainerInVT, V: Op1, DAG, Subtarget);
14489 Op2 = convertToScalableVector(VT: ContainerInVT, V: Op2, DAG, Subtarget);
14490 }
14491 MVT MaskVT = getMaskTypeFor(VecVT: ContainerInVT);
14492
14493 auto [Mask, VL] = getDefaultVLOps(VecVT: InVT, ContainerVT: ContainerInVT, DL, DAG, Subtarget);
14494
14495 SDValue Res;
14496 if (Opc == ISD::STRICT_FSETCC &&
14497 (CCVal == ISD::SETLT || CCVal == ISD::SETOLT || CCVal == ISD::SETLE ||
14498 CCVal == ISD::SETOLE)) {
14499 // VMFLT/VMFLE/VMFGT/VMFGE raise exception for qNan. Generate a mask to only
14500 // active when both input elements are ordered.
14501 SDValue True = getAllOnesMask(VecVT: ContainerInVT, VL, DL, DAG);
14502 SDValue OrderMask1 = DAG.getNode(
14503 Opcode: RISCVISD::STRICT_FSETCC_VL, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
14504 Ops: {Chain, Op1, Op1, DAG.getCondCode(Cond: ISD::SETOEQ), DAG.getUNDEF(VT: MaskVT),
14505 True, VL});
14506 SDValue OrderMask2 = DAG.getNode(
14507 Opcode: RISCVISD::STRICT_FSETCC_VL, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
14508 Ops: {Chain, Op2, Op2, DAG.getCondCode(Cond: ISD::SETOEQ), DAG.getUNDEF(VT: MaskVT),
14509 True, VL});
14510 Mask =
14511 DAG.getNode(Opcode: RISCVISD::VMAND_VL, DL, VT: MaskVT, N1: OrderMask1, N2: OrderMask2, N3: VL);
14512 // Use Mask as the passthru operand to let the result be 0 if either of the
14513 // inputs is unordered.
14514 Res = DAG.getNode(Opcode: RISCVISD::STRICT_FSETCCS_VL, DL,
14515 VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
14516 Ops: {Chain, Op1, Op2, CC, Mask, Mask, VL});
14517 } else {
14518 unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL
14519 : RISCVISD::STRICT_FSETCCS_VL;
14520 Res = DAG.getNode(Opcode: RVVOpc, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
14521 Ops: {Chain, Op1, Op2, CC, DAG.getUNDEF(VT: MaskVT), Mask, VL});
14522 }
14523
14524 if (VT.isFixedLengthVector()) {
14525 SDValue SubVec = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
14526 return DAG.getMergeValues(Ops: {SubVec, Res.getValue(R: 1)}, dl: DL);
14527 }
14528 return Res;
14529}
14530
14531// Lower vector ABS to smax(X, sub(0, X)).
14532SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
14533 SDLoc DL(Op);
14534 MVT VT = Op.getSimpleValueType();
14535 SDValue X = Op.getOperand(i: 0);
14536
14537 assert(VT.isFixedLengthVector() && "Unexpected type for ISD::ABS");
14538
14539 MVT ContainerVT = getContainerForFixedLengthVector(VT);
14540 X = convertToScalableVector(VT: ContainerVT, V: X, DAG, Subtarget);
14541
14542 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
14543
14544 SDValue Result;
14545 if (Subtarget.hasStdExtZvabd()) {
14546 Result = DAG.getNode(Opcode: RISCVISD::ABS_VL, DL, VT: ContainerVT, N1: X,
14547 N2: DAG.getUNDEF(VT: ContainerVT), N3: Mask, N4: VL);
14548 } else {
14549 SDValue SplatZero = DAG.getNode(
14550 Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT),
14551 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()), N3: VL);
14552 SDValue NegX = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: ContainerVT, N1: SplatZero, N2: X,
14553 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
14554 Result = DAG.getNode(Opcode: RISCVISD::SMAX_VL, DL, VT: ContainerVT, N1: X, N2: NegX,
14555 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
14556 }
14557 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14558}
14559
14560SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op,
14561 SelectionDAG &DAG) const {
14562 const auto &TSInfo =
14563 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
14564
14565 unsigned NewOpc = getRISCVVLOp(Op);
14566 bool HasPassthruOp = TSInfo.hasPassthruOp(Opcode: NewOpc);
14567 bool HasMask = TSInfo.hasMaskOp(Opcode: NewOpc);
14568
14569 MVT VT = Op.getSimpleValueType();
14570 MVT ContainerVT = getContainerForFixedLengthVector(VT);
14571
14572 // Create list of operands by converting existing ones to scalable types.
14573 SmallVector<SDValue, 6> Ops;
14574 for (const SDValue &V : Op->op_values()) {
14575 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
14576
14577 // Pass through non-vector operands.
14578 if (!V.getValueType().isVector()) {
14579 Ops.push_back(Elt: V);
14580 continue;
14581 }
14582
14583 // "cast" fixed length vector to a scalable vector.
14584 assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
14585 "Only fixed length vectors are supported!");
14586 MVT VContainerVT = ContainerVT.changeVectorElementType(
14587 EltVT: V.getSimpleValueType().getVectorElementType());
14588 Ops.push_back(Elt: convertToScalableVector(VT: VContainerVT, V, DAG, Subtarget));
14589 }
14590
14591 SDLoc DL(Op);
14592 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
14593 if (HasPassthruOp)
14594 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14595 if (HasMask)
14596 Ops.push_back(Elt: Mask);
14597 Ops.push_back(Elt: VL);
14598
14599 // StrictFP operations have two result values. Their lowered result should
14600 // have same result count.
14601 if (Op->isStrictFPOpcode()) {
14602 SDValue ScalableRes =
14603 DAG.getNode(Opcode: NewOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), Ops,
14604 Flags: Op->getFlags());
14605 SDValue SubVec = convertFromScalableVector(VT, V: ScalableRes, DAG, Subtarget);
14606 return DAG.getMergeValues(Ops: {SubVec, ScalableRes.getValue(R: 1)}, dl: DL);
14607 }
14608
14609 SDValue ScalableRes =
14610 DAG.getNode(Opcode: NewOpc, DL, VT: ContainerVT, Ops, Flags: Op->getFlags());
14611 return convertFromScalableVector(VT, V: ScalableRes, DAG, Subtarget);
14612}
14613
14614// Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
14615// * Operands of each node are assumed to be in the same order.
14616// * The EVL operand is promoted from i32 to i64 on RV64.
14617// * Fixed-length vectors are converted to their scalable-vector container
14618// types.
14619SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG) const {
14620 const auto &TSInfo =
14621 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
14622
14623 unsigned RISCVISDOpc = getRISCVVLOp(Op);
14624 bool HasPassthruOp = TSInfo.hasPassthruOp(Opcode: RISCVISDOpc);
14625
14626 SDLoc DL(Op);
14627 MVT VT = Op.getSimpleValueType();
14628 SmallVector<SDValue, 4> Ops;
14629
14630 MVT ContainerVT = VT;
14631 if (VT.isFixedLengthVector())
14632 ContainerVT = getContainerForFixedLengthVector(VT);
14633
14634 for (const auto &OpIdx : enumerate(First: Op->ops())) {
14635 SDValue V = OpIdx.value();
14636 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
14637 // Add dummy passthru value before the mask. Or if there isn't a mask,
14638 // before EVL.
14639 if (HasPassthruOp) {
14640 auto MaskIdx = ISD::getVPMaskIdx(Opcode: Op.getOpcode());
14641 if (MaskIdx) {
14642 if (*MaskIdx == OpIdx.index())
14643 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14644 } else if (ISD::getVPExplicitVectorLengthIdx(Opcode: Op.getOpcode()) ==
14645 OpIdx.index()) {
14646 assert(Op.getOpcode() == ISD::VP_MERGE);
14647 // For VP_MERGE, copy the false operand instead of an undef value.
14648 Ops.push_back(Elt: Ops.back());
14649 }
14650 }
14651 // VFCVT_RM_X_F_VL requires a rounding mode to be injected before the VL.
14652 if (RISCVISDOpc == RISCVISD::VFCVT_RM_X_F_VL &&
14653 ISD::getVPExplicitVectorLengthIdx(Opcode: Op.getOpcode()) == OpIdx.index())
14654 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVFPRndMode::DYN, DL,
14655 VT: Subtarget.getXLenVT()));
14656 // Pass through operands which aren't fixed-length vectors.
14657 if (!V.getValueType().isFixedLengthVector()) {
14658 Ops.push_back(Elt: V);
14659 continue;
14660 }
14661 // "cast" fixed length vector to a scalable vector.
14662 MVT OpVT = V.getSimpleValueType();
14663 MVT ContainerVT = getContainerForFixedLengthVector(VT: OpVT);
14664 assert(useRVVForFixedLengthVectorVT(OpVT) &&
14665 "Only fixed length vectors are supported!");
14666 Ops.push_back(Elt: convertToScalableVector(VT: ContainerVT, V, DAG, Subtarget));
14667 }
14668
14669 if (!VT.isFixedLengthVector())
14670 return DAG.getNode(Opcode: RISCVISDOpc, DL, VT, Ops, Flags: Op->getFlags());
14671
14672 SDValue VPOp = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: ContainerVT, Ops, Flags: Op->getFlags());
14673
14674 return convertFromScalableVector(VT, V: VPOp, DAG, Subtarget);
14675}
14676
14677SDValue RISCVTargetLowering::lowerVPMergeMask(SDValue Op,
14678 SelectionDAG &DAG) const {
14679 SDLoc DL(Op);
14680 MVT VT = Op.getSimpleValueType();
14681 MVT XLenVT = Subtarget.getXLenVT();
14682
14683 SDValue Mask = Op.getOperand(i: 0);
14684 SDValue TrueVal = Op.getOperand(i: 1);
14685 SDValue FalseVal = Op.getOperand(i: 2);
14686 SDValue VL = Op.getOperand(i: 3);
14687
14688 // Use default legalization if a vector of EVL type would be legal.
14689 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VL.getValueType(),
14690 EC: VT.getVectorElementCount());
14691 if (isTypeLegal(VT: EVLVecVT))
14692 return SDValue();
14693
14694 MVT ContainerVT = VT;
14695 if (VT.isFixedLengthVector()) {
14696 ContainerVT = getContainerForFixedLengthVector(VT);
14697 Mask = convertToScalableVector(VT: ContainerVT, V: Mask, DAG, Subtarget);
14698 TrueVal = convertToScalableVector(VT: ContainerVT, V: TrueVal, DAG, Subtarget);
14699 FalseVal = convertToScalableVector(VT: ContainerVT, V: FalseVal, DAG, Subtarget);
14700 }
14701
14702 // Promote to a vector of i8.
14703 MVT PromotedVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
14704
14705 // Promote TrueVal and FalseVal using VLMax.
14706 // FIXME: Is there a better way to do this?
14707 SDValue VLMax = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
14708 SDValue SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: PromotedVT,
14709 N1: DAG.getUNDEF(VT: PromotedVT),
14710 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: VLMax);
14711 SDValue SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: PromotedVT,
14712 N1: DAG.getUNDEF(VT: PromotedVT),
14713 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: VLMax);
14714 TrueVal = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: TrueVal, N2: SplatOne,
14715 N3: SplatZero, N4: DAG.getUNDEF(VT: PromotedVT), N5: VL);
14716 // Any element past VL uses FalseVal, so use VLMax
14717 FalseVal = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: FalseVal,
14718 N2: SplatOne, N3: SplatZero, N4: DAG.getUNDEF(VT: PromotedVT), N5: VLMax);
14719
14720 // VP_MERGE the two promoted values.
14721 SDValue VPMerge = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: Mask,
14722 N2: TrueVal, N3: FalseVal, N4: FalseVal, N5: VL);
14723
14724 // Convert back to mask.
14725 SDValue TrueMask = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
14726 SDValue Result = DAG.getNode(
14727 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
14728 Ops: {VPMerge, DAG.getConstant(Val: 0, DL, VT: PromotedVT), DAG.getCondCode(Cond: ISD::SETNE),
14729 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), TrueMask, VLMax});
14730
14731 if (VT.isFixedLengthVector())
14732 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14733 return Result;
14734}
14735
14736SDValue
14737RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
14738 SelectionDAG &DAG) const {
14739 using namespace SDPatternMatch;
14740
14741 SDLoc DL(Op);
14742
14743 SDValue Op1 = Op.getOperand(i: 0);
14744 SDValue Op2 = Op.getOperand(i: 1);
14745 SDValue Offset = Op.getOperand(i: 2);
14746 SDValue Mask = Op.getOperand(i: 3);
14747 SDValue EVL1 = Op.getOperand(i: 4);
14748 SDValue EVL2 = Op.getOperand(i: 5);
14749
14750 const MVT XLenVT = Subtarget.getXLenVT();
14751 MVT VT = Op.getSimpleValueType();
14752 MVT ContainerVT = VT;
14753 if (VT.isFixedLengthVector()) {
14754 ContainerVT = getContainerForFixedLengthVector(VT);
14755 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
14756 Op2 = convertToScalableVector(VT: ContainerVT, V: Op2, DAG, Subtarget);
14757 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14758 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14759 }
14760
14761 bool IsMaskVector = VT.getVectorElementType() == MVT::i1;
14762 if (IsMaskVector) {
14763 ContainerVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
14764
14765 // Expand input operands
14766 SDValue SplatOneOp1 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14767 N1: DAG.getUNDEF(VT: ContainerVT),
14768 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL1);
14769 SDValue SplatZeroOp1 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14770 N1: DAG.getUNDEF(VT: ContainerVT),
14771 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL1);
14772 Op1 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: Op1, N2: SplatOneOp1,
14773 N3: SplatZeroOp1, N4: DAG.getUNDEF(VT: ContainerVT), N5: EVL1);
14774
14775 SDValue SplatOneOp2 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14776 N1: DAG.getUNDEF(VT: ContainerVT),
14777 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL2);
14778 SDValue SplatZeroOp2 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14779 N1: DAG.getUNDEF(VT: ContainerVT),
14780 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL2);
14781 Op2 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: Op2, N2: SplatOneOp2,
14782 N3: SplatZeroOp2, N4: DAG.getUNDEF(VT: ContainerVT), N5: EVL2);
14783 }
14784
14785 auto getVectorFirstEle = [](SDValue Vec) {
14786 SDValue FirstEle;
14787 if (sd_match(N: Vec, P: m_InsertElt(Vec: m_Value(), Val: m_Value(N&: FirstEle), Idx: m_Zero())))
14788 return FirstEle;
14789
14790 if (Vec.getOpcode() == ISD::SPLAT_VECTOR ||
14791 Vec.getOpcode() == ISD::BUILD_VECTOR)
14792 return Vec.getOperand(i: 0);
14793
14794 return SDValue();
14795 };
14796
14797 if (!IsMaskVector && isNullConstant(V: Offset) && isOneConstant(V: EVL1))
14798 if (auto FirstEle = getVectorFirstEle(Op->getOperand(Num: 0))) {
14799 MVT EltVT = ContainerVT.getVectorElementType();
14800 SDValue Result;
14801 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
14802 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
14803 EltVT = EltVT.changeTypeToInteger();
14804 ContainerVT = ContainerVT.changeVectorElementType(EltVT);
14805 Op2 = DAG.getBitcast(VT: ContainerVT, V: Op2);
14806 FirstEle =
14807 DAG.getAnyExtOrTrunc(Op: DAG.getBitcast(VT: EltVT, V: FirstEle), DL, VT: XLenVT);
14808 }
14809 Result = DAG.getNode(Opcode: EltVT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL
14810 : RISCVISD::VSLIDE1UP_VL,
14811 DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Op2,
14812 N3: FirstEle, N4: Mask, N5: EVL2);
14813 Result = DAG.getBitcast(
14814 VT: ContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType()),
14815 V: Result);
14816 return VT.isFixedLengthVector()
14817 ? convertFromScalableVector(VT, V: Result, DAG, Subtarget)
14818 : Result;
14819 }
14820
14821 int64_t ImmValue = cast<ConstantSDNode>(Val&: Offset)->getSExtValue();
14822 SDValue DownOffset, UpOffset;
14823 if (ImmValue >= 0) {
14824 // The operand is a TargetConstant, we need to rebuild it as a regular
14825 // constant.
14826 DownOffset = DAG.getConstant(Val: ImmValue, DL, VT: XLenVT);
14827 UpOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL1, N2: DownOffset);
14828 } else {
14829 // The operand is a TargetConstant, we need to rebuild it as a regular
14830 // constant rather than negating the original operand.
14831 UpOffset = DAG.getConstant(Val: -ImmValue, DL, VT: XLenVT);
14832 DownOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL1, N2: UpOffset);
14833 }
14834
14835 if (ImmValue != 0)
14836 Op1 = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
14837 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Op1, Offset: DownOffset, Mask,
14838 VL: Subtarget.hasVLDependentLatency() ? UpOffset : EVL2);
14839 SDValue Result = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Op1, Op: Op2,
14840 Offset: UpOffset, Mask, VL: EVL2, Policy: RISCVVType::TAIL_AGNOSTIC);
14841
14842 if (IsMaskVector) {
14843 // Truncate Result back to a mask vector (Result has same EVL as Op2)
14844 Result = DAG.getNode(
14845 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT.changeVectorElementType(EltVT: MVT::i1),
14846 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: ContainerVT),
14847 DAG.getCondCode(Cond: ISD::SETNE), DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)),
14848 Mask, EVL2});
14849 }
14850
14851 if (!VT.isFixedLengthVector())
14852 return Result;
14853 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14854}
14855
14856SDValue
14857RISCVTargetLowering::lowerVPReverseExperimental(SDValue Op,
14858 SelectionDAG &DAG) const {
14859 SDLoc DL(Op);
14860 MVT VT = Op.getSimpleValueType();
14861 MVT XLenVT = Subtarget.getXLenVT();
14862
14863 SDValue Op1 = Op.getOperand(i: 0);
14864 SDValue Mask = Op.getOperand(i: 1);
14865 SDValue EVL = Op.getOperand(i: 2);
14866
14867 MVT ContainerVT = VT;
14868 if (VT.isFixedLengthVector()) {
14869 ContainerVT = getContainerForFixedLengthVector(VT);
14870 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
14871 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14872 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14873 }
14874
14875 MVT GatherVT = ContainerVT;
14876 MVT IndicesVT = ContainerVT.changeVectorElementTypeToInteger();
14877 // Check if we are working with mask vectors
14878 bool IsMaskVector = ContainerVT.getVectorElementType() == MVT::i1;
14879 if (IsMaskVector) {
14880 GatherVT = IndicesVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
14881
14882 // Expand input operand
14883 SDValue SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
14884 N1: DAG.getUNDEF(VT: IndicesVT),
14885 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL);
14886 SDValue SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
14887 N1: DAG.getUNDEF(VT: IndicesVT),
14888 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL);
14889 Op1 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: IndicesVT, N1: Op1, N2: SplatOne,
14890 N3: SplatZero, N4: DAG.getUNDEF(VT: IndicesVT), N5: EVL);
14891 }
14892
14893 unsigned EltSize = GatherVT.getScalarSizeInBits();
14894 unsigned MinSize = GatherVT.getSizeInBits().getKnownMinValue();
14895 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
14896 unsigned MaxVLMAX =
14897 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
14898
14899 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
14900 // If this is SEW=8 and VLMAX is unknown or more than 256, we need
14901 // to use vrgatherei16.vv.
14902 // TODO: It's also possible to use vrgatherei16.vv for other types to
14903 // decrease register width for the index calculation.
14904 // NOTE: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
14905 if (MaxVLMAX > 256 && EltSize == 8) {
14906 // If this is LMUL=8, we have to split before using vrgatherei16.vv.
14907 // Split the vector in half and reverse each half using a full register
14908 // reverse.
14909 // Swap the halves and concatenate them.
14910 // Slide the concatenated result by (VLMax - VL).
14911 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
14912 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: GatherVT);
14913 auto [Lo, Hi] = DAG.SplitVector(N: Op1, DL);
14914
14915 SDValue LoRev = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: LoVT, Operand: Lo);
14916 SDValue HiRev = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: HiVT, Operand: Hi);
14917
14918 // Reassemble the low and high pieces reversed.
14919 // NOTE: this Result is unmasked (because we do not need masks for
14920 // shuffles). If in the future this has to change, we can use a SELECT_VL
14921 // between Result and UNDEF using the mask originally passed to VP_REVERSE
14922 SDValue Result =
14923 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: GatherVT, N1: HiRev, N2: LoRev);
14924
14925 // Slide off any elements from past EVL that were reversed into the low
14926 // elements.
14927 SDValue VLMax =
14928 DAG.getElementCount(DL, VT: XLenVT, EC: GatherVT.getVectorElementCount());
14929 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: EVL);
14930
14931 Result = getVSlidedown(DAG, Subtarget, DL, VT: GatherVT,
14932 Passthru: DAG.getUNDEF(VT: GatherVT), Op: Result, Offset: Diff, Mask, VL: EVL);
14933
14934 if (IsMaskVector) {
14935 // Truncate Result back to a mask vector
14936 Result =
14937 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
14938 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: GatherVT),
14939 DAG.getCondCode(Cond: ISD::SETNE),
14940 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), Mask, EVL});
14941 }
14942
14943 if (!VT.isFixedLengthVector())
14944 return Result;
14945 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14946 }
14947
14948 // Just promote the int type to i16 which will double the LMUL.
14949 IndicesVT = MVT::getVectorVT(VT: MVT::i16, EC: IndicesVT.getVectorElementCount());
14950 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
14951 }
14952
14953 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: IndicesVT, N1: Mask, N2: EVL);
14954 SDValue VecLen =
14955 DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
14956 SDValue VecLenSplat = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
14957 N1: DAG.getUNDEF(VT: IndicesVT), N2: VecLen, N3: EVL);
14958 SDValue VRSUB = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: IndicesVT, N1: VecLenSplat, N2: VID,
14959 N3: DAG.getUNDEF(VT: IndicesVT), N4: Mask, N5: EVL);
14960 SDValue Result = DAG.getNode(Opcode: GatherOpc, DL, VT: GatherVT, N1: Op1, N2: VRSUB,
14961 N3: DAG.getUNDEF(VT: GatherVT), N4: Mask, N5: EVL);
14962
14963 if (IsMaskVector) {
14964 // Truncate Result back to a mask vector
14965 Result = DAG.getNode(
14966 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
14967 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: GatherVT), DAG.getCondCode(Cond: ISD::SETNE),
14968 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), Mask, EVL});
14969 }
14970
14971 if (!VT.isFixedLengthVector())
14972 return Result;
14973 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14974}
14975
14976SDValue RISCVTargetLowering::lowerVPStridedLoad(SDValue Op,
14977 SelectionDAG &DAG) const {
14978 SDLoc DL(Op);
14979 MVT XLenVT = Subtarget.getXLenVT();
14980 MVT VT = Op.getSimpleValueType();
14981 MVT ContainerVT = VT;
14982 if (VT.isFixedLengthVector())
14983 ContainerVT = getContainerForFixedLengthVector(VT);
14984
14985 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
14986
14987 auto *VPNode = cast<VPStridedLoadSDNode>(Val&: Op);
14988 // Check if the mask is known to be all ones
14989 SDValue Mask = VPNode->getMask();
14990 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
14991
14992 SDValue IntID = DAG.getTargetConstant(Val: IsUnmasked ? Intrinsic::riscv_vlse
14993 : Intrinsic::riscv_vlse_mask,
14994 DL, VT: XLenVT);
14995 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID,
14996 DAG.getUNDEF(VT: ContainerVT), VPNode->getBasePtr(),
14997 VPNode->getStride()};
14998 if (!IsUnmasked) {
14999 if (VT.isFixedLengthVector()) {
15000 MVT MaskVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
15001 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15002 }
15003 Ops.push_back(Elt: Mask);
15004 }
15005 Ops.push_back(Elt: VPNode->getVectorLength());
15006 if (!IsUnmasked) {
15007 SDValue Policy =
15008 DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
15009 Ops.push_back(Elt: Policy);
15010 }
15011
15012 SDValue Result =
15013 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
15014 MemVT: VPNode->getMemoryVT(), MMO: VPNode->getMemOperand());
15015 SDValue Chain = Result.getValue(R: 1);
15016
15017 if (VT.isFixedLengthVector())
15018 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
15019
15020 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
15021}
15022
15023SDValue RISCVTargetLowering::lowerVPStridedStore(SDValue Op,
15024 SelectionDAG &DAG) const {
15025 SDLoc DL(Op);
15026 MVT XLenVT = Subtarget.getXLenVT();
15027
15028 auto *VPNode = cast<VPStridedStoreSDNode>(Val&: Op);
15029 SDValue StoreVal = VPNode->getValue();
15030 MVT VT = StoreVal.getSimpleValueType();
15031 MVT ContainerVT = VT;
15032 if (VT.isFixedLengthVector()) {
15033 ContainerVT = getContainerForFixedLengthVector(VT);
15034 StoreVal = convertToScalableVector(VT: ContainerVT, V: StoreVal, DAG, Subtarget);
15035 }
15036
15037 // Check if the mask is known to be all ones
15038 SDValue Mask = VPNode->getMask();
15039 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
15040
15041 SDValue IntID = DAG.getTargetConstant(Val: IsUnmasked ? Intrinsic::riscv_vsse
15042 : Intrinsic::riscv_vsse_mask,
15043 DL, VT: XLenVT);
15044 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID, StoreVal,
15045 VPNode->getBasePtr(), VPNode->getStride()};
15046 if (!IsUnmasked) {
15047 if (VT.isFixedLengthVector()) {
15048 MVT MaskVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
15049 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15050 }
15051 Ops.push_back(Elt: Mask);
15052 }
15053 Ops.push_back(Elt: VPNode->getVectorLength());
15054
15055 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: VPNode->getVTList(),
15056 Ops, MemVT: VPNode->getMemoryVT(),
15057 MMO: VPNode->getMemOperand());
15058}
15059
15060// Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
15061// matched to a RVV indexed load. The RVV indexed load instructions only
15062// support the "unsigned unscaled" addressing mode; indices are implicitly
15063// zero-extended or truncated to XLEN and are treated as byte offsets. Any
15064// signed or scaled indexing is extended to the XLEN value type and scaled
15065// accordingly.
15066SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
15067 SelectionDAG &DAG) const {
15068 SDLoc DL(Op);
15069 MVT VT = Op.getSimpleValueType();
15070
15071 const auto *MemSD = cast<MemSDNode>(Val: Op.getNode());
15072 EVT MemVT = MemSD->getMemoryVT();
15073 MachineMemOperand *MMO = MemSD->getMemOperand();
15074 SDValue Chain = MemSD->getChain();
15075 SDValue BasePtr = MemSD->getBasePtr();
15076
15077 [[maybe_unused]] ISD::LoadExtType LoadExtType;
15078 SDValue Index, Mask, PassThru, VL;
15079
15080 if (auto *VPGN = dyn_cast<VPGatherSDNode>(Val: Op.getNode())) {
15081 Index = VPGN->getIndex();
15082 Mask = VPGN->getMask();
15083 PassThru = DAG.getUNDEF(VT);
15084 VL = VPGN->getVectorLength();
15085 // VP doesn't support extending loads.
15086 LoadExtType = ISD::NON_EXTLOAD;
15087 } else {
15088 // Else it must be a MGATHER.
15089 auto *MGN = cast<MaskedGatherSDNode>(Val: Op.getNode());
15090 Index = MGN->getIndex();
15091 Mask = MGN->getMask();
15092 PassThru = MGN->getPassThru();
15093 LoadExtType = MGN->getExtensionType();
15094 }
15095
15096 MVT IndexVT = Index.getSimpleValueType();
15097 MVT XLenVT = Subtarget.getXLenVT();
15098
15099 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
15100 "Unexpected VTs!");
15101 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
15102 // Targets have to explicitly opt-in for extending vector loads.
15103 assert(LoadExtType == ISD::NON_EXTLOAD &&
15104 "Unexpected extending MGATHER/VP_GATHER");
15105
15106 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
15107 // the selection of the masked intrinsics doesn't do this for us.
15108 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
15109
15110 MVT ContainerVT = VT;
15111 if (VT.isFixedLengthVector()) {
15112 ContainerVT = getContainerForFixedLengthVector(VT);
15113 IndexVT = MVT::getVectorVT(VT: IndexVT.getVectorElementType(),
15114 EC: ContainerVT.getVectorElementCount());
15115
15116 Index = convertToScalableVector(VT: IndexVT, V: Index, DAG, Subtarget);
15117
15118 if (!IsUnmasked) {
15119 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15120 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15121 PassThru = convertToScalableVector(VT: ContainerVT, V: PassThru, DAG, Subtarget);
15122 }
15123 }
15124
15125 if (!VL)
15126 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
15127
15128 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(VT: XLenVT)) {
15129 IndexVT = IndexVT.changeVectorElementType(EltVT: XLenVT);
15130 Index = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IndexVT, Operand: Index);
15131 }
15132
15133 unsigned IntID =
15134 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
15135 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
15136 if (IsUnmasked)
15137 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
15138 else
15139 Ops.push_back(Elt: PassThru);
15140 Ops.push_back(Elt: BasePtr);
15141 Ops.push_back(Elt: Index);
15142 if (!IsUnmasked)
15143 Ops.push_back(Elt: Mask);
15144 Ops.push_back(Elt: VL);
15145 if (!IsUnmasked)
15146 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT));
15147
15148 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
15149 SDValue Result =
15150 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
15151 Chain = Result.getValue(R: 1);
15152
15153 if (VT.isFixedLengthVector())
15154 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
15155
15156 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
15157}
15158
15159// Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
15160// matched to a RVV indexed store. The RVV indexed store instructions only
15161// support the "unsigned unscaled" addressing mode; indices are implicitly
15162// zero-extended or truncated to XLEN and are treated as byte offsets. Any
15163// signed or scaled indexing is extended to the XLEN value type and scaled
15164// accordingly.
15165SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
15166 SelectionDAG &DAG) const {
15167 SDLoc DL(Op);
15168 const auto *MemSD = cast<MemSDNode>(Val: Op.getNode());
15169 EVT MemVT = MemSD->getMemoryVT();
15170 MachineMemOperand *MMO = MemSD->getMemOperand();
15171 SDValue Chain = MemSD->getChain();
15172 SDValue BasePtr = MemSD->getBasePtr();
15173
15174 [[maybe_unused]] bool IsTruncatingStore = false;
15175 SDValue Index, Mask, Val, VL;
15176
15177 if (auto *VPSN = dyn_cast<VPScatterSDNode>(Val: Op.getNode())) {
15178 Index = VPSN->getIndex();
15179 Mask = VPSN->getMask();
15180 Val = VPSN->getValue();
15181 VL = VPSN->getVectorLength();
15182 // VP doesn't support truncating stores.
15183 IsTruncatingStore = false;
15184 } else {
15185 // Else it must be a MSCATTER.
15186 auto *MSN = cast<MaskedScatterSDNode>(Val: Op.getNode());
15187 Index = MSN->getIndex();
15188 Mask = MSN->getMask();
15189 Val = MSN->getValue();
15190 IsTruncatingStore = MSN->isTruncatingStore();
15191 }
15192
15193 MVT VT = Val.getSimpleValueType();
15194 MVT IndexVT = Index.getSimpleValueType();
15195 MVT XLenVT = Subtarget.getXLenVT();
15196
15197 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
15198 "Unexpected VTs!");
15199 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
15200 // Targets have to explicitly opt-in for extending vector loads and
15201 // truncating vector stores.
15202 assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
15203
15204 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
15205 // the selection of the masked intrinsics doesn't do this for us.
15206 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
15207
15208 MVT ContainerVT = VT;
15209 if (VT.isFixedLengthVector()) {
15210 ContainerVT = getContainerForFixedLengthVector(VT);
15211 IndexVT = MVT::getVectorVT(VT: IndexVT.getVectorElementType(),
15212 EC: ContainerVT.getVectorElementCount());
15213
15214 Index = convertToScalableVector(VT: IndexVT, V: Index, DAG, Subtarget);
15215 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
15216
15217 if (!IsUnmasked) {
15218 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
15219 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
15220 }
15221 }
15222
15223 if (!VL)
15224 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
15225
15226 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(VT: XLenVT)) {
15227 IndexVT = IndexVT.changeVectorElementType(EltVT: XLenVT);
15228 Index = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IndexVT, Operand: Index);
15229 }
15230
15231 unsigned IntID =
15232 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
15233 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
15234 Ops.push_back(Elt: Val);
15235 Ops.push_back(Elt: BasePtr);
15236 Ops.push_back(Elt: Index);
15237 if (!IsUnmasked)
15238 Ops.push_back(Elt: Mask);
15239 Ops.push_back(Elt: VL);
15240
15241 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
15242 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
15243}
15244
15245SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
15246 SelectionDAG &DAG) const {
15247 const MVT XLenVT = Subtarget.getXLenVT();
15248 SDLoc DL(Op);
15249 SDValue Chain = Op->getOperand(Num: 0);
15250 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::frm, DL, VT: XLenVT);
15251 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
15252 SDValue RM = DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
15253
15254 // Encoding used for rounding mode in RISC-V differs from that used in
15255 // FLT_ROUNDS. To convert it the RISC-V rounding mode is used as an index in a
15256 // table, which consists of a sequence of 4-bit fields, each representing
15257 // corresponding FLT_ROUNDS mode.
15258 static const int Table =
15259 (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
15260 (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
15261 (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
15262 (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
15263 (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
15264
15265 SDValue Shift =
15266 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: RM, N2: DAG.getConstant(Val: 2, DL, VT: XLenVT));
15267 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT,
15268 N1: DAG.getConstant(Val: Table, DL, VT: XLenVT), N2: Shift);
15269 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Shifted,
15270 N2: DAG.getConstant(Val: 7, DL, VT: XLenVT));
15271
15272 return DAG.getMergeValues(Ops: {Masked, Chain}, dl: DL);
15273}
15274
15275SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
15276 SelectionDAG &DAG) const {
15277 const MVT XLenVT = Subtarget.getXLenVT();
15278 SDLoc DL(Op);
15279 SDValue Chain = Op->getOperand(Num: 0);
15280 SDValue RMValue = Op->getOperand(Num: 1);
15281 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::frm, DL, VT: XLenVT);
15282
15283 // Encoding used for rounding mode in RISC-V differs from that used in
15284 // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
15285 // a table, which consists of a sequence of 4-bit fields, each representing
15286 // corresponding RISC-V mode.
15287 static const unsigned Table =
15288 (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
15289 (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
15290 (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
15291 (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
15292 (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
15293
15294 RMValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: RMValue);
15295
15296 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: RMValue,
15297 N2: DAG.getConstant(Val: 2, DL, VT: XLenVT));
15298 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT,
15299 N1: DAG.getConstant(Val: Table, DL, VT: XLenVT), N2: Shift);
15300 RMValue = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Shifted,
15301 N2: DAG.getConstant(Val: 0x7, DL, VT: XLenVT));
15302 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15303 N3: RMValue);
15304}
15305
15306SDValue RISCVTargetLowering::lowerGET_FPENV(SDValue Op,
15307 SelectionDAG &DAG) const {
15308 const MVT XLenVT = Subtarget.getXLenVT();
15309 SDLoc DL(Op);
15310 SDValue Chain = Op->getOperand(Num: 0);
15311 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15312 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
15313 return DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
15314}
15315
15316SDValue RISCVTargetLowering::lowerSET_FPENV(SDValue Op,
15317 SelectionDAG &DAG) const {
15318 const MVT XLenVT = Subtarget.getXLenVT();
15319 SDLoc DL(Op);
15320 SDValue Chain = Op->getOperand(Num: 0);
15321 SDValue EnvValue = Op->getOperand(Num: 1);
15322 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15323
15324 EnvValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: EnvValue);
15325 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15326 N3: EnvValue);
15327}
15328
15329SDValue RISCVTargetLowering::lowerRESET_FPENV(SDValue Op,
15330 SelectionDAG &DAG) const {
15331 const MVT XLenVT = Subtarget.getXLenVT();
15332 SDLoc DL(Op);
15333 SDValue Chain = Op->getOperand(Num: 0);
15334 SDValue EnvValue = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
15335 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15336
15337 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15338 N3: EnvValue);
15339}
15340
15341const uint64_t ModeMask64 = ~RISCVExceptFlags::ALL;
15342const uint32_t ModeMask32 = ~RISCVExceptFlags::ALL;
15343
15344SDValue RISCVTargetLowering::lowerGET_FPMODE(SDValue Op,
15345 SelectionDAG &DAG) const {
15346 const MVT XLenVT = Subtarget.getXLenVT();
15347 SDLoc DL(Op);
15348 SDValue Chain = Op->getOperand(Num: 0);
15349 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15350 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
15351 SDValue Result = DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
15352 Chain = Result.getValue(R: 1);
15353 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
15354}
15355
15356SDValue RISCVTargetLowering::lowerSET_FPMODE(SDValue Op,
15357 SelectionDAG &DAG) const {
15358 const MVT XLenVT = Subtarget.getXLenVT();
15359 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
15360 SDLoc DL(Op);
15361 SDValue Chain = Op->getOperand(Num: 0);
15362 SDValue EnvValue = Op->getOperand(Num: 1);
15363 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15364 SDValue ModeMask = DAG.getConstant(Val: ModeMaskValue, DL, VT: XLenVT);
15365
15366 EnvValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: EnvValue);
15367 EnvValue = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: EnvValue, N2: ModeMask);
15368 Chain = DAG.getNode(Opcode: RISCVISD::CLEAR_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15369 N3: ModeMask);
15370 return DAG.getNode(Opcode: RISCVISD::SET_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15371 N3: EnvValue);
15372}
15373
15374SDValue RISCVTargetLowering::lowerRESET_FPMODE(SDValue Op,
15375 SelectionDAG &DAG) const {
15376 const MVT XLenVT = Subtarget.getXLenVT();
15377 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
15378 SDLoc DL(Op);
15379 SDValue Chain = Op->getOperand(Num: 0);
15380 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15381 SDValue ModeMask = DAG.getConstant(Val: ModeMaskValue, DL, VT: XLenVT);
15382
15383 return DAG.getNode(Opcode: RISCVISD::CLEAR_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15384 N3: ModeMask);
15385}
15386
15387SDValue RISCVTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
15388 SelectionDAG &DAG) const {
15389 MachineFunction &MF = DAG.getMachineFunction();
15390
15391 bool isRISCV64 = Subtarget.is64Bit();
15392 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
15393
15394 int FI = MF.getFrameInfo().CreateFixedObject(Size: isRISCV64 ? 8 : 4, SPOffset: 0, IsImmutable: false);
15395 return DAG.getFrameIndex(FI, VT: PtrVT);
15396}
15397
15398// Returns the opcode of the target-specific SDNode that implements the 32-bit
15399// form of the given Opcode.
15400static unsigned getRISCVWOpcode(unsigned Opcode) {
15401 switch (Opcode) {
15402 default:
15403 llvm_unreachable("Unexpected opcode");
15404 case ISD::SHL:
15405 return RISCVISD::SLLW;
15406 case ISD::SRA:
15407 return RISCVISD::SRAW;
15408 case ISD::SRL:
15409 return RISCVISD::SRLW;
15410 case ISD::SDIV:
15411 return RISCVISD::DIVW;
15412 case ISD::UDIV:
15413 return RISCVISD::DIVUW;
15414 case ISD::UREM:
15415 return RISCVISD::REMUW;
15416 case ISD::ROTL:
15417 return RISCVISD::ROLW;
15418 case ISD::ROTR:
15419 return RISCVISD::RORW;
15420 }
15421}
15422
15423// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
15424// node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
15425// otherwise be promoted to i64, making it difficult to select the
15426// SLLW/DIVUW/.../*W later one because the fact the operation was originally of
15427// type i8/i16/i32 is lost.
15428static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
15429 unsigned ExtOpc = ISD::ANY_EXTEND) {
15430 SDLoc DL(N);
15431 unsigned WOpcode = getRISCVWOpcode(Opcode: N->getOpcode());
15432 SDValue NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15433 SDValue NewOp1 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15434 SDValue NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
15435 // ReplaceNodeResults requires we maintain the same type for the return value.
15436 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewRes);
15437}
15438
15439// Converts the given 32-bit operation to a i64 operation with signed extension
15440// semantic to reduce the signed extension instructions.
15441static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
15442 SDLoc DL(N);
15443 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15444 SDValue NewOp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15445 SDValue NewWOp = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
15446 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
15447 N2: DAG.getValueType(MVT::i32));
15448 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes);
15449}
15450
15451// Zero-extend a 32-bit packed vector to the 64-bit packed type WideVT,
15452// clearing the upper lanes.
15453static SDValue widenPackedVectorWithZeros(SelectionDAG &DAG, const SDLoc &DL,
15454 SDValue V, MVT WideVT) {
15455 SDValue Wide =
15456 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: DAG.getBitcast(VT: MVT::i32, V));
15457 return DAG.getBitcast(VT: WideVT, V: Wide);
15458}
15459
15460void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
15461 SmallVectorImpl<SDValue> &Results,
15462 SelectionDAG &DAG) const {
15463 SDLoc DL(N);
15464 switch (N->getOpcode()) {
15465 default:
15466 llvm_unreachable("Don't know how to custom type legalize this operation!");
15467 case ISD::STRICT_FP_TO_SINT:
15468 case ISD::STRICT_FP_TO_UINT:
15469 case ISD::FP_TO_SINT:
15470 case ISD::FP_TO_UINT: {
15471 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15472 "Unexpected custom legalisation");
15473 bool IsStrict = N->isStrictFPOpcode();
15474 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
15475 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
15476 SDValue Op0 = IsStrict ? N->getOperand(Num: 1) : N->getOperand(Num: 0);
15477 if (getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) !=
15478 TargetLowering::TypeSoftenFloat) {
15479 if (!isTypeLegal(VT: Op0.getValueType()))
15480 return;
15481 if (IsStrict) {
15482 SDValue Chain = N->getOperand(Num: 0);
15483 // In absence of Zfh, promote f16 to f32, then convert.
15484 if (Op0.getValueType() == MVT::f16 &&
15485 !Subtarget.hasStdExtZfhOrZhinx()) {
15486 Op0 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
15487 Ops: {Chain, Op0});
15488 Chain = Op0.getValue(R: 1);
15489 }
15490 unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
15491 : RISCVISD::STRICT_FCVT_WU_RV64;
15492 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other);
15493 SDValue Res = DAG.getNode(
15494 Opcode: Opc, DL, VTList: VTs, N1: Chain, N2: Op0,
15495 N3: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: MVT::i64));
15496 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15497 Results.push_back(Elt: Res.getValue(R: 1));
15498 return;
15499 }
15500 // For bf16, or f16 in absence of Zfh, promote [b]f16 to f32 and then
15501 // convert.
15502 if ((Op0.getValueType() == MVT::f16 &&
15503 !Subtarget.hasStdExtZfhOrZhinx()) ||
15504 Op0.getValueType() == MVT::bf16)
15505 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
15506
15507 unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
15508 SDValue Res =
15509 DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: Op0,
15510 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: MVT::i64));
15511 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15512 return;
15513 }
15514 // If the FP type needs to be softened, emit a library call using the 'si'
15515 // version. If we left it to default legalization we'd end up with 'di'. If
15516 // the FP type doesn't need to be softened just let generic type
15517 // legalization promote the result type.
15518 RTLIB::Libcall LC;
15519 if (IsSigned)
15520 LC = RTLIB::getFPTOSINT(OpVT: Op0.getValueType(), RetVT: N->getValueType(ResNo: 0));
15521 else
15522 LC = RTLIB::getFPTOUINT(OpVT: Op0.getValueType(), RetVT: N->getValueType(ResNo: 0));
15523 MakeLibCallOptions CallOptions;
15524 EVT OpVT = Op0.getValueType();
15525 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
15526 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
15527 SDValue Result;
15528 std::tie(args&: Result, args&: Chain) =
15529 makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0), Ops: Op0, CallOptions, dl: DL, Chain);
15530 Results.push_back(Elt: Result);
15531 if (IsStrict)
15532 Results.push_back(Elt: Chain);
15533 break;
15534 }
15535 case ISD::LROUND: {
15536 SDValue Op0 = N->getOperand(Num: 0);
15537 EVT Op0VT = Op0.getValueType();
15538 if (getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) !=
15539 TargetLowering::TypeSoftenFloat) {
15540 if (!isTypeLegal(VT: Op0VT))
15541 return;
15542
15543 // In absence of Zfh, promote f16 to f32, then convert.
15544 if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx())
15545 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
15546
15547 SDValue Res =
15548 DAG.getNode(Opcode: RISCVISD::FCVT_W_RV64, DL, VT: MVT::i64, N1: Op0,
15549 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RMM, DL, VT: MVT::i64));
15550 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15551 return;
15552 }
15553 // If the FP type needs to be softened, emit a library call to lround. We'll
15554 // need to truncate the result. We assume any value that doesn't fit in i32
15555 // is allowed to return an unspecified value.
15556 RTLIB::Libcall LC =
15557 Op0.getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
15558 MakeLibCallOptions CallOptions;
15559 EVT OpVT = Op0.getValueType();
15560 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: MVT::i64);
15561 SDValue Result = makeLibCall(DAG, LC, RetVT: MVT::i64, Ops: Op0, CallOptions, dl: DL).first;
15562 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Result);
15563 Results.push_back(Elt: Result);
15564 break;
15565 }
15566 case ISD::READCYCLECOUNTER:
15567 case ISD::READSTEADYCOUNTER: {
15568 assert(!Subtarget.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
15569 "has custom type legalization on riscv32");
15570
15571 SDValue LoCounter, HiCounter;
15572 MVT XLenVT = Subtarget.getXLenVT();
15573 if (N->getOpcode() == ISD::READCYCLECOUNTER) {
15574 LoCounter = DAG.getTargetConstant(Val: RISCVSysReg::cycle, DL, VT: XLenVT);
15575 HiCounter = DAG.getTargetConstant(Val: RISCVSysReg::cycleh, DL, VT: XLenVT);
15576 } else {
15577 LoCounter = DAG.getTargetConstant(Val: RISCVSysReg::time, DL, VT: XLenVT);
15578 HiCounter = DAG.getTargetConstant(Val: RISCVSysReg::timeh, DL, VT: XLenVT);
15579 }
15580 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other);
15581 SDValue RCW = DAG.getNode(Opcode: RISCVISD::READ_COUNTER_WIDE, DL, VTList: VTs,
15582 N1: N->getOperand(Num: 0), N2: LoCounter, N3: HiCounter);
15583
15584 Results.push_back(
15585 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: RCW, N2: RCW.getValue(R: 1)));
15586 Results.push_back(Elt: RCW.getValue(R: 2));
15587 break;
15588 }
15589 case ISD::LOAD: {
15590 if (!ISD::isNON_EXTLoad(N))
15591 return;
15592
15593 // Use a SEXTLOAD instead of the default EXTLOAD. Similar to the
15594 // sext_inreg we emit for ADD/SUB/MUL/SLLI.
15595 LoadSDNode *Ld = cast<LoadSDNode>(Val: N);
15596
15597 if (N->getValueType(ResNo: 0) == MVT::i64) {
15598 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
15599 "Unexpected custom legalisation");
15600
15601 if (Ld->getAlign() < Subtarget.getZilsdAlign())
15602 return;
15603
15604 SDLoc DL(N);
15605 SDValue Result = DAG.getMemIntrinsicNode(
15606 Opcode: RISCVISD::LD_RV32, dl: DL,
15607 VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32, MVT::Other}),
15608 Ops: {Ld->getChain(), Ld->getBasePtr()}, MemVT: MVT::i64, MMO: Ld->getMemOperand());
15609 SDValue Lo = Result.getValue(R: 0);
15610 SDValue Hi = Result.getValue(R: 1);
15611 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Lo, N2: Hi);
15612 Results.append(IL: {Pair, Result.getValue(R: 2)});
15613 return;
15614 }
15615
15616 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15617 "Unexpected custom legalisation");
15618
15619 SDLoc dl(N);
15620 SDValue Res = DAG.getExtLoad(ExtType: ISD::SEXTLOAD, dl, VT: MVT::i64, Chain: Ld->getChain(),
15621 Ptr: Ld->getBasePtr(), MemVT: Ld->getMemoryVT(),
15622 MMO: Ld->getMemOperand());
15623 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Res));
15624 Results.push_back(Elt: Res.getValue(R: 1));
15625 return;
15626 }
15627 case ISD::MUL: {
15628 unsigned Size = N->getSimpleValueType(ResNo: 0).getSizeInBits();
15629 unsigned XLen = Subtarget.getXLen();
15630 if (Size > XLen) {
15631 // This multiply needs to be expanded, try to use MULH+MUL or WMUL if
15632 // possible. We duplicate the default legalization to
15633 // MULHU/MULHS/UMUL_LOHI/SMUL_LOHI to minimize the number of calls to
15634 // MaskedValueIsZero and ComputeNumSignBits
15635 // FIXME: Should we have a target independent MULHSU/WMULSU node? Are
15636 // there are other targets that could use it?
15637 assert(Size == (XLen * 2) && "Unexpected custom legalisation");
15638
15639 auto MakeMULPair = [&](SDValue L, SDValue R, unsigned HighOpc,
15640 unsigned LoHiOpc) {
15641 MVT XLenVT = Subtarget.getXLenVT();
15642 L = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: XLenVT, Operand: L);
15643 R = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: XLenVT, Operand: R);
15644 SDValue Lo, Hi;
15645 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
15646 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32);
15647 Lo = DAG.getNode(Opcode: LoHiOpc, DL, VTList: VTs, N1: L, N2: R);
15648 Hi = Lo.getValue(R: 1);
15649 } else {
15650 Lo = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: L, N2: R);
15651 Hi = DAG.getNode(Opcode: HighOpc, DL, VT: XLenVT, N1: L, N2: R);
15652 }
15653 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
15654 };
15655
15656 SDValue LHS = N->getOperand(Num: 0);
15657 SDValue RHS = N->getOperand(Num: 1);
15658
15659 APInt HighMask = APInt::getHighBitsSet(numBits: Size, hiBitsSet: XLen);
15660 bool LHSIsU = DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask);
15661 bool RHSIsU = DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask);
15662 if (LHSIsU && RHSIsU) {
15663 Results.push_back(Elt: MakeMULPair(LHS, RHS, ISD::MULHU, ISD::UMUL_LOHI));
15664 return;
15665 }
15666
15667 bool LHSIsS = DAG.ComputeNumSignBits(Op: LHS) > XLen;
15668 bool RHSIsS = DAG.ComputeNumSignBits(Op: RHS) > XLen;
15669 if (LHSIsS && RHSIsS)
15670 Results.push_back(Elt: MakeMULPair(LHS, RHS, ISD::MULHS, ISD::SMUL_LOHI));
15671 else if (RHSIsU && LHSIsS)
15672 Results.push_back(
15673 Elt: MakeMULPair(LHS, RHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
15674 else if (LHSIsU && RHSIsS)
15675 Results.push_back(
15676 Elt: MakeMULPair(RHS, LHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
15677
15678 return;
15679 }
15680 [[fallthrough]];
15681 }
15682 case ISD::ADD:
15683 case ISD::SUB:
15684 if (N->getValueType(ResNo: 0) == MVT::i64) {
15685 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
15686 "Unexpected custom legalisation");
15687
15688 // Expand to ADDD/SUBD.
15689 auto [LHSLo, LHSHi] =
15690 DAG.SplitScalar(N: N->getOperand(Num: 0), DL, LoVT: MVT::i32, HiVT: MVT::i32);
15691 auto [RHSLo, RHSHi] =
15692 DAG.SplitScalar(N: N->getOperand(Num: 1), DL, LoVT: MVT::i32, HiVT: MVT::i32);
15693 unsigned Opc =
15694 N->getOpcode() == ISD::ADD ? RISCVISD::ADDD : RISCVISD::SUBD;
15695 SDValue Res = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
15696 N1: LHSLo, N2: LHSHi, N3: RHSLo, N4: RHSHi);
15697 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Res, N2: Res.getValue(R: 1));
15698 Results.push_back(Elt: Res);
15699 return;
15700 }
15701
15702 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15703 "Unexpected custom legalisation");
15704 Results.push_back(Elt: customLegalizeToWOpWithSExt(N, DAG));
15705 break;
15706 case ISD::SHL:
15707 case ISD::SRA:
15708 case ISD::SRL: {
15709 EVT VT = N->getValueType(ResNo: 0);
15710 if (VT.isFixedLengthVector() && Subtarget.hasStdExtP()) {
15711 assert(Subtarget.is64Bit() && (VT == MVT::v2i16 || VT == MVT::v4i8) &&
15712 "Unexpected vector type for P-extension shift");
15713
15714 // If shift amount is a splat, don't scalarize - let normal widening
15715 // and SIMD patterns handle it (pslli.h, psrli.h, etc.)
15716 SDValue ShiftAmt = N->getOperand(Num: 1);
15717 if (DAG.isSplatValue(V: ShiftAmt, /*AllowUndefs=*/true))
15718 break;
15719
15720 EVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
15721 unsigned WidenNumElts = WidenVT.getVectorNumElements();
15722 // Unroll with OrigNumElts operations, padding result to WidenNumElts
15723 SDValue Res = DAG.UnrollVectorOp(N, ResNE: WidenNumElts);
15724 Results.push_back(Elt: Res);
15725 break;
15726 }
15727
15728 if (VT == MVT::i64) {
15729 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
15730 "Unexpected custom legalisation");
15731
15732 SDValue LHS = N->getOperand(Num: 0);
15733 SDValue ShAmt = N->getOperand(Num: 1);
15734
15735 unsigned WideOpc = 0;
15736 APInt HighMask = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
15737 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask))
15738 WideOpc = RISCVISD::WSLL;
15739 else if (DAG.ComputeMaxSignificantBits(Op: LHS) <= 32)
15740 WideOpc = RISCVISD::WSLA;
15741
15742 if (WideOpc) {
15743 SDValue Res =
15744 DAG.getNode(Opcode: WideOpc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
15745 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LHS),
15746 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: ShAmt));
15747 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: N->getValueType(ResNo: 0),
15748 N1: Res, N2: Res.getValue(R: 1)));
15749 return;
15750 }
15751
15752 // Only handle constant shifts < 32. Non-constant shifts are handled by
15753 // lowerShiftLeftParts/lowerShiftRightParts, and shifts >= 32 use default
15754 // legalization.
15755 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
15756 if (!ShAmtC || ShAmtC->getZExtValue() >= 32)
15757 break;
15758
15759 auto [Lo, Hi] = DAG.SplitScalar(N: LHS, DL, LoVT: MVT::i32, HiVT: MVT::i32);
15760
15761 // If the shift amount operand is coming from a vector legalization it may
15762 // have an illegal type.
15763 if (ShAmt.getValueType() != MVT::i32)
15764 ShAmt = DAG.getZExtOrTrunc(Op: ShAmt, DL, VT: MVT::i32);
15765
15766 SDValue LoRes, HiRes;
15767 if (N->getOpcode() == ISD::SHL) {
15768 // Lo = slli Lo, shamt
15769 // Hi = nsrli {Hi, Lo}, (32 - shamt)
15770 uint64_t ShAmtVal = ShAmtC->getZExtValue();
15771 LoRes = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: Lo, N2: ShAmt);
15772 HiRes = DAG.getNode(Opcode: RISCVISD::NSRL, DL, VT: MVT::i32, N1: Lo, N2: Hi,
15773 N3: DAG.getConstant(Val: 32 - ShAmtVal, DL, VT: MVT::i32));
15774 } else {
15775 bool IsSRA = N->getOpcode() == ISD::SRA;
15776 LoRes = DAG.getNode(Opcode: IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL,
15777 VT: MVT::i32, N1: Lo, N2: Hi, N3: ShAmt);
15778 HiRes =
15779 DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32, N1: Hi, N2: ShAmt);
15780 }
15781 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: LoRes, N2: HiRes);
15782 Results.push_back(Elt: Res);
15783 return;
15784 }
15785
15786 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
15787 "Unexpected custom legalisation");
15788 if (N->getOperand(Num: 1).getOpcode() != ISD::Constant) {
15789 // If we can use a BSET instruction, allow default promotion to apply.
15790 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() &&
15791 isOneConstant(V: N->getOperand(Num: 0)))
15792 break;
15793 Results.push_back(Elt: customLegalizeToWOp(N, DAG));
15794 break;
15795 }
15796
15797 // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
15798 // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
15799 // shift amount.
15800 if (N->getOpcode() == ISD::SHL) {
15801 SDLoc DL(N);
15802 SDValue NewOp0 =
15803 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15804 SDValue NewOp1 =
15805 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15806 SDValue NewWOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
15807 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
15808 N2: DAG.getValueType(MVT::i32));
15809 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes));
15810 }
15811
15812 break;
15813 }
15814 case ISD::ROTL:
15815 case ISD::ROTR:
15816 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15817 "Unexpected custom legalisation");
15818 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
15819 Subtarget.hasVendorXTHeadBb()) &&
15820 "Unexpected custom legalization");
15821 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15822 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
15823 return;
15824 Results.push_back(Elt: customLegalizeToWOp(N, DAG));
15825 break;
15826 case ISD::CTTZ:
15827 case ISD::CTTZ_ZERO_POISON:
15828 case ISD::CTLZ:
15829 case ISD::CTLZ_ZERO_POISON:
15830 case ISD::CTLS: {
15831 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15832 "Unexpected custom legalisation");
15833
15834 SDValue NewOp0 =
15835 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15836 unsigned Opc;
15837 switch (N->getOpcode()) {
15838 default: llvm_unreachable("Unexpected opcode");
15839 case ISD::CTTZ:
15840 case ISD::CTTZ_ZERO_POISON:
15841 Opc = RISCVISD::CTZW;
15842 break;
15843 case ISD::CTLZ:
15844 case ISD::CTLZ_ZERO_POISON:
15845 Opc = RISCVISD::CLZW;
15846 break;
15847 case ISD::CTLS:
15848 Opc = RISCVISD::CLSW;
15849 break;
15850 }
15851
15852 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, Operand: NewOp0);
15853 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15854 return;
15855 }
15856 case ISD::SDIV:
15857 case ISD::UDIV:
15858 case ISD::UREM: {
15859 MVT VT = N->getSimpleValueType(ResNo: 0);
15860 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
15861 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
15862 "Unexpected custom legalisation");
15863 // Don't promote division/remainder by constant since we should expand those
15864 // to multiply by magic constant.
15865 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
15866 if (N->getOperand(Num: 1).getOpcode() == ISD::Constant &&
15867 !isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
15868 return;
15869
15870 // If the input is i32, use ANY_EXTEND since the W instructions don't read
15871 // the upper 32 bits. For other types we need to sign or zero extend
15872 // based on the opcode.
15873 unsigned ExtOpc = ISD::ANY_EXTEND;
15874 if (VT != MVT::i32)
15875 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
15876 : ISD::ZERO_EXTEND;
15877
15878 Results.push_back(Elt: customLegalizeToWOp(N, DAG, ExtOpc));
15879 break;
15880 }
15881 case ISD::SADDO:
15882 case ISD::SSUBO: {
15883 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15884 "Unexpected custom legalisation");
15885
15886 // This is similar to the default legalization, but we return the
15887 // sext_inreg instead of the add/sub.
15888 bool IsAdd = N->getOpcode() == ISD::SADDO;
15889 SDValue LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15890 SDValue RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15891 SDValue Op =
15892 DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL, VT: MVT::i64, N1: LHS, N2: RHS);
15893 SDValue Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Op,
15894 N2: DAG.getValueType(MVT::i32));
15895
15896 SDValue Overflow;
15897
15898 // If the RHS is a constant, we can simplify ConditionRHS below. Otherwise
15899 // use the default legalization.
15900 if (IsAdd && isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
15901 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
15902
15903 // For an addition, the result should be less than one of the operands
15904 // (LHS) if and only if the other operand (RHS) is negative, otherwise
15905 // there will be overflow.
15906 EVT OType = N->getValueType(ResNo: 1);
15907 SDValue ResultLowerThanLHS =
15908 DAG.getSetCC(DL, VT: OType, LHS: Res, RHS: LHS, Cond: ISD::SETLT);
15909 SDValue ConditionRHS = DAG.getSetCC(DL, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
15910
15911 Overflow =
15912 DAG.getNode(Opcode: ISD::XOR, DL, VT: OType, N1: ConditionRHS, N2: ResultLowerThanLHS);
15913 } else {
15914 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res, RHS: Op, Cond: ISD::SETNE);
15915 }
15916
15917 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15918 Results.push_back(Elt: Overflow);
15919 return;
15920 }
15921 case ISD::UADDO:
15922 case ISD::USUBO: {
15923 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15924 "Unexpected custom legalisation");
15925 bool IsAdd = N->getOpcode() == ISD::UADDO;
15926 // Create an ADDW or SUBW.
15927 SDValue LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15928 SDValue RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15929 SDValue Res =
15930 DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL, VT: MVT::i64, N1: LHS, N2: RHS);
15931 Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Res,
15932 N2: DAG.getValueType(MVT::i32));
15933
15934 SDValue Overflow;
15935 if (IsAdd && isOneConstant(V: RHS)) {
15936 // Special case uaddo X, 1 overflowed if the addition result is 0.
15937 // The general case (X + C) < C is not necessarily beneficial. Although we
15938 // reduce the live range of X, we may introduce the materialization of
15939 // constant C, especially when the setcc result is used by branch. We have
15940 // no compare with constant and branch instructions.
15941 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res,
15942 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i64), Cond: ISD::SETEQ);
15943 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
15944 // Special case uaddo X, -1 overflowed if X != 0.
15945 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: N->getOperand(Num: 0),
15946 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Cond: ISD::SETNE);
15947 } else {
15948 // Sign extend the LHS and perform an unsigned compare with the ADDW
15949 // result. Since the inputs are sign extended from i32, this is equivalent
15950 // to comparing the lower 32 bits.
15951 LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15952 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res, RHS: LHS,
15953 Cond: IsAdd ? ISD::SETULT : ISD::SETUGT);
15954 }
15955
15956 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15957 Results.push_back(Elt: Overflow);
15958 return;
15959 }
15960 case ISD::UADDSAT:
15961 case ISD::USUBSAT:
15962 case ISD::SADDSAT:
15963 case ISD::SSUBSAT: {
15964 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15965 "Unexpected custom legalisation");
15966
15967 if (Subtarget.hasStdExtP()) {
15968 // On RV64, map scalar i32 saturating add/sub through lane 0 of a packed
15969 // v2i32 operation so we can select ps*.w instructions.
15970 SDValue LHS = DAG.getNode(
15971 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
15972 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0)));
15973 SDValue RHS = DAG.getNode(
15974 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
15975 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1)));
15976 SDValue VecRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v2i32, N1: LHS, N2: RHS);
15977 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
15978 Results.push_back(
15979 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: VecRes, N2: Zero));
15980 return;
15981 }
15982
15983 assert(!Subtarget.hasStdExtZbb() && "Unexpected custom legalisation");
15984 Results.push_back(Elt: expandAddSubSat(Node: N, DAG));
15985 return;
15986 }
15987 case ISD::ABS:
15988 case ISD::ABS_MIN_POISON: {
15989 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15990 "Unexpected custom legalisation");
15991
15992 if (Subtarget.hasStdExtP()) {
15993 SDValue Src =
15994 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15995 SDValue Abs = DAG.getNode(Opcode: RISCVISD::ABSW, DL, VT: MVT::i64, Operand: Src);
15996 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Abs));
15997 return;
15998 }
15999
16000 if (Subtarget.hasStdExtZbb()) {
16001 // Emit a special node that will be expanded to NEGW+MAX at isel.
16002 // This allows us to remember that the result is sign extended. Expanding
16003 // to NEGW+MAX here requires a Freeze which breaks ComputeNumSignBits.
16004 SDValue Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64,
16005 Operand: N->getOperand(Num: 0));
16006 SDValue Abs = DAG.getNode(Opcode: RISCVISD::NEGW_MAX, DL, VT: MVT::i64, Operand: Src);
16007 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Abs));
16008 return;
16009 }
16010
16011 // Expand abs to Y = (sraiw X, 31); subw(xor(X, Y), Y)
16012 SDValue Src = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
16013
16014 // Freeze the source so we can increase it's use count.
16015 Src = DAG.getFreeze(V: Src);
16016
16017 // Copy sign bit to all bits using the sraiw pattern.
16018 SDValue SignFill = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Src,
16019 N2: DAG.getValueType(MVT::i32));
16020 SignFill = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i64, N1: SignFill,
16021 N2: DAG.getConstant(Val: 31, DL, VT: MVT::i64));
16022
16023 SDValue NewRes = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, N1: Src, N2: SignFill);
16024 NewRes = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: NewRes, N2: SignFill);
16025
16026 // NOTE: The result is only required to be anyextended, but sext is
16027 // consistent with type legalization of sub.
16028 NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewRes,
16029 N2: DAG.getValueType(MVT::i32));
16030 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes));
16031 return;
16032 }
16033 case ISD::BITCAST: {
16034 EVT VT = N->getValueType(ResNo: 0);
16035 assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
16036 SDValue Op0 = N->getOperand(Num: 0);
16037 EVT Op0VT = Op0.getValueType();
16038 MVT XLenVT = Subtarget.getXLenVT();
16039 if (VT == MVT::i16 &&
16040 ((Op0VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
16041 (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
16042 SDValue FPConv = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Op0);
16043 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FPConv));
16044 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
16045 Subtarget.hasStdExtFOrZfinx()) {
16046 SDValue FPConv =
16047 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Op0);
16048 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: FPConv));
16049 } else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.is64Bit() &&
16050 Subtarget.hasStdExtDOrZdinx()) {
16051 SDValue NewReg = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
16052 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Op0);
16053 SDValue Lo = NewReg.getValue(R: 0);
16054 SDValue Hi = NewReg.getValue(R: 1);
16055 // For big-endian, swap the order when building the i64 pair.
16056 if (!Subtarget.isLittleEndian())
16057 std::swap(a&: Lo, b&: Hi);
16058 SDValue RetReg = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Lo, N2: Hi);
16059 Results.push_back(Elt: RetReg);
16060 } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
16061 isTypeLegal(VT: Op0VT)) {
16062 // Custom-legalize bitcasts from fixed-length vector types to illegal
16063 // scalar types in order to improve codegen. Bitcast the vector to a
16064 // one-element vector type whose element type is the same as the result
16065 // type, and extract the first element.
16066 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 1);
16067 if (isTypeLegal(VT: BVT)) {
16068 SDValue BVec = DAG.getBitcast(VT: BVT, V: Op0);
16069 Results.push_back(Elt: DAG.getExtractVectorElt(DL, VT, Vec: BVec, Idx: 0));
16070 }
16071 }
16072 break;
16073 }
16074 case ISD::BITREVERSE: {
16075 assert(N->getValueType(0) == MVT::i8 && Subtarget.hasStdExtZbkb() &&
16076 "Unexpected custom legalisation");
16077 MVT XLenVT = Subtarget.getXLenVT();
16078 SDValue NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 0));
16079 SDValue NewRes = DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT: XLenVT, Operand: NewOp);
16080 // ReplaceNodeResults requires we maintain the same type for the return
16081 // value.
16082 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: NewRes));
16083 break;
16084 }
16085 case RISCVISD::BREV8:
16086 case RISCVISD::ORC_B: {
16087 MVT VT = N->getSimpleValueType(ResNo: 0);
16088 MVT XLenVT = Subtarget.getXLenVT();
16089 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
16090 "Unexpected custom legalisation");
16091 assert(((N->getOpcode() == RISCVISD::BREV8 && Subtarget.hasStdExtZbkb()) ||
16092 (N->getOpcode() == RISCVISD::ORC_B && Subtarget.hasStdExtZbb())) &&
16093 "Unexpected extension");
16094 SDValue NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 0));
16095 SDValue NewRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: XLenVT, Operand: NewOp);
16096 // ReplaceNodeResults requires we maintain the same type for the return
16097 // value.
16098 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: NewRes));
16099 break;
16100 }
16101 case RISCVISD::ASUB:
16102 case RISCVISD::ASUBU:
16103 case RISCVISD::MULHSU:
16104 case RISCVISD::MULHR:
16105 case RISCVISD::MULHRU:
16106 case RISCVISD::MULHRSU: {
16107 MVT VT = N->getSimpleValueType(ResNo: 0);
16108 SDValue Op0 = N->getOperand(Num: 0);
16109 SDValue Op1 = N->getOperand(Num: 1);
16110 unsigned Opcode = N->getOpcode();
16111 // PMULH* variants don't support i8
16112 [[maybe_unused]] bool IsMulH =
16113 Opcode == RISCVISD::MULHSU || Opcode == RISCVISD::MULHR ||
16114 Opcode == RISCVISD::MULHRU || Opcode == RISCVISD::MULHRSU;
16115 assert(VT == MVT::v2i16 || (!IsMulH && VT == MVT::v4i8));
16116 MVT NewVT = MVT::v4i16;
16117 if (VT == MVT::v4i8)
16118 NewVT = MVT::v8i8;
16119 SDValue Undef = DAG.getUNDEF(VT);
16120 Op0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, Ops: {Op0, Undef});
16121 Op1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, Ops: {Op1, Undef});
16122 Results.push_back(Elt: DAG.getNode(Opcode, DL, VT: NewVT, Ops: {Op0, Op1}));
16123 return;
16124 }
16125 case ISD::EXTRACT_VECTOR_ELT: {
16126 // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
16127 // type is illegal (currently only vXi64 RV32).
16128 // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
16129 // transferred to the destination register. We issue two of these from the
16130 // upper- and lower- halves of the SEW-bit vector element, slid down to the
16131 // first element.
16132 SDValue Vec = N->getOperand(Num: 0);
16133 SDValue Idx = N->getOperand(Num: 1);
16134
16135 // The vector type hasn't been legalized yet so we can't issue target
16136 // specific nodes if it needs legalization.
16137 // FIXME: We would manually legalize if it's important.
16138 if (!isTypeLegal(VT: Vec.getValueType()))
16139 return;
16140
16141 MVT VecVT = Vec.getSimpleValueType();
16142
16143 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
16144 VecVT.getVectorElementType() == MVT::i64 &&
16145 "Unexpected EXTRACT_VECTOR_ELT legalization");
16146
16147 // If this is a fixed vector, we need to convert it to a scalable vector.
16148 MVT ContainerVT = VecVT;
16149 if (VecVT.isFixedLengthVector()) {
16150 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
16151 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
16152 }
16153
16154 MVT XLenVT = Subtarget.getXLenVT();
16155
16156 // Use a VL of 1 to avoid processing more elements than we need.
16157 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT, DL, DAG, Subtarget);
16158
16159 // Unless the index is known to be 0, we must slide the vector down to get
16160 // the desired element into index 0.
16161 if (!isNullConstant(V: Idx)) {
16162 Vec = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
16163 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: Idx, Mask, VL);
16164 }
16165
16166 // Extract the lower XLEN bits of the correct vector element.
16167 SDValue EltLo = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
16168
16169 // To extract the upper XLEN bits of the vector element, shift the first
16170 // element right by 32 bits and re-extract the lower XLEN bits.
16171 SDValue ThirtyTwoV = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
16172 N1: DAG.getUNDEF(VT: ContainerVT),
16173 N2: DAG.getConstant(Val: 32, DL, VT: XLenVT), N3: VL);
16174 SDValue LShr32 =
16175 DAG.getNode(Opcode: RISCVISD::SRL_VL, DL, VT: ContainerVT, N1: Vec, N2: ThirtyTwoV,
16176 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
16177
16178 SDValue EltHi = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: LShr32);
16179
16180 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: EltLo, N2: EltHi));
16181 break;
16182 }
16183 case ISD::INTRINSIC_WO_CHAIN: {
16184 unsigned IntNo = N->getConstantOperandVal(Num: 0);
16185 switch (IntNo) {
16186 default:
16187 llvm_unreachable(
16188 "Don't know how to custom type legalize this intrinsic!");
16189 case Intrinsic::experimental_get_vector_length: {
16190 SDValue Res = lowerGetVectorLength(N, DAG, Subtarget);
16191 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16192 return;
16193 }
16194 case Intrinsic::riscv_paadd:
16195 case Intrinsic::riscv_paaddu:
16196 case Intrinsic::riscv_pasub:
16197 case Intrinsic::riscv_pasubu:
16198 case Intrinsic::riscv_pabd:
16199 case Intrinsic::riscv_pabdu:
16200 case Intrinsic::riscv_pas:
16201 case Intrinsic::riscv_psa:
16202 case Intrinsic::riscv_psas:
16203 case Intrinsic::riscv_pssa:
16204 case Intrinsic::riscv_paas:
16205 case Intrinsic::riscv_pasa:
16206 case Intrinsic::riscv_pmerge:
16207 case Intrinsic::riscv_psabs: {
16208 EVT VT = N->getValueType(ResNo: 0);
16209 if (!Subtarget.is64Bit() || (VT != MVT::v4i8 && VT != MVT::v2i16))
16210 return;
16211
16212 unsigned Opc;
16213 switch (IntNo) {
16214 case Intrinsic::riscv_paadd:
16215 Opc = ISD::AVGFLOORS;
16216 break;
16217 case Intrinsic::riscv_paaddu:
16218 Opc = ISD::AVGFLOORU;
16219 break;
16220 case Intrinsic::riscv_pasub:
16221 Opc = RISCVISD::ASUB;
16222 break;
16223 case Intrinsic::riscv_pasubu:
16224 Opc = RISCVISD::ASUBU;
16225 break;
16226 case Intrinsic::riscv_pabd:
16227 Opc = ISD::ABDS;
16228 break;
16229 case Intrinsic::riscv_pabdu:
16230 Opc = ISD::ABDU;
16231 break;
16232 case Intrinsic::riscv_psabs:
16233 Opc = RISCVISD::PSABS;
16234 break;
16235 default:
16236 // pas/psa/psas/pssa/paas/pasa and pmerge: re-emit at the widened type
16237 // rather than lowering to a generic node.
16238 Opc = ISD::INTRINSIC_WO_CHAIN;
16239 break;
16240 }
16241
16242 EVT WideVT = VT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16;
16243 SDValue Undef = DAG.getUNDEF(VT);
16244 SmallVector<SDValue, 4> Ops(N->ops());
16245 for (SDValue &Op : Ops) {
16246 if (Op.getValueType() == VT)
16247 Op = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideVT, N1: Op, N2: Undef);
16248 }
16249 SDValue Res;
16250 if (Opc == ISD::INTRINSIC_WO_CHAIN)
16251 Res = DAG.getNode(Opcode: Opc, DL, VT: WideVT, Ops);
16252 else
16253 Res = DAG.getNode(Opcode: Opc, DL, VT: WideVT, Ops: ArrayRef(Ops).slice(N: 1));
16254 Results.push_back(Elt: DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: Res,
16255 N2: DAG.getVectorIdxConstant(Val: 0, DL)));
16256 return;
16257 }
16258 case Intrinsic::riscv_pssha:
16259 case Intrinsic::riscv_psshar:
16260 case Intrinsic::riscv_psshl:
16261 case Intrinsic::riscv_psshlr: {
16262 MVT VT = N->getSimpleValueType(ResNo: 0);
16263 if (!Subtarget.is64Bit() || VT != MVT::v2i16)
16264 return;
16265
16266 MVT WideVT = MVT::v4i16;
16267 SDValue Op0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: WideVT,
16268 N1: N->getOperand(Num: 1), N2: DAG.getUNDEF(VT));
16269 SDValue ShAmt = N->getOperand(Num: 2);
16270 ShAmt = DAG.getAnyExtOrTrunc(Op: ShAmt, DL, VT: Subtarget.getXLenVT());
16271 SDValue Res =
16272 DAG.getNode(Opcode: getRVPShiftOpcode(IntNo), DL, VT: WideVT, N1: Op0, N2: ShAmt);
16273 Results.push_back(Elt: DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0));
16274 return;
16275 }
16276 case Intrinsic::riscv_predsum:
16277 case Intrinsic::riscv_predsumu: {
16278 bool IsSigned = IntNo == Intrinsic::riscv_predsum;
16279 SDValue Vec = N->getOperand(Num: 1);
16280 MVT VecVT = Vec.getSimpleValueType();
16281 auto Ext = [&](SDValue V) {
16282 return DAG.getNode(Opcode: IsSigned ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND, DL,
16283 VT: MVT::i64, Operand: V);
16284 };
16285 auto RedSum = [&](MVT VT, SDValue V, SDValue Acc) {
16286 return DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT, N1: N->getOperand(Num: 0), N2: V,
16287 N3: Acc);
16288 };
16289
16290 // RV32: i64 accumulator. Reduce to a 32-bit partial sum, then
16291 // widening-accumulate into i64 via wadda/waddau (v2i32 uses wadda alone).
16292 if (!Subtarget.is64Bit() && N->getValueType(ResNo: 0) == MVT::i64) {
16293 SDValue Acc = N->getOperand(Num: 2);
16294 SDValue Res;
16295 if (VecVT == MVT::v2i32) {
16296 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Acc,
16297 N2: Ext(DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec, Idx: 0)));
16298 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Res,
16299 N2: Ext(DAG.getExtractVectorElt(DL, VT: MVT::i32, Vec, Idx: 1)));
16300 } else {
16301 // The paired predsum.dbs/dhs computes the 32-bit element sum.
16302 SDValue Partial =
16303 RedSum(MVT::i32, Vec, DAG.getConstant(Val: 0, DL, VT: MVT::i32));
16304 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Acc, N2: Ext(Partial));
16305 }
16306 Results.push_back(Elt: Res);
16307 return;
16308 }
16309
16310 // RV64: i32 accumulator. Reduce at i64 (XLEN), then truncate.
16311 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
16312 return;
16313
16314 // Zero the upper lanes (zext.w) so they don't contribute to the sum.
16315 if (VecVT == MVT::v4i8 || VecVT == MVT::v2i16)
16316 Vec = widenPackedVectorWithZeros(
16317 DAG, DL, V: Vec, WideVT: VecVT == MVT::v4i8 ? MVT::v8i8 : MVT::v4i16);
16318
16319 // The result is truncated to i32, so the accumulator's upper bits are
16320 // unused and need no sign/zero extension.
16321 SDValue Acc =
16322 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
16323 SDValue Res = RedSum(MVT::i64, Vec, Acc);
16324 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16325 return;
16326 }
16327 case Intrinsic::riscv_pabdsumu:
16328 case Intrinsic::riscv_pabdsumau: {
16329 bool IsAcc = IntNo == Intrinsic::riscv_pabdsumau;
16330 // The two packed sources (rs1, rs2) are the last two operands.
16331 SDValue Rs1 = N->getOperand(Num: N->getNumOperands() - 2);
16332 SDValue Rs2 = N->getOperand(Num: N->getNumOperands() - 1);
16333 MVT VecVT = Rs1.getSimpleValueType();
16334
16335 // RV32: i64 result, always from a v8i8 source. The accumulator, if any,
16336 // folds into the widening add below.
16337 if (!Subtarget.is64Bit() && N->getValueType(ResNo: 0) == MVT::i64) {
16338 // Sum of absolute differences of two v4i8 halves.
16339 auto Sad = [&](SDValue A, SDValue B) {
16340 SDValue S = DAG.getNode(
16341 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i32,
16342 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_pabdsumu, DL, VT: MVT::i32), N2: A,
16343 N3: B);
16344 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: S);
16345 };
16346 auto [Rs1Lo, Rs1Hi] = DAG.SplitVector(N: Rs1, DL);
16347 auto [Rs2Lo, Rs2Hi] = DAG.SplitVector(N: Rs2, DL);
16348 SDValue Lo = Sad(Rs1Lo, Rs2Lo);
16349 SDValue Hi = Sad(Rs1Hi, Rs2Hi);
16350 // (acc + lo) + hi keeps the accumulate chained so it folds into a
16351 // single waddau; without an accumulator lo + hi folds into waddu.
16352 SDValue Res =
16353 IsAcc ? DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: N->getOperand(Num: 1), N2: Lo)
16354 : Lo;
16355 Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: Res, N2: Hi);
16356 Results.push_back(Elt: Res);
16357 return;
16358 }
16359
16360 // RV64: i32 result, so reduce at i64 and truncate. The source is v4i8 or
16361 // v8i8; widen a v4i8 to v8i8, zeroing the upper bytes (v8i8 is legal).
16362 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
16363 return;
16364 if (VecVT == MVT::v4i8) {
16365 Rs1 = widenPackedVectorWithZeros(DAG, DL, V: Rs1, WideVT: MVT::v8i8);
16366 Rs2 = widenPackedVectorWithZeros(DAG, DL, V: Rs2, WideVT: MVT::v8i8);
16367 }
16368 SmallVector<SDValue, 4> Ops = {N->getOperand(Num: 0)};
16369 if (IsAcc)
16370 Ops.push_back(
16371 Elt: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1)));
16372 Ops.push_back(Elt: Rs1);
16373 Ops.push_back(Elt: Rs2);
16374 SDValue Res = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: MVT::i64, Ops);
16375 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16376 return;
16377 }
16378 case Intrinsic::riscv_orc_b:
16379 case Intrinsic::riscv_brev8:
16380 case Intrinsic::riscv_sha256sig0:
16381 case Intrinsic::riscv_sha256sig1:
16382 case Intrinsic::riscv_sha256sum0:
16383 case Intrinsic::riscv_sha256sum1:
16384 case Intrinsic::riscv_sm3p0:
16385 case Intrinsic::riscv_sm3p1: {
16386 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
16387 return;
16388 unsigned Opc;
16389 switch (IntNo) {
16390 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
16391 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
16392 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
16393 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
16394 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
16395 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
16396 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
16397 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
16398 }
16399
16400 SDValue NewOp =
16401 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16402 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, Operand: NewOp);
16403 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16404 return;
16405 }
16406 case Intrinsic::riscv_sm4ks:
16407 case Intrinsic::riscv_sm4ed: {
16408 unsigned Opc =
16409 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
16410 SDValue NewOp0 =
16411 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16412 SDValue NewOp1 =
16413 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
16414 SDValue Res =
16415 DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1, N3: N->getOperand(Num: 3));
16416 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16417 return;
16418 }
16419 case Intrinsic::riscv_mopr: {
16420 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
16421 return;
16422 SDValue NewOp =
16423 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16424 SDValue Res = DAG.getNode(
16425 Opcode: RISCVISD::MOP_R, DL, VT: MVT::i64, N1: NewOp,
16426 N2: DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 2), DL, VT: MVT::i64));
16427 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16428 return;
16429 }
16430 case Intrinsic::riscv_moprr: {
16431 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
16432 return;
16433 SDValue NewOp0 =
16434 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16435 SDValue NewOp1 =
16436 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
16437 SDValue Res = DAG.getNode(
16438 Opcode: RISCVISD::MOP_RR, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1,
16439 N3: DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 3), DL, VT: MVT::i64));
16440 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16441 return;
16442 }
16443 case Intrinsic::riscv_clmulh:
16444 case Intrinsic::riscv_clmulr: {
16445 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
16446 return;
16447
16448 // Extend inputs to XLen, and shift by 32. This will add 64 trailing zeros
16449 // to the full 128-bit clmul result of multiplying two xlen values.
16450 // Perform clmulr or clmulh on the shifted values. Finally, extract the
16451 // upper 32 bits.
16452 //
16453 // The alternative is to mask the inputs to 32 bits and use clmul, but
16454 // that requires two shifts to mask each input without zext.w.
16455 // FIXME: If the inputs are known zero extended or could be freely
16456 // zero extended, the mask form would be better.
16457 SDValue NewOp0 =
16458 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
16459 SDValue NewOp1 =
16460 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
16461 NewOp0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp0,
16462 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
16463 NewOp1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp1,
16464 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
16465 unsigned Opc =
16466 IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
16467 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
16468 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Res,
16469 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
16470 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16471 return;
16472 }
16473 case Intrinsic::riscv_vmv_x_s: {
16474 EVT VT = N->getValueType(ResNo: 0);
16475 MVT XLenVT = Subtarget.getXLenVT();
16476 if (VT.bitsLT(VT: XLenVT)) {
16477 // Simple case just extract using vmv.x.s and truncate.
16478 SDValue Extract = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL,
16479 VT: Subtarget.getXLenVT(), Operand: N->getOperand(Num: 1));
16480 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Extract));
16481 return;
16482 }
16483
16484 assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
16485 "Unexpected custom legalization");
16486
16487 // We need to do the move in two steps.
16488 SDValue Vec = N->getOperand(Num: 1);
16489 MVT VecVT = Vec.getSimpleValueType();
16490
16491 // First extract the lower XLEN bits of the element.
16492 SDValue EltLo = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
16493
16494 // To extract the upper XLEN bits of the vector element, shift the first
16495 // element right by 32 bits and re-extract the lower XLEN bits.
16496 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT: VecVT, DL, DAG, Subtarget);
16497
16498 SDValue ThirtyTwoV =
16499 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: VecVT, N1: DAG.getUNDEF(VT: VecVT),
16500 N2: DAG.getConstant(Val: 32, DL, VT: XLenVT), N3: VL);
16501 SDValue LShr32 = DAG.getNode(Opcode: RISCVISD::SRL_VL, DL, VT: VecVT, N1: Vec, N2: ThirtyTwoV,
16502 N3: DAG.getUNDEF(VT: VecVT), N4: Mask, N5: VL);
16503 SDValue EltHi = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: LShr32);
16504
16505 Results.push_back(
16506 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: EltLo, N2: EltHi));
16507 break;
16508 }
16509 }
16510 break;
16511 }
16512 case ISD::VECREDUCE_ADD:
16513 case ISD::VECREDUCE_AND:
16514 case ISD::VECREDUCE_OR:
16515 case ISD::VECREDUCE_XOR:
16516 case ISD::VECREDUCE_SMAX:
16517 case ISD::VECREDUCE_UMAX:
16518 case ISD::VECREDUCE_SMIN:
16519 case ISD::VECREDUCE_UMIN:
16520 if (SDValue V = lowerVECREDUCE(Op: SDValue(N, 0), DAG))
16521 Results.push_back(Elt: V);
16522 break;
16523 case ISD::VP_REDUCE_ADD:
16524 case ISD::VP_REDUCE_AND:
16525 case ISD::VP_REDUCE_OR:
16526 case ISD::VP_REDUCE_XOR:
16527 case ISD::VP_REDUCE_SMAX:
16528 case ISD::VP_REDUCE_UMAX:
16529 case ISD::VP_REDUCE_SMIN:
16530 case ISD::VP_REDUCE_UMIN:
16531 if (SDValue V = lowerVPREDUCE(Op: SDValue(N, 0), DAG))
16532 Results.push_back(Elt: V);
16533 break;
16534 case ISD::GET_ROUNDING: {
16535 SDVTList VTs = DAG.getVTList(VT1: Subtarget.getXLenVT(), VT2: MVT::Other);
16536 SDValue Res = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL, VTList: VTs, N: N->getOperand(Num: 0));
16537 Results.push_back(Elt: Res.getValue(R: 0));
16538 Results.push_back(Elt: Res.getValue(R: 1));
16539 break;
16540 }
16541 }
16542}
16543
16544/// Given a binary operator, return the *associative* generic ISD::VECREDUCE_OP
16545/// which corresponds to it.
16546static unsigned getVecReduceOpcode(unsigned Opc) {
16547 switch (Opc) {
16548 default:
16549 llvm_unreachable("Unhandled binary to transform reduction");
16550 case ISD::ADD:
16551 return ISD::VECREDUCE_ADD;
16552 case ISD::UMAX:
16553 return ISD::VECREDUCE_UMAX;
16554 case ISD::SMAX:
16555 return ISD::VECREDUCE_SMAX;
16556 case ISD::UMIN:
16557 return ISD::VECREDUCE_UMIN;
16558 case ISD::SMIN:
16559 return ISD::VECREDUCE_SMIN;
16560 case ISD::AND:
16561 return ISD::VECREDUCE_AND;
16562 case ISD::OR:
16563 return ISD::VECREDUCE_OR;
16564 case ISD::XOR:
16565 return ISD::VECREDUCE_XOR;
16566 case ISD::FADD:
16567 // Note: This is the associative form of the generic reduction opcode.
16568 return ISD::VECREDUCE_FADD;
16569 case ISD::FMAXNUM:
16570 return ISD::VECREDUCE_FMAX;
16571 case ISD::FMINNUM:
16572 return ISD::VECREDUCE_FMIN;
16573 }
16574}
16575
16576/// Perform two related transforms whose purpose is to incrementally recognize
16577/// an explode_vector followed by scalar reduction as a vector reduction node.
16578/// This exists to recover from a deficiency in SLP which can't handle
16579/// forests with multiple roots sharing common nodes. In some cases, one
16580/// of the trees will be vectorized, and the other will remain (unprofitably)
16581/// scalarized.
16582static SDValue
16583combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
16584 const RISCVSubtarget &Subtarget) {
16585
16586 // This transforms need to run before all integer types have been legalized
16587 // to i64 (so that the vector element type matches the add type), and while
16588 // it's safe to introduce odd sized vector types.
16589 if (DAG.NewNodesMustHaveLegalTypes)
16590 return SDValue();
16591
16592 // Without V, this transform isn't useful. We could form the (illegal)
16593 // operations and let them be scalarized again, but there's really no point.
16594 if (!Subtarget.hasVInstructions())
16595 return SDValue();
16596
16597 const SDLoc DL(N);
16598 const EVT VT = N->getValueType(ResNo: 0);
16599 const unsigned Opc = N->getOpcode();
16600
16601 if (!VT.isInteger()) {
16602 switch (Opc) {
16603 default:
16604 return SDValue();
16605 case ISD::FADD:
16606 // For FADD, we only handle the case with reassociation allowed. We
16607 // could handle strict reduction order, but at the moment, there's no
16608 // known reason to, and the complexity isn't worth it.
16609 if (!N->getFlags().hasAllowReassociation())
16610 return SDValue();
16611 break;
16612 case ISD::FMAXNUM:
16613 case ISD::FMINNUM:
16614 break;
16615 }
16616 }
16617
16618 const unsigned ReduceOpc = getVecReduceOpcode(Opc);
16619 assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) &&
16620 "Inconsistent mappings");
16621 SDValue LHS = N->getOperand(Num: 0);
16622 SDValue RHS = N->getOperand(Num: 1);
16623
16624 if (!LHS.hasOneUse() || !RHS.hasOneUse())
16625 return SDValue();
16626
16627 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16628 std::swap(a&: LHS, b&: RHS);
16629
16630 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
16631 !isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)))
16632 return SDValue();
16633
16634 uint64_t RHSIdx = cast<ConstantSDNode>(Val: RHS.getOperand(i: 1))->getLimitedValue();
16635 SDValue SrcVec = RHS.getOperand(i: 0);
16636 EVT SrcVecVT = SrcVec.getValueType();
16637 assert(SrcVecVT.getVectorElementType() == VT);
16638 if (SrcVecVT.isScalableVector())
16639 return SDValue();
16640
16641 if (SrcVecVT.getScalarSizeInBits() > Subtarget.getELen())
16642 return SDValue();
16643
16644 // match binop (extract_vector_elt V, 0), (extract_vector_elt V, 1) to
16645 // reduce_op (extract_subvector [2 x VT] from V). This will form the
16646 // root of our reduction tree. TODO: We could extend this to any two
16647 // adjacent aligned constant indices if desired.
16648 if (LHS.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
16649 LHS.getOperand(i: 0) == SrcVec && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
16650 uint64_t LHSIdx =
16651 cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))->getLimitedValue();
16652 if (0 == std::min(a: LHSIdx, b: RHSIdx) && 1 == std::max(a: LHSIdx, b: RHSIdx)) {
16653 EVT ReduceVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 2);
16654 SDValue Vec = DAG.getExtractSubvector(DL, VT: ReduceVT, Vec: SrcVec, Idx: 0);
16655 return DAG.getNode(Opcode: ReduceOpc, DL, VT, Operand: Vec, Flags: N->getFlags());
16656 }
16657 }
16658
16659 // Match (binop (reduce (extract_subvector V, 0),
16660 // (extract_vector_elt V, sizeof(SubVec))))
16661 // into a reduction of one more element from the original vector V.
16662 if (LHS.getOpcode() != ReduceOpc)
16663 return SDValue();
16664
16665 SDValue ReduceVec = LHS.getOperand(i: 0);
16666 if (ReduceVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
16667 ReduceVec.hasOneUse() && ReduceVec.getOperand(i: 0) == RHS.getOperand(i: 0) &&
16668 isNullConstant(V: ReduceVec.getOperand(i: 1)) &&
16669 ReduceVec.getValueType().getVectorNumElements() == RHSIdx) {
16670 // For illegal types (e.g. 3xi32), most will be combined again into a
16671 // wider (hopefully legal) type. If this is a terminal state, we are
16672 // relying on type legalization here to produce something reasonable
16673 // and this lowering quality could probably be improved. (TODO)
16674 EVT ReduceVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: RHSIdx + 1);
16675 SDValue Vec = DAG.getExtractSubvector(DL, VT: ReduceVT, Vec: SrcVec, Idx: 0);
16676 return DAG.getNode(Opcode: ReduceOpc, DL, VT, Operand: Vec,
16677 Flags: ReduceVec->getFlags() & N->getFlags());
16678 }
16679
16680 return SDValue();
16681}
16682
16683
16684// Try to fold (<bop> x, (reduction.<bop> vec, start))
16685static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
16686 const RISCVSubtarget &Subtarget) {
16687 auto BinOpToRVVReduce = [](unsigned Opc) {
16688 switch (Opc) {
16689 default:
16690 llvm_unreachable("Unhandled binary to transform reduction");
16691 case ISD::ADD:
16692 return RISCVISD::VECREDUCE_ADD_VL;
16693 case ISD::UMAX:
16694 return RISCVISD::VECREDUCE_UMAX_VL;
16695 case ISD::SMAX:
16696 return RISCVISD::VECREDUCE_SMAX_VL;
16697 case ISD::UMIN:
16698 return RISCVISD::VECREDUCE_UMIN_VL;
16699 case ISD::SMIN:
16700 return RISCVISD::VECREDUCE_SMIN_VL;
16701 case ISD::AND:
16702 return RISCVISD::VECREDUCE_AND_VL;
16703 case ISD::OR:
16704 return RISCVISD::VECREDUCE_OR_VL;
16705 case ISD::XOR:
16706 return RISCVISD::VECREDUCE_XOR_VL;
16707 case ISD::FADD:
16708 return RISCVISD::VECREDUCE_FADD_VL;
16709 case ISD::FMAXNUM:
16710 return RISCVISD::VECREDUCE_FMAX_VL;
16711 case ISD::FMINNUM:
16712 return RISCVISD::VECREDUCE_FMIN_VL;
16713 }
16714 };
16715
16716 auto IsReduction = [&BinOpToRVVReduce](SDValue V, unsigned Opc) {
16717 return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
16718 isNullConstant(V: V.getOperand(i: 1)) &&
16719 V.getOperand(i: 0).getOpcode() == BinOpToRVVReduce(Opc);
16720 };
16721
16722 unsigned Opc = N->getOpcode();
16723 unsigned ReduceIdx;
16724 if (IsReduction(N->getOperand(Num: 0), Opc))
16725 ReduceIdx = 0;
16726 else if (IsReduction(N->getOperand(Num: 1), Opc))
16727 ReduceIdx = 1;
16728 else
16729 return SDValue();
16730
16731 // Skip if FADD disallows reassociation but the combiner needs.
16732 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation())
16733 return SDValue();
16734
16735 SDValue Extract = N->getOperand(Num: ReduceIdx);
16736 SDValue Reduce = Extract.getOperand(i: 0);
16737 if (!Extract.hasOneUse() || !Reduce.hasOneUse())
16738 return SDValue();
16739
16740 SDValue ScalarV = Reduce.getOperand(i: 2);
16741 EVT ScalarVT = ScalarV.getValueType();
16742 if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR &&
16743 ScalarV.getOperand(i: 0)->isUndef() &&
16744 isNullConstant(V: ScalarV.getOperand(i: 2)))
16745 ScalarV = ScalarV.getOperand(i: 1);
16746
16747 // Make sure that ScalarV is a splat with VL=1.
16748 if (ScalarV.getOpcode() != RISCVISD::VFMV_S_F_VL &&
16749 ScalarV.getOpcode() != RISCVISD::VMV_S_X_VL &&
16750 ScalarV.getOpcode() != RISCVISD::VMV_V_X_VL)
16751 return SDValue();
16752
16753 if (!isNonZeroAVL(AVL: ScalarV.getOperand(i: 2)))
16754 return SDValue();
16755
16756 // Check the scalar of ScalarV is neutral element
16757 // TODO: Deal with value other than neutral element.
16758 if (!DAG.isIdentityElement(Opc: N->getOpcode(), Flags: N->getFlags(),
16759 V: ScalarV.getOperand(i: 1), OperandNo: 0))
16760 return SDValue();
16761
16762 // If the AVL is zero, operand 0 will be returned. So it's not safe to fold.
16763 // FIXME: We might be able to improve this if operand 0 is undef.
16764 if (!isNonZeroAVL(AVL: Reduce.getOperand(i: 5)))
16765 return SDValue();
16766
16767 SDValue NewStart = N->getOperand(Num: 1 - ReduceIdx);
16768
16769 SDLoc DL(N);
16770 SDValue NewScalarV =
16771 lowerScalarInsert(Scalar: NewStart, VL: ScalarV.getOperand(i: 2),
16772 VT: ScalarV.getSimpleValueType(), DL, DAG, Subtarget);
16773
16774 // If we looked through an INSERT_SUBVECTOR we need to restore it.
16775 if (ScalarVT != ScalarV.getValueType())
16776 NewScalarV =
16777 DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ScalarVT), SubVec: NewScalarV, Idx: 0);
16778
16779 SDValue Ops[] = {Reduce.getOperand(i: 0), Reduce.getOperand(i: 1),
16780 NewScalarV, Reduce.getOperand(i: 3),
16781 Reduce.getOperand(i: 4), Reduce.getOperand(i: 5)};
16782 SDValue NewReduce =
16783 DAG.getNode(Opcode: Reduce.getOpcode(), DL, VT: Reduce.getValueType(), Ops);
16784 return DAG.getNode(Opcode: Extract.getOpcode(), DL, VT: Extract.getValueType(), N1: NewReduce,
16785 N2: Extract.getOperand(i: 1));
16786}
16787
16788// Optimize (add (shl x, c0), (shl y, c1)) ->
16789// (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
16790// or
16791// (SLLI (QC.SHLADD x, y, c1 - c0), c0), if 4 <= (c1-c0) <=31.
16792static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
16793 const RISCVSubtarget &Subtarget) {
16794 // Perform this optimization only in the zba/xandesperf/xqciac/xtheadba
16795 // extension.
16796 if (!Subtarget.hasShlAdd(ShAmt: 3))
16797 return SDValue();
16798
16799 // Skip for vector types and larger types.
16800 EVT VT = N->getValueType(ResNo: 0);
16801 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
16802 return SDValue();
16803
16804 // The two operand nodes must be SHL and have no other use.
16805 SDValue N0 = N->getOperand(Num: 0);
16806 SDValue N1 = N->getOperand(Num: 1);
16807 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
16808 !N0->hasOneUse() || !N1->hasOneUse())
16809 return SDValue();
16810
16811 // Check c0 and c1.
16812 auto *N0C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
16813 auto *N1C = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1));
16814 if (!N0C || !N1C)
16815 return SDValue();
16816 int64_t C0 = N0C->getSExtValue();
16817 int64_t C1 = N1C->getSExtValue();
16818 if (C0 <= 0 || C1 <= 0)
16819 return SDValue();
16820
16821 int64_t Diff = std::abs(i: C0 - C1);
16822 if (!Subtarget.hasShlAdd(ShAmt: Diff))
16823 return SDValue();
16824
16825 // Build nodes.
16826 SDLoc DL(N);
16827 int64_t Bits = std::min(a: C0, b: C1);
16828 SDValue NS = (C0 < C1) ? N0->getOperand(Num: 0) : N1->getOperand(Num: 0);
16829 SDValue NL = (C0 > C1) ? N0->getOperand(Num: 0) : N1->getOperand(Num: 0);
16830 SDValue SHADD = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: NL,
16831 N2: DAG.getTargetConstant(Val: Diff, DL, VT), N3: NS);
16832 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: SHADD, N2: DAG.getConstant(Val: Bits, DL, VT));
16833}
16834
16835// Check if this SDValue is an add immediate that is fed by a shift of 1, 2,
16836// or 3.
16837static SDValue combineShlAddIAddImpl(SDNode *N, SDValue AddI, SDValue Other,
16838 SelectionDAG &DAG) {
16839 using namespace llvm::SDPatternMatch;
16840
16841 // Looking for a reg-reg add and not an addi.
16842 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
16843 return SDValue();
16844
16845 // Based on testing it seems that performance degrades if the ADDI has
16846 // more than 2 uses.
16847 if (AddI->use_size() > 2)
16848 return SDValue();
16849
16850 APInt AddVal;
16851 SDValue SHLVal;
16852 if (!sd_match(N: AddI, P: m_Add(L: m_Value(N&: SHLVal), R: m_ConstInt(V&: AddVal))))
16853 return SDValue();
16854
16855 APInt VShift;
16856 if (!sd_match(N: SHLVal, P: m_OneUse(P: m_Shl(L: m_Value(), R: m_ConstInt(V&: VShift)))))
16857 return SDValue();
16858
16859 if (VShift.slt(RHS: 1) || VShift.sgt(RHS: 3))
16860 return SDValue();
16861
16862 SDLoc DL(N);
16863 EVT VT = N->getValueType(ResNo: 0);
16864 // The shift must be positive but the add can be signed.
16865 uint64_t ShlConst = VShift.getZExtValue();
16866 int64_t AddConst = AddVal.getSExtValue();
16867
16868 SDValue SHADD = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: SHLVal->getOperand(Num: 0),
16869 N2: DAG.getTargetConstant(Val: ShlConst, DL, VT), N3: Other);
16870 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: SHADD,
16871 N2: DAG.getSignedConstant(Val: AddConst, DL, VT));
16872}
16873
16874// Optimize (add (add (shl x, c0), c1), y) ->
16875// (ADDI (SH*ADD y, x), c1), if c0 equals to [1|2|3].
16876static SDValue combineShlAddIAdd(SDNode *N, SelectionDAG &DAG,
16877 const RISCVSubtarget &Subtarget) {
16878 // Perform this optimization only in the zba extension.
16879 if (!ReassocShlAddiAdd || !Subtarget.hasShlAdd(ShAmt: 3))
16880 return SDValue();
16881
16882 // Skip for vector types and larger types.
16883 EVT VT = N->getValueType(ResNo: 0);
16884 if (VT != Subtarget.getXLenVT())
16885 return SDValue();
16886
16887 SDValue AddI = N->getOperand(Num: 0);
16888 SDValue Other = N->getOperand(Num: 1);
16889 if (SDValue V = combineShlAddIAddImpl(N, AddI, Other, DAG))
16890 return V;
16891 if (SDValue V = combineShlAddIAddImpl(N, AddI: Other, Other: AddI, DAG))
16892 return V;
16893 return SDValue();
16894}
16895
16896// Combine a constant select operand into its use:
16897//
16898// (and (select cond, -1, c), x)
16899// -> (select cond, x, (and x, c)) [AllOnes=1]
16900// (or (select cond, 0, c), x)
16901// -> (select cond, x, (or x, c)) [AllOnes=0]
16902// (xor (select cond, 0, c), x)
16903// -> (select cond, x, (xor x, c)) [AllOnes=0]
16904// (add (select cond, 0, c), x)
16905// -> (select cond, x, (add x, c)) [AllOnes=0]
16906// (sub x, (select cond, 0, c))
16907// -> (select cond, x, (sub x, c)) [AllOnes=0]
16908static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
16909 SelectionDAG &DAG, bool AllOnes,
16910 const RISCVSubtarget &Subtarget) {
16911 EVT VT = N->getValueType(ResNo: 0);
16912
16913 // Skip vectors.
16914 if (VT.isVector())
16915 return SDValue();
16916
16917 if (!Subtarget.hasConditionalMoveFusion()) {
16918 // (select cond, x, (and x, c)) has custom lowering with Zicond.
16919 if (!Subtarget.hasCZEROLike() || N->getOpcode() != ISD::AND)
16920 return SDValue();
16921
16922 // Maybe harmful when condition code has multiple use.
16923 if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(i: 0).hasOneUse())
16924 return SDValue();
16925
16926 // Maybe harmful when VT is wider than XLen.
16927 if (VT.getSizeInBits() > Subtarget.getXLen())
16928 return SDValue();
16929 }
16930
16931 if ((Slct.getOpcode() != ISD::SELECT &&
16932 Slct.getOpcode() != RISCVISD::SELECT_CC) ||
16933 !Slct.hasOneUse())
16934 return SDValue();
16935
16936 auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
16937 return AllOnes ? isAllOnesConstant(V: N) : isNullConstant(V: N);
16938 };
16939
16940 bool SwapSelectOps;
16941 unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
16942 SDValue TrueVal = Slct.getOperand(i: 1 + OpOffset);
16943 SDValue FalseVal = Slct.getOperand(i: 2 + OpOffset);
16944 SDValue NonConstantVal;
16945 if (isZeroOrAllOnes(TrueVal, AllOnes)) {
16946 SwapSelectOps = false;
16947 NonConstantVal = FalseVal;
16948 } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
16949 SwapSelectOps = true;
16950 NonConstantVal = TrueVal;
16951 } else
16952 return SDValue();
16953
16954 // Slct is now know to be the desired identity constant when CC is true.
16955 TrueVal = OtherOp;
16956 FalseVal = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, N1: OtherOp, N2: NonConstantVal);
16957 // Unless SwapSelectOps says the condition should be false.
16958 if (SwapSelectOps)
16959 std::swap(a&: TrueVal, b&: FalseVal);
16960
16961 if (Slct.getOpcode() == RISCVISD::SELECT_CC)
16962 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL: SDLoc(N), VT,
16963 Ops: {Slct.getOperand(i: 0), Slct.getOperand(i: 1),
16964 Slct.getOperand(i: 2), TrueVal, FalseVal});
16965
16966 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT,
16967 Ops: {Slct.getOperand(i: 0), TrueVal, FalseVal});
16968}
16969
16970// Attempt combineSelectAndUse on each operand of a commutative operator N.
16971static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
16972 bool AllOnes,
16973 const RISCVSubtarget &Subtarget) {
16974 SDValue N0 = N->getOperand(Num: 0);
16975 SDValue N1 = N->getOperand(Num: 1);
16976 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DAG, AllOnes, Subtarget))
16977 return Result;
16978 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DAG, AllOnes, Subtarget))
16979 return Result;
16980 return SDValue();
16981}
16982
16983// Transform (add (mul x, c0), c1) ->
16984// (add (mul (add x, c1/c0), c0), c1%c0).
16985// if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
16986// that should be excluded is when c0*(c1/c0) is simm12, which will lead
16987// to an infinite loop in DAGCombine if transformed.
16988// Or transform (add (mul x, c0), c1) ->
16989// (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
16990// if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
16991// case that should be excluded is when c0*(c1/c0+1) is simm12, which will
16992// lead to an infinite loop in DAGCombine if transformed.
16993// Or transform (add (mul x, c0), c1) ->
16994// (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
16995// if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
16996// case that should be excluded is when c0*(c1/c0-1) is simm12, which will
16997// lead to an infinite loop in DAGCombine if transformed.
16998// Or transform (add (mul x, c0), c1) ->
16999// (mul (add x, c1/c0), c0).
17000// if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
17001static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
17002 const RISCVSubtarget &Subtarget) {
17003 // Skip for vector types and larger types.
17004 EVT VT = N->getValueType(ResNo: 0);
17005 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
17006 return SDValue();
17007 // The first operand node must be a MUL and has no other use.
17008 SDValue N0 = N->getOperand(Num: 0);
17009 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
17010 return SDValue();
17011 // Check if c0 and c1 match above conditions.
17012 auto *N0C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
17013 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17014 if (!N0C || !N1C)
17015 return SDValue();
17016 // If N0C has multiple uses it's possible one of the cases in
17017 // DAGCombiner::isMulAddWithConstProfitable will be true, which would result
17018 // in an infinite loop.
17019 if (!N0C->hasOneUse())
17020 return SDValue();
17021 int64_t C0 = N0C->getSExtValue();
17022 int64_t C1 = N1C->getSExtValue();
17023 int64_t CA, CB;
17024 // If C1 already fits in an add immediate, there is nothing to split out: the
17025 // (add (mul x, c0), c1) form is already canonical/cheap. Splitting it would
17026 // fight the generic DAGCombiner fold add(mul(add(A, CA), CM), CB) ->
17027 // add(mul(A, CM), CM*CA+CB) (which is gated on isLegalAddImmediate) and cause
17028 // an infinite loop.
17029 if (C0 == -1 || C0 == 0 || C0 == 1 ||
17030 Subtarget.getTargetLowering()->isLegalAddImmediate(Imm: C1))
17031 return SDValue();
17032 // Search for proper CA (non-zero) and CB that both are simm12.
17033 if ((C1 / C0) != 0 && isInt<12>(x: C1 / C0) && isInt<12>(x: C1 % C0) &&
17034 !isInt<12>(x: C0 * (C1 / C0))) {
17035 CA = C1 / C0;
17036 CB = C1 % C0;
17037 } else if ((C1 / C0 + 1) != 0 && isInt<12>(x: C1 / C0 + 1) &&
17038 isInt<12>(x: C1 % C0 - C0) && !isInt<12>(x: C0 * (C1 / C0 + 1))) {
17039 CA = C1 / C0 + 1;
17040 CB = C1 % C0 - C0;
17041 } else if ((C1 / C0 - 1) != 0 && isInt<12>(x: C1 / C0 - 1) &&
17042 isInt<12>(x: C1 % C0 + C0) && !isInt<12>(x: C0 * (C1 / C0 - 1))) {
17043 CA = C1 / C0 - 1;
17044 CB = C1 % C0 + C0;
17045 } else
17046 return SDValue();
17047 // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
17048 SDLoc DL(N);
17049 SDValue New0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0->getOperand(Num: 0),
17050 N2: DAG.getSignedConstant(Val: CA, DL, VT));
17051 SDValue New1 =
17052 DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: New0, N2: DAG.getSignedConstant(Val: C0, DL, VT));
17053 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: New1, N2: DAG.getSignedConstant(Val: CB, DL, VT));
17054}
17055
17056// add (zext, zext) -> zext (add (zext, zext))
17057// sub (zext, zext) -> sext (sub (zext, zext))
17058// mul (zext, zext) -> zext (mul (zext, zext))
17059// sdiv (zext, zext) -> zext (sdiv (zext, zext))
17060// udiv (zext, zext) -> zext (udiv (zext, zext))
17061// srem (zext, zext) -> zext (srem (zext, zext))
17062// urem (zext, zext) -> zext (urem (zext, zext))
17063//
17064// where the sum of the extend widths match, and the the range of the bin op
17065// fits inside the width of the narrower bin op. (For profitability on rvv, we
17066// use a power of two for both inner and outer extend.)
17067static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) {
17068
17069 EVT VT = N->getValueType(ResNo: 0);
17070 if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
17071 return SDValue();
17072
17073 SDValue N0 = N->getOperand(Num: 0);
17074 SDValue N1 = N->getOperand(Num: 1);
17075 if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND)
17076 return SDValue();
17077 if (!N0.hasOneUse() || !N1.hasOneUse())
17078 return SDValue();
17079
17080 SDValue Src0 = N0.getOperand(i: 0);
17081 SDValue Src1 = N1.getOperand(i: 0);
17082 EVT SrcVT = Src0.getValueType();
17083 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT: SrcVT) ||
17084 SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 ||
17085 SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2)
17086 return SDValue();
17087
17088 LLVMContext &C = *DAG.getContext();
17089 EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(Context&: C);
17090 EVT NarrowVT = EVT::getVectorVT(Context&: C, VT: ElemVT, EC: VT.getVectorElementCount());
17091
17092 Src0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Src0), VT: NarrowVT, Operand: Src0);
17093 Src1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Src1), VT: NarrowVT, Operand: Src1);
17094
17095 // Src0 and Src1 are zero extended, so they're always positive if signed.
17096 //
17097 // sub can produce a negative from two positive operands, so it needs sign
17098 // extended. Other nodes produce a positive from two positive operands, so
17099 // zero extend instead.
17100 unsigned OuterExtend =
17101 N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
17102
17103 return DAG.getNode(
17104 Opcode: OuterExtend, DL: SDLoc(N), VT,
17105 Operand: DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NarrowVT, N1: Src0, N2: Src1));
17106}
17107
17108// Try to turn (add (xor bool, 1) -1) into (neg bool).
17109static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
17110 SDValue N0 = N->getOperand(Num: 0);
17111 SDValue N1 = N->getOperand(Num: 1);
17112 EVT VT = N->getValueType(ResNo: 0);
17113 SDLoc DL(N);
17114
17115 // RHS should be -1.
17116 if (!isAllOnesConstant(V: N1))
17117 return SDValue();
17118
17119 // Look for (xor X, 1).
17120 if (N0.getOpcode() != ISD::XOR || !isOneConstant(V: N0.getOperand(i: 1)))
17121 return SDValue();
17122
17123 // First xor input should be 0 or 1.
17124 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
17125 if (!DAG.MaskedValueIsZero(Op: N0.getOperand(i: 0), Mask))
17126 return SDValue();
17127
17128 // Emit a negate of the setcc.
17129 return DAG.getNegative(Val: N0.getOperand(i: 0), DL, VT);
17130}
17131
17132// Fold (add X, (mulhs X, C)) -> (mulhsu X, C) if C is negative. This occurs
17133// in the expansion of sdiv i32 X, 7 using magic multiply.
17134//
17135// mulhs returns the hi from X * C = hi * 2^32 + lo.
17136//
17137// Since C<0, u(C) as an unsigned constant is 2^32 + C = u(C).
17138// mulhsu computes
17139// X * u(C0) = X * (C + 2^32)
17140// = X * 2^32 + C * X // C * X is the same as mulhs
17141// = X * 2^32 + hi * 2^32 + lo
17142// = (X + hi) * 2^32 + lo
17143// So mulhsu computes (X + hi).
17144static SDValue combineAddMulh(SDNode *N, SelectionDAG &DAG,
17145 const RISCVSubtarget &Subtarget) {
17146 EVT VT = N->getValueType(ResNo: 0);
17147 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17148 bool IsPExtPackedDoubleType =
17149 VT.isSimple() && Subtarget.isPExtPackedDoubleType(VT: VT.getSimpleVT());
17150 if (!TLI.isOperationLegal(Op: ISD::MULHS, VT) && !IsPExtPackedDoubleType &&
17151 !(Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8))
17152 return SDValue();
17153
17154 using namespace SDPatternMatch;
17155 SDValue X, Mulh;
17156 APInt C;
17157 if (!sd_match(N,
17158 P: m_Add(L: m_Value(N&: X),
17159 R: m_OneUse(P: m_Value(N&: Mulh, P: m_BinOp(Opc: ISD::MULHS, L: m_Deferred(V&: X),
17160 R: m_ConstInt(V&: C)))))) ||
17161 !C.isNegative())
17162 return SDValue();
17163
17164 SDLoc DL(N);
17165
17166 // We don't have a v4i8 MULHSU instruction, use a WMULSU+SRL+TRUNC.
17167 auto MakePWMulSU = [&](SDValue A, SDValue B) -> SDValue {
17168 SDValue WMul = DAG.getNode(Opcode: RISCVISD::PWMULSU, DL, VT: MVT::v4i16, N1: A, N2: B);
17169 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::v4i16, N1: WMul,
17170 N2: DAG.getConstant(Val: 8, DL, VT: MVT::v4i16));
17171 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::v4i8, Operand: Shifted);
17172 };
17173
17174 // We need to split double wide vectors ourselves, op legalization won't
17175 // run for custom nodes.
17176 if (IsPExtPackedDoubleType) {
17177 MVT HalfVT = VT.getSimpleVT().getHalfNumVectorElementsVT();
17178 auto [XLo, XHi] = DAG.SplitVector(N: X, DL, LoVT: HalfVT, HiVT: HalfVT);
17179 auto [CLo, CHi] = DAG.SplitVector(N: Mulh.getOperand(i: 1), DL, LoVT: HalfVT, HiVT: HalfVT);
17180 SDValue ResLo, ResHi;
17181 if (HalfVT == MVT::v4i8) {
17182 ResLo = MakePWMulSU(XLo, CLo);
17183 ResHi = MakePWMulSU(XHi, CHi);
17184 } else {
17185 ResLo = DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT: HalfVT, N1: XLo, N2: CLo);
17186 ResHi = DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT: HalfVT, N1: XHi, N2: CHi);
17187 }
17188 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: ResLo, N2: ResHi);
17189 }
17190
17191 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit() && VT == MVT::v4i8)
17192 return MakePWMulSU(X, Mulh.getOperand(i: 1));
17193
17194 return DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT, N1: X, N2: Mulh.getOperand(i: 1));
17195}
17196
17197static SDValue performADDCombine(SDNode *N,
17198 TargetLowering::DAGCombinerInfo &DCI,
17199 const RISCVSubtarget &Subtarget) {
17200 SelectionDAG &DAG = DCI.DAG;
17201 if (SDValue V = combineAddOfBooleanXor(N, DAG))
17202 return V;
17203 if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
17204 return V;
17205 if (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) {
17206 if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
17207 return V;
17208 if (SDValue V = combineShlAddIAdd(N, DAG, Subtarget))
17209 return V;
17210 }
17211 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17212 return V;
17213 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17214 return V;
17215 if (SDValue V = combineBinOpOfZExt(N, DAG))
17216 return V;
17217 if (SDValue V = combineAddMulh(N, DAG, Subtarget))
17218 return V;
17219
17220 // fold (add (select lhs, rhs, cc, 0, y), x) ->
17221 // (select lhs, rhs, cc, x, (add x, y))
17222 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
17223}
17224
17225// Try to turn a sub boolean RHS and constant LHS into an addi.
17226static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
17227 SDValue N0 = N->getOperand(Num: 0);
17228 SDValue N1 = N->getOperand(Num: 1);
17229 EVT VT = N->getValueType(ResNo: 0);
17230 SDLoc DL(N);
17231
17232 // Require a constant LHS.
17233 auto *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
17234 if (!N0C)
17235 return SDValue();
17236
17237 // All our optimizations involve subtracting 1 from the immediate and forming
17238 // an ADDI. Make sure the new immediate is valid for an ADDI.
17239 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
17240 if (!ImmValMinus1.isSignedIntN(N: 12))
17241 return SDValue();
17242
17243 SDValue NewLHS;
17244 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
17245 // (sub constant, (setcc x, y, eq/neq)) ->
17246 // (add (setcc x, y, neq/eq), constant - 1)
17247 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
17248 EVT SetCCOpVT = N1.getOperand(i: 0).getValueType();
17249 if (!isIntEqualitySetCC(Code: CCVal) || !SetCCOpVT.isInteger())
17250 return SDValue();
17251 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: SetCCOpVT);
17252 NewLHS =
17253 DAG.getSetCC(DL: SDLoc(N1), VT, LHS: N1.getOperand(i: 0), RHS: N1.getOperand(i: 1), Cond: CCVal);
17254 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(V: N1.getOperand(i: 1)) &&
17255 N1.getOperand(i: 0).getOpcode() == ISD::SETCC) {
17256 // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
17257 // Since setcc returns a bool the xor is equivalent to 1-setcc.
17258 NewLHS = N1.getOperand(i: 0);
17259 } else
17260 return SDValue();
17261
17262 SDValue NewRHS = DAG.getConstant(Val: ImmValMinus1, DL, VT);
17263 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NewLHS, N2: NewRHS);
17264}
17265
17266// Looks for (sub (shl X, 8-Y), (shr X, Y)) where the Y-th bit in each byte is
17267// potentially set. It is fine for Y to be 0, meaning that (sub (shl X, 8), X)
17268// is also valid. Replace with (orc.b X). For example, 0b0000_1000_0000_1000 is
17269// valid with Y=3, while 0b0000_1000_0000_0100 is not.
17270static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
17271 const RISCVSubtarget &Subtarget) {
17272 if (!Subtarget.hasStdExtZbb())
17273 return SDValue();
17274
17275 EVT VT = N->getValueType(ResNo: 0);
17276
17277 if (VT != Subtarget.getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
17278 return SDValue();
17279
17280 SDValue N0 = N->getOperand(Num: 0);
17281 SDValue N1 = N->getOperand(Num: 1);
17282
17283 if (N0->getOpcode() != ISD::SHL)
17284 return SDValue();
17285
17286 auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
17287 if (!ShAmtCLeft)
17288 return SDValue();
17289 unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
17290
17291 if (ShiftedAmount >= 8)
17292 return SDValue();
17293
17294 SDValue LeftShiftOperand = N0->getOperand(Num: 0);
17295 SDValue RightShiftOperand = N1;
17296
17297 if (ShiftedAmount != 0) { // Right operand must be a right shift.
17298 if (N1->getOpcode() != ISD::SRL)
17299 return SDValue();
17300 auto *ShAmtCRight = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
17301 if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
17302 return SDValue();
17303 RightShiftOperand = N1.getOperand(i: 0);
17304 }
17305
17306 // At least one shift should have a single use.
17307 if (!N0.hasOneUse() && (ShiftedAmount == 0 || !N1.hasOneUse()))
17308 return SDValue();
17309
17310 if (LeftShiftOperand != RightShiftOperand)
17311 return SDValue();
17312
17313 APInt Mask = APInt::getSplat(NewLen: VT.getSizeInBits(), V: APInt(8, 0x1));
17314 Mask <<= ShiftedAmount;
17315 // Check that X has indeed the right shape (only the Y-th bit can be set in
17316 // every byte).
17317 if (!DAG.MaskedValueIsZero(Op: LeftShiftOperand, Mask: ~Mask))
17318 return SDValue();
17319
17320 return DAG.getNode(Opcode: RISCVISD::ORC_B, DL: SDLoc(N), VT, Operand: LeftShiftOperand);
17321}
17322
17323static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
17324 const RISCVSubtarget &Subtarget) {
17325 if (SDValue V = combineSubOfBoolean(N, DAG))
17326 return V;
17327
17328 EVT VT = N->getValueType(ResNo: 0);
17329 SDValue N0 = N->getOperand(Num: 0);
17330 SDValue N1 = N->getOperand(Num: 1);
17331 // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
17332 if (isNullConstant(V: N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
17333 isNullConstant(V: N1.getOperand(i: 1)) &&
17334 N1.getValueType() == N1.getOperand(i: 0).getValueType()) {
17335 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
17336 if (CCVal == ISD::SETLT) {
17337 SDLoc DL(N);
17338 unsigned ShAmt = N0.getValueSizeInBits() - 1;
17339 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N1.getOperand(i: 0),
17340 N2: DAG.getConstant(Val: ShAmt, DL, VT));
17341 }
17342 }
17343
17344 if (SDValue V = combineBinOpOfZExt(N, DAG))
17345 return V;
17346 if (SDValue V = combineSubShiftToOrcB(N, DAG, Subtarget))
17347 return V;
17348
17349 // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
17350 // (select lhs, rhs, cc, x, (sub x, y))
17351 return combineSelectAndUse(N, Slct: N1, OtherOp: N0, DAG, /*AllOnes*/ false, Subtarget);
17352}
17353
17354// Apply DeMorgan's law to (and/or (xor X, 1), (xor Y, 1)) if X and Y are 0/1.
17355// Legalizing setcc can introduce xors like this. Doing this transform reduces
17356// the number of xors and may allow the xor to fold into a branch condition.
17357static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG) {
17358 SDValue N0 = N->getOperand(Num: 0);
17359 SDValue N1 = N->getOperand(Num: 1);
17360 bool IsAnd = N->getOpcode() == ISD::AND;
17361
17362 if (N0.getOpcode() != ISD::XOR || N1.getOpcode() != ISD::XOR)
17363 return SDValue();
17364
17365 if (!N0.hasOneUse() || !N1.hasOneUse())
17366 return SDValue();
17367
17368 SDValue N01 = N0.getOperand(i: 1);
17369 SDValue N11 = N1.getOperand(i: 1);
17370
17371 // For AND, SimplifyDemandedBits may have turned one of the (xor X, 1) into
17372 // (xor X, -1) based on the upper bits of the other operand being 0. If the
17373 // operation is And, allow one of the Xors to use -1.
17374 if (isOneConstant(V: N01)) {
17375 if (!isOneConstant(V: N11) && !(IsAnd && isAllOnesConstant(V: N11)))
17376 return SDValue();
17377 } else if (isOneConstant(V: N11)) {
17378 // N01 and N11 being 1 was already handled. Handle N11==1 and N01==-1.
17379 if (!(IsAnd && isAllOnesConstant(V: N01)))
17380 return SDValue();
17381 } else
17382 return SDValue();
17383
17384 EVT VT = N->getValueType(ResNo: 0);
17385
17386 SDValue N00 = N0.getOperand(i: 0);
17387 SDValue N10 = N1.getOperand(i: 0);
17388
17389 // The LHS of the xors needs to be 0/1.
17390 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
17391 if (!DAG.MaskedValueIsZero(Op: N00, Mask) || !DAG.MaskedValueIsZero(Op: N10, Mask))
17392 return SDValue();
17393
17394 // Invert the opcode and insert a new xor.
17395 SDLoc DL(N);
17396 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
17397 SDValue Logic = DAG.getNode(Opcode: Opc, DL, VT, N1: N00, N2: N10);
17398 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Logic, N2: DAG.getConstant(Val: 1, DL, VT));
17399}
17400
17401// Fold (vXi8 (trunc (vselect (setltu, X, 256), X, (sext (setgt X, 0))))) to
17402// (vXi8 (trunc (smin (smax X, 0), 255))). This represents saturating a signed
17403// value to an unsigned value. This will be lowered to vmax and series of
17404// vnclipu instructions later. This can be extended to other truncated types
17405// other than i8 by replacing 256 and 255 with the equivalent constants for the
17406// type.
17407static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG) {
17408 EVT VT = N->getValueType(ResNo: 0);
17409 SDValue N0 = N->getOperand(Num: 0);
17410 EVT SrcVT = N0.getValueType();
17411
17412 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17413 if (!VT.isVector() || !TLI.isTypeLegal(VT) || !TLI.isTypeLegal(VT: SrcVT))
17414 return SDValue();
17415
17416 if (N0.getOpcode() != ISD::VSELECT || !N0.hasOneUse())
17417 return SDValue();
17418
17419 SDValue Cond = N0.getOperand(i: 0);
17420 SDValue True = N0.getOperand(i: 1);
17421 SDValue False = N0.getOperand(i: 2);
17422
17423 if (Cond.getOpcode() != ISD::SETCC)
17424 return SDValue();
17425
17426 // FIXME: Support the version of this pattern with the select operands
17427 // swapped.
17428 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
17429 if (CCVal != ISD::SETULT)
17430 return SDValue();
17431
17432 SDValue CondLHS = Cond.getOperand(i: 0);
17433 SDValue CondRHS = Cond.getOperand(i: 1);
17434
17435 if (CondLHS != True)
17436 return SDValue();
17437
17438 unsigned ScalarBits = VT.getScalarSizeInBits();
17439
17440 // FIXME: Support other constants.
17441 ConstantSDNode *CondRHSC = isConstOrConstSplat(N: CondRHS);
17442 if (!CondRHSC || CondRHSC->getAPIntValue() != (1ULL << ScalarBits))
17443 return SDValue();
17444
17445 if (False.getOpcode() != ISD::SIGN_EXTEND)
17446 return SDValue();
17447
17448 False = False.getOperand(i: 0);
17449
17450 if (False.getOpcode() != ISD::SETCC || False.getOperand(i: 0) != True)
17451 return SDValue();
17452
17453 ConstantSDNode *FalseRHSC = isConstOrConstSplat(N: False.getOperand(i: 1));
17454 if (!FalseRHSC || !FalseRHSC->isZero())
17455 return SDValue();
17456
17457 ISD::CondCode CCVal2 = cast<CondCodeSDNode>(Val: False.getOperand(i: 2))->get();
17458 if (CCVal2 != ISD::SETGT)
17459 return SDValue();
17460
17461 // Emit the signed to unsigned saturation pattern.
17462 SDLoc DL(N);
17463 SDValue Max =
17464 DAG.getNode(Opcode: ISD::SMAX, DL, VT: SrcVT, N1: True, N2: DAG.getConstant(Val: 0, DL, VT: SrcVT));
17465 SDValue Min =
17466 DAG.getNode(Opcode: ISD::SMIN, DL, VT: SrcVT, N1: Max,
17467 N2: DAG.getConstant(Val: (1ULL << ScalarBits) - 1, DL, VT: SrcVT));
17468 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Min);
17469}
17470
17471// Handle P extension truncate patterns, both on packed vectors and on scalar
17472// i32 (the RV32-only asub/asubu and mulhr* instructions):
17473// ASUB/ASUBU: (trunc (srl (sub ([s|z]ext a), ([s|z]ext b)), 1))
17474// MULHSU: (trunc (srl (mul (sext a), (zext b)), EltBits))
17475// MULHR*: (trunc (srl (add (mul (sext a), (zext b)), round_const), EltBits))
17476static SDValue combinePExtTruncate(SDNode *N, SelectionDAG &DAG,
17477 const RISCVSubtarget &Subtarget) {
17478 SDValue N0 = N->getOperand(Num: 0);
17479 EVT VT = N->getValueType(ResNo: 0);
17480 if (N0.getOpcode() != ISD::SRL)
17481 return SDValue();
17482
17483 if (VT != MVT::v4i16 && VT != MVT::v2i16 && VT != MVT::v8i8 &&
17484 VT != MVT::v4i8 && VT != MVT::v2i32 && VT != MVT::i32)
17485 return SDValue();
17486
17487 ConstantSDNode *C = isConstOrConstSplat(N: N0.getOperand(i: 1));
17488 if (!C)
17489 return SDValue();
17490
17491 SDValue Op = N0.getOperand(i: 0);
17492 unsigned ShAmtVal = C->getZExtValue();
17493 unsigned EltBits = VT.getScalarSizeInBits();
17494
17495 // Check for rounding pattern: (add (mul ...), round_const)
17496 bool IsRounding = false;
17497 if (Op.getOpcode() == ISD::ADD && (EltBits == 16 || EltBits == 32)) {
17498 ConstantSDNode *RndC = isConstOrConstSplat(N: Op.getOperand(i: 1));
17499 if (RndC && RndC->getZExtValue() == (1ULL << (EltBits - 1)) &&
17500 Op.getOperand(i: 0).getOpcode() == ISD::MUL) {
17501 Op = Op.getOperand(i: 0);
17502 IsRounding = true;
17503 }
17504 }
17505
17506 // Ensure Op is a binary operation before accessing its operands.
17507 if (Op.getNumOperands() != 2)
17508 return SDValue();
17509
17510 SDValue LHS = Op.getOperand(i: 0);
17511 SDValue RHS = Op.getOperand(i: 1);
17512
17513 bool LHSIsSExt = LHS.getOpcode() == ISD::SIGN_EXTEND;
17514 bool LHSIsZExt = LHS.getOpcode() == ISD::ZERO_EXTEND;
17515 bool RHSIsSExt = RHS.getOpcode() == ISD::SIGN_EXTEND;
17516 bool RHSIsZExt = RHS.getOpcode() == ISD::ZERO_EXTEND;
17517
17518 if (!(LHSIsSExt || LHSIsZExt) || !(RHSIsSExt || RHSIsZExt))
17519 return SDValue();
17520
17521 SDValue A = LHS.getOperand(i: 0);
17522 SDValue B = RHS.getOperand(i: 0);
17523
17524 if (A.getValueType() != VT || B.getValueType() != VT)
17525 return SDValue();
17526
17527 unsigned Opc;
17528 switch (Op.getOpcode()) {
17529 default:
17530 return SDValue();
17531 case ISD::SUB:
17532 // PASUB/PASUBU: shift amount must be 1
17533 if (ShAmtVal != 1)
17534 return SDValue();
17535 if (LHSIsSExt && RHSIsSExt)
17536 Opc = RISCVISD::ASUB;
17537 else if (LHSIsZExt && RHSIsZExt)
17538 Opc = RISCVISD::ASUBU;
17539 else
17540 return SDValue();
17541 break;
17542 case ISD::MUL:
17543 // MULH*/MULHR*: shift amount must be element size, only for i16/i32
17544 if (ShAmtVal != EltBits || (EltBits != 16 && EltBits != 32))
17545 return SDValue();
17546 if (!Subtarget.is64Bit() && (VT == MVT::v2i32 || VT == MVT::v4i16))
17547 return SDValue();
17548 if (IsRounding) {
17549 if (LHSIsSExt && RHSIsSExt) {
17550 Opc = RISCVISD::MULHR;
17551 } else if (LHSIsZExt && RHSIsZExt) {
17552 Opc = RISCVISD::MULHRU;
17553 } else if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
17554 Opc = RISCVISD::MULHRSU;
17555 // commuted case
17556 if (LHSIsZExt && RHSIsSExt)
17557 std::swap(a&: A, b&: B);
17558 } else {
17559 return SDValue();
17560 }
17561 } else {
17562 // Scalar mulhsu is handled elsewhere, only match the packed MULHSU here.
17563 if (!VT.isVector())
17564 return SDValue();
17565 if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
17566 Opc = RISCVISD::MULHSU;
17567 // commuted case
17568 if (LHSIsZExt && RHSIsSExt)
17569 std::swap(a&: A, b&: B);
17570 } else
17571 return SDValue();
17572 }
17573 break;
17574 }
17575
17576 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT, Ops: {A, B});
17577}
17578
17579static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
17580 const RISCVSubtarget &Subtarget) {
17581 SDValue N0 = N->getOperand(Num: 0);
17582 EVT VT = N->getValueType(ResNo: 0);
17583
17584 // P truncate patterns: packed vectors, plus RV32-only scalar i32.
17585 if (Subtarget.hasStdExtP() &&
17586 (VT.isFixedLengthVector() || (VT == MVT::i32 && !Subtarget.is64Bit())))
17587 return combinePExtTruncate(N, DAG, Subtarget);
17588
17589 // Pre-promote (i1 (truncate (srl X, Y))) on RV64 with Zbs without zero
17590 // extending X. This is safe since we only need the LSB after the shift and
17591 // shift amounts larger than 31 would produce poison. If we wait until
17592 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
17593 // to use a BEXT instruction.
17594 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
17595 N0.getValueType() == MVT::i32 && N0.getOpcode() == ISD::SRL &&
17596 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) && N0.hasOneUse()) {
17597 SDLoc DL(N0);
17598 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
17599 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
17600 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
17601 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT, Operand: Srl);
17602 }
17603
17604 return combineTruncSelectToSMaxUSat(N, DAG);
17605}
17606
17607// InstCombinerImpl::transformZExtICmp will narrow a zext of an icmp with a
17608// truncation. But RVV doesn't have truncation instructions for more than twice
17609// the bitwidth.
17610//
17611// E.g. trunc <vscale x 1 x i64> %x to <vscale x 1 x i8> will generate:
17612//
17613// vsetvli a0, zero, e32, m2, ta, ma
17614// vnsrl.wi v12, v8, 0
17615// vsetvli zero, zero, e16, m1, ta, ma
17616// vnsrl.wi v8, v12, 0
17617// vsetvli zero, zero, e8, mf2, ta, ma
17618// vnsrl.wi v8, v8, 0
17619//
17620// So reverse the combine so we generate an vmseq/vmsne again:
17621//
17622// and (lshr (trunc X), ShAmt), 1
17623// -->
17624// zext (icmp ne (and X, (1 << ShAmt)), 0)
17625//
17626// and (lshr (not (trunc X)), ShAmt), 1
17627// -->
17628// zext (icmp eq (and X, (1 << ShAmt)), 0)
17629static SDValue reverseZExtICmpCombine(SDNode *N, SelectionDAG &DAG,
17630 const RISCVSubtarget &Subtarget) {
17631 using namespace SDPatternMatch;
17632 SDLoc DL(N);
17633
17634 if (!Subtarget.hasVInstructions())
17635 return SDValue();
17636
17637 EVT VT = N->getValueType(ResNo: 0);
17638 if (!VT.isVector())
17639 return SDValue();
17640
17641 APInt ShAmt;
17642 SDValue Inner;
17643 if (!sd_match(N, P: m_And(L: m_OneUse(P: m_Srl(L: m_Value(N&: Inner), R: m_ConstInt(V&: ShAmt))),
17644 R: m_One())))
17645 return SDValue();
17646
17647 SDValue X;
17648 bool IsNot;
17649 if (sd_match(N: Inner, P: m_Not(V: m_Trunc(Op: m_Value(N&: X)))))
17650 IsNot = true;
17651 else if (sd_match(N: Inner, P: m_Trunc(Op: m_Value(N&: X))))
17652 IsNot = false;
17653 else
17654 return SDValue();
17655
17656 EVT WideVT = X.getValueType();
17657 if (VT.getScalarSizeInBits() >= WideVT.getScalarSizeInBits() / 2)
17658 return SDValue();
17659
17660 SDValue Res =
17661 DAG.getNode(Opcode: ISD::AND, DL, VT: WideVT, N1: X,
17662 N2: DAG.getConstant(Val: 1ULL << ShAmt.getZExtValue(), DL, VT: WideVT));
17663 Res = DAG.getSetCC(DL,
17664 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
17665 EC: WideVT.getVectorElementCount()),
17666 LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: WideVT),
17667 Cond: IsNot ? ISD::SETEQ : ISD::SETNE);
17668 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Res);
17669}
17670
17671// (and (i1) f, (setcc c, 0, ne)) -> (czero.nez f, c)
17672// (and (i1) f, (setcc c, 0, eq)) -> (czero.eqz f, c)
17673// (and (setcc c, 0, ne), (i1) g) -> (czero.nez g, c)
17674// (and (setcc c, 0, eq), (i1) g) -> (czero.eqz g, c)
17675static SDValue combineANDOfSETCCToCZERO(SDNode *N, SelectionDAG &DAG,
17676 const RISCVSubtarget &Subtarget) {
17677 if (!Subtarget.hasCZEROLike())
17678 return SDValue();
17679
17680 SDValue N0 = N->getOperand(Num: 0);
17681 SDValue N1 = N->getOperand(Num: 1);
17682
17683 auto IsEqualCompZero = [](SDValue &V) -> bool {
17684 if (V.getOpcode() == ISD::SETCC && isNullConstant(V: V.getOperand(i: 1))) {
17685 ISD::CondCode CC = cast<CondCodeSDNode>(Val: V.getOperand(i: 2))->get();
17686 if (ISD::isIntEqualitySetCC(Code: CC))
17687 return true;
17688 }
17689 return false;
17690 };
17691
17692 if (!IsEqualCompZero(N0) || !N0.hasOneUse())
17693 std::swap(a&: N0, b&: N1);
17694 if (!IsEqualCompZero(N0) || !N0.hasOneUse())
17695 return SDValue();
17696
17697 KnownBits Known = DAG.computeKnownBits(Op: N1);
17698 if (Known.getMaxValue().ugt(RHS: 1))
17699 return SDValue();
17700
17701 unsigned CzeroOpcode =
17702 (cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get() == ISD::SETNE)
17703 ? RISCVISD::CZERO_EQZ
17704 : RISCVISD::CZERO_NEZ;
17705
17706 EVT VT = N->getValueType(ResNo: 0);
17707 SDLoc DL(N);
17708 return DAG.getNode(Opcode: CzeroOpcode, DL, VT, N1, N2: N0.getOperand(i: 0));
17709}
17710
17711static SDValue reduceANDOfAtomicLoad(SDNode *N,
17712 TargetLowering::DAGCombinerInfo &DCI) {
17713 SelectionDAG &DAG = DCI.DAG;
17714 if (N->getOpcode() != ISD::AND)
17715 return SDValue();
17716
17717 SDValue N0 = N->getOperand(Num: 0);
17718 if (N0.getOpcode() != ISD::ATOMIC_LOAD)
17719 return SDValue();
17720 if (!N0.hasOneUse())
17721 return SDValue();
17722
17723 AtomicSDNode *ALoad = cast<AtomicSDNode>(Val: N0.getNode());
17724 if (isStrongerThanMonotonic(AO: ALoad->getSuccessOrdering()))
17725 return SDValue();
17726
17727 EVT LoadedVT = ALoad->getMemoryVT();
17728 ConstantSDNode *MaskConst = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17729 if (!MaskConst)
17730 return SDValue();
17731 uint64_t Mask = MaskConst->getZExtValue();
17732 uint64_t ExpectedMask = maskTrailingOnes<uint64_t>(N: LoadedVT.getSizeInBits());
17733 if (Mask != ExpectedMask)
17734 return SDValue();
17735
17736 SDValue ZextLoad = DAG.getAtomicLoad(
17737 ExtType: ISD::ZEXTLOAD, dl: SDLoc(N), MemVT: ALoad->getMemoryVT(), VT: N->getValueType(ResNo: 0),
17738 Chain: ALoad->getChain(), Ptr: ALoad->getBasePtr(), MMO: ALoad->getMemOperand());
17739 DCI.CombineTo(N, Res: ZextLoad);
17740 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N0.getNode(), 1), To: ZextLoad.getValue(R: 1));
17741 DCI.recursivelyDeleteUnusedNodes(N: N0.getNode());
17742 return SDValue(N, 0);
17743}
17744
17745// Sometimes a mask is applied after a shift. If that shift was fed by a
17746// load, there is sometimes the opportunity to narrow the load, which is
17747// hidden by the intermediate shift. Detect that case and commute the
17748// shift/and in order to enable load narrowing.
17749static SDValue combineNarrowableShiftedLoad(SDNode *N, SelectionDAG &DAG) {
17750 EVT VT = N->getValueType(ResNo: 0);
17751 if (!VT.isScalarInteger())
17752 return SDValue();
17753
17754 using namespace SDPatternMatch;
17755 SDValue LoadNode;
17756 APInt MaskVal, ShiftVal;
17757 // (and (shl (load ...), ShiftAmt), Mask)
17758 if (!sd_match(
17759 N, P: m_And(L: m_OneUse(P: m_Shl(L: m_Value(N&: LoadNode, P: m_SpecificOpc(Opcode: ISD::LOAD)),
17760 R: m_ConstInt(V&: ShiftVal))),
17761 R: m_ConstInt(V&: MaskVal)))) {
17762 return SDValue();
17763 }
17764
17765 uint64_t ShiftAmt = ShiftVal.getZExtValue();
17766
17767 if (ShiftAmt >= VT.getSizeInBits())
17768 return SDValue();
17769
17770 // Calculate the appropriate mask if it were applied before the shift.
17771 APInt InnerMask = MaskVal.lshr(shiftAmt: ShiftAmt);
17772 bool IsNarrowable =
17773 InnerMask == 0xff || InnerMask == 0xffff || InnerMask == 0xffffffff;
17774
17775 if (!IsNarrowable)
17776 return SDValue();
17777
17778 // AND the loaded value and change the shift appropriately, allowing
17779 // the load to be narrowed.
17780 SDLoc DL(N);
17781 SDValue InnerAnd = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoadNode,
17782 N2: DAG.getConstant(Val: InnerMask, DL, VT));
17783 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: InnerAnd,
17784 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
17785}
17786
17787// Combines two comparison operation and logic operation to one selection
17788// operation(min, max) and logic operation. Returns new constructed Node if
17789// conditions for optimization are satisfied.
17790static SDValue performANDCombine(SDNode *N,
17791 TargetLowering::DAGCombinerInfo &DCI,
17792 const RISCVSubtarget &Subtarget) {
17793 SelectionDAG &DAG = DCI.DAG;
17794 SDValue N0 = N->getOperand(Num: 0);
17795
17796 // Pre-promote (i32 (and (srl X, Y), 1)) on RV64 with Zbs without zero
17797 // extending X. This is safe since we only need the LSB after the shift and
17798 // shift amounts larger than 31 would produce poison. If we wait until
17799 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
17800 // to use a BEXT instruction.
17801 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
17802 N->getValueType(ResNo: 0) == MVT::i32 && isOneConstant(V: N->getOperand(Num: 1)) &&
17803 N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
17804 N0.hasOneUse()) {
17805 SDLoc DL(N);
17806 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
17807 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
17808 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
17809 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Srl,
17810 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i64));
17811 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: And);
17812 }
17813
17814 if (SDValue V = combineNarrowableShiftedLoad(N, DAG))
17815 return V;
17816 if (SDValue V = reverseZExtICmpCombine(N, DAG, Subtarget))
17817 return V;
17818 if (DCI.isAfterLegalizeDAG())
17819 if (SDValue V = combineANDOfSETCCToCZERO(N, DAG, Subtarget))
17820 return V;
17821 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17822 return V;
17823 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17824 return V;
17825 if (SDValue V = reduceANDOfAtomicLoad(N, DCI))
17826 return V;
17827
17828 if (DCI.isAfterLegalizeDAG())
17829 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
17830 return V;
17831
17832 // fold (and (select lhs, rhs, cc, -1, y), x) ->
17833 // (select lhs, rhs, cc, x, (and x, y))
17834 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget);
17835}
17836
17837// Try to pull an xor with 1 through a select idiom that uses czero_eqz/nez.
17838// FIXME: Generalize to other binary operators with same operand.
17839static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1,
17840 SelectionDAG &DAG) {
17841 assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
17842
17843 if (N0.getOpcode() != RISCVISD::CZERO_EQZ ||
17844 N1.getOpcode() != RISCVISD::CZERO_NEZ ||
17845 !N0.hasOneUse() || !N1.hasOneUse())
17846 return SDValue();
17847
17848 // Should have the same condition.
17849 SDValue Cond = N0.getOperand(i: 1);
17850 if (Cond != N1.getOperand(i: 1))
17851 return SDValue();
17852
17853 SDValue TrueV = N0.getOperand(i: 0);
17854 SDValue FalseV = N1.getOperand(i: 0);
17855
17856 if (TrueV.getOpcode() != ISD::XOR || FalseV.getOpcode() != ISD::XOR ||
17857 TrueV.getOperand(i: 1) != FalseV.getOperand(i: 1) ||
17858 !isOneConstant(V: TrueV.getOperand(i: 1)) ||
17859 !TrueV.hasOneUse() || !FalseV.hasOneUse())
17860 return SDValue();
17861
17862 EVT VT = N->getValueType(ResNo: 0);
17863 SDLoc DL(N);
17864
17865 SDValue NewN0 = DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV.getOperand(i: 0),
17866 N2: Cond);
17867 SDValue NewN1 =
17868 DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV.getOperand(i: 0), N2: Cond);
17869 SDValue NewOr =
17870 DAG.getNode(Opcode: ISD::OR, DL, VT, N1: NewN0, N2: NewN1, Flags: SDNodeFlags::Disjoint);
17871 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewOr, N2: TrueV.getOperand(i: 1));
17872}
17873
17874// (xor X, (xor (and X, C2), Y))
17875// ->(qc_insb X, (sra Y, ShAmt), Width, ShAmt)
17876// where C2 is a shifted mask with width = Width and shift = ShAmt
17877// qc_insb might become qc.insb or qc.insbi depending on the operands.
17878static SDValue combineXorToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17879 const RISCVSubtarget &Subtarget) {
17880 if (!Subtarget.hasVendorXqcibm())
17881 return SDValue();
17882
17883 using namespace SDPatternMatch;
17884 SDValue Base, Inserted;
17885 APInt CMask;
17886 if (!sd_match(N, P: m_Xor(L: m_Value(N&: Base),
17887 R: m_OneUse(P: m_Xor(L: m_OneUse(P: m_And(L: m_Deferred(V&: Base),
17888 R: m_ConstInt(V&: CMask))),
17889 R: m_Value(N&: Inserted))))))
17890 return SDValue();
17891
17892 if (N->getValueType(ResNo: 0) != MVT::i32)
17893 return SDValue();
17894 unsigned Width, ShAmt;
17895 if (!CMask.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width))
17896 return SDValue();
17897
17898 // Check if all zero bits in CMask are also zero in Inserted
17899 if (!DAG.MaskedValueIsZero(Op: Inserted, Mask: ~CMask))
17900 return SDValue();
17901
17902 SDLoc DL(N);
17903
17904 // `Inserted` needs to be right shifted before it is put into the
17905 // instruction.
17906 Inserted = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i32, N1: Inserted,
17907 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
17908
17909 SDValue Ops[] = {Base, Inserted, DAG.getConstant(Val: Width, DL, VT: MVT::i32),
17910 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
17911 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
17912}
17913
17914static SDValue combineOrToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17915 const RISCVSubtarget &Subtarget) {
17916 if (!Subtarget.hasVendorXqcibm())
17917 return SDValue();
17918
17919 using namespace SDPatternMatch;
17920
17921 SDValue X;
17922 APInt MaskImm;
17923 if (!sd_match(N, P: m_Or(L: m_OneUse(P: m_Value(N&: X)), R: m_ConstInt(V&: MaskImm))))
17924 return SDValue();
17925
17926 unsigned ShAmt, Width;
17927 if (!MaskImm.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width) || MaskImm.isSignedIntN(N: 12))
17928 return SDValue();
17929
17930 if (N->getValueType(ResNo: 0) != MVT::i32)
17931 return SDValue();
17932
17933 // If Zbs is enabled and it is a single bit set we can use BSETI which
17934 // can be compressed to C_BSETI when Xqcibm in enabled.
17935 if (Width == 1 && Subtarget.hasStdExtZbs())
17936 return SDValue();
17937
17938 // If C1 is a shifted mask (but can't be formed as an ORI),
17939 // use a bitfield insert of -1.
17940 // Transform (or x, C1)
17941 // -> (qc.insbi x, -1, width, shift)
17942 SDLoc DL(N);
17943
17944 SDValue Ops[] = {X, DAG.getSignedConstant(Val: -1, DL, VT: MVT::i32),
17945 DAG.getConstant(Val: Width, DL, VT: MVT::i32),
17946 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
17947 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
17948}
17949
17950// Generate a QC_INSB/QC_INSBI from 'or (and X, MaskImm), OrImm' iff the value
17951// being inserted only sets known zero bits.
17952static SDValue combineOrAndToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17953 const RISCVSubtarget &Subtarget) {
17954 // Supported only in Xqcibm for now.
17955 if (!Subtarget.hasVendorXqcibm())
17956 return SDValue();
17957
17958 using namespace SDPatternMatch;
17959
17960 SDValue Inserted;
17961 APInt MaskImm, OrImm;
17962 if (!sd_match(
17963 N, P: m_SpecificVT(RefVT: MVT::i32, P: m_Or(L: m_OneUse(P: m_And(L: m_Value(N&: Inserted),
17964 R: m_ConstInt(V&: MaskImm))),
17965 R: m_ConstInt(V&: OrImm)))))
17966 return SDValue();
17967
17968 // Compute the Known Zero for the AND as this allows us to catch more general
17969 // cases than just looking for AND with imm.
17970 KnownBits Known = DAG.computeKnownBits(Op: N->getOperand(Num: 0));
17971
17972 // The bits being inserted must only set those bits that are known to be
17973 // zero.
17974 if (!OrImm.isSubsetOf(RHS: Known.Zero)) {
17975 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
17976 // currently handle this case.
17977 return SDValue();
17978 }
17979
17980 unsigned ShAmt, Width;
17981 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
17982 if (!Known.Zero.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width))
17983 return SDValue();
17984
17985 // QC_INSB(I) dst, src, #width, #shamt.
17986 SDLoc DL(N);
17987
17988 SDValue ImmNode =
17989 DAG.getSignedConstant(Val: OrImm.getSExtValue() >> ShAmt, DL, VT: MVT::i32);
17990
17991 SDValue Ops[] = {Inserted, ImmNode, DAG.getConstant(Val: Width, DL, VT: MVT::i32),
17992 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
17993 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
17994}
17995
17996static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
17997 const RISCVSubtarget &Subtarget) {
17998 SelectionDAG &DAG = DCI.DAG;
17999
18000 if (SDValue V = combineOrAndToBitfieldInsert(N, DAG, Subtarget))
18001 return V;
18002 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
18003 return V;
18004 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
18005 return V;
18006
18007 if (DCI.isAfterLegalizeDAG()) {
18008 if (SDValue V = combineOrToBitfieldInsert(N, DAG, Subtarget))
18009 return V;
18010 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
18011 return V;
18012 }
18013
18014 // Look for Or of CZERO_EQZ/NEZ with same condition which is the select idiom.
18015 // We may be able to pull a common operation out of the true and false value.
18016 SDValue N0 = N->getOperand(Num: 0);
18017 SDValue N1 = N->getOperand(Num: 1);
18018 if (SDValue V = combineOrOfCZERO(N, N0, N1, DAG))
18019 return V;
18020 if (SDValue V = combineOrOfCZERO(N, N0: N1, N1: N0, DAG))
18021 return V;
18022
18023 // fold (or (select cond, 0, y), x) ->
18024 // (select cond, x, (or x, y))
18025 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
18026}
18027
18028static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
18029 const RISCVSubtarget &Subtarget) {
18030 SDValue N0 = N->getOperand(Num: 0);
18031 SDValue N1 = N->getOperand(Num: 1);
18032
18033 // Pre-promote (i32 (xor (shl -1, X), ~0)) on RV64 with Zbs so we can use
18034 // (ADDI (BSET X0, X), -1). If we wait until type legalization, we'll create
18035 // RISCVISD:::SLLW and we can't recover it to use a BSET instruction.
18036 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
18037 N->getValueType(ResNo: 0) == MVT::i32 && isAllOnesConstant(V: N1) &&
18038 N0.getOpcode() == ISD::SHL && isAllOnesConstant(V: N0.getOperand(i: 0)) &&
18039 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) && N0.hasOneUse()) {
18040 SDLoc DL(N);
18041 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
18042 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
18043 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
18044 SDValue Not = DAG.getNOT(DL, Val: Shl, VT: MVT::i64);
18045 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Not);
18046 }
18047
18048 // fold (xor (sllw 1, x), -1) -> (rolw ~1, x)
18049 // NOTE: Assumes ROL being legal means ROLW is legal.
18050 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18051 if (N0.getOpcode() == RISCVISD::SLLW &&
18052 isAllOnesConstant(V: N1) && isOneConstant(V: N0.getOperand(i: 0)) &&
18053 TLI.isOperationLegal(Op: ISD::ROTL, VT: MVT::i64)) {
18054 SDLoc DL(N);
18055 return DAG.getNode(Opcode: RISCVISD::ROLW, DL, VT: MVT::i64,
18056 N1: DAG.getConstant(Val: ~1, DL, VT: MVT::i64), N2: N0.getOperand(i: 1));
18057 }
18058
18059 // Fold (xor (setcc constant, y, setlt), 1) -> (setcc y, constant + 1, setlt)
18060 if (N0.getOpcode() == ISD::SETCC && isOneConstant(V: N1) && N0.hasOneUse()) {
18061 auto *ConstN00 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
18062 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
18063 if (ConstN00 && CC == ISD::SETLT) {
18064 EVT VT = N0.getValueType();
18065 SDLoc DL(N0);
18066 const APInt &Imm = ConstN00->getAPIntValue();
18067 if ((Imm + 1).isSignedIntN(N: 12))
18068 return DAG.getSetCC(DL, VT, LHS: N0.getOperand(i: 1),
18069 RHS: DAG.getConstant(Val: Imm + 1, DL, VT), Cond: CC);
18070 }
18071 }
18072
18073 if (SDValue V = combineXorToBitfieldInsert(N, DAG, Subtarget))
18074 return V;
18075
18076 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
18077 return V;
18078 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
18079 return V;
18080
18081 // fold (xor (select cond, 0, y), x) ->
18082 // (select cond, x, (xor x, y))
18083 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
18084}
18085
18086// Try to expand a multiply to a sequence of shifts and add/subs,
18087// for a machine without native mul instruction.
18088static SDValue expandMulToNAFSequence(SDNode *N, SelectionDAG &DAG,
18089 uint64_t MulAmt) {
18090 SDLoc DL(N);
18091 EVT VT = N->getValueType(ResNo: 0);
18092 const uint64_t BitWidth = VT.getFixedSizeInBits();
18093
18094 SDValue Result = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
18095 SDValue N0 = N->getOperand(Num: 0);
18096
18097 // Find the Non-adjacent form of the multiplier.
18098 for (uint64_t E = MulAmt, I = 0; E && I < BitWidth; ++I, E >>= 1) {
18099 if (E & 1) {
18100 bool IsAdd = (E & 3) == 1;
18101 E -= IsAdd ? 1 : -1;
18102 SDValue ShiftVal = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0,
18103 N2: DAG.getShiftAmountConstant(Val: I, VT, DL));
18104 ISD::NodeType AddSubOp = IsAdd ? ISD::ADD : ISD::SUB;
18105 Result = DAG.getNode(Opcode: AddSubOp, DL, VT, N1: Result, N2: ShiftVal);
18106 }
18107 }
18108
18109 return Result;
18110}
18111
18112// X * (2^N +/- 2^M) -> (add/sub (shl X, C1), (shl X, C2))
18113static SDValue expandMulToAddOrSubOfShl(SDNode *N, SelectionDAG &DAG,
18114 uint64_t MulAmt) {
18115 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
18116 SDValue X = N->getOperand(Num: 0);
18117 ISD::NodeType Op;
18118 uint64_t ShiftAmt1;
18119 bool CanSub = isPowerOf2_64(Value: MulAmt + MulAmtLowBit);
18120 auto PreferSub = [X, MulAmtLowBit]() {
18121 // For MulAmt == 3 << M both (X << M + 2) - (X << M)
18122 // and (X << M + 1) + (X << M) are valid expansions.
18123 // Prefer SUB if we can get (X << M + 2) for free,
18124 // because X is exact (Y >> M + 2).
18125 uint64_t ShAmt = Log2_64(Value: MulAmtLowBit) + 2;
18126 using namespace SDPatternMatch;
18127 return sd_match(N: X, P: m_ExactSr(L: m_Value(), R: m_SpecificInt(V: ShAmt)));
18128 };
18129 if (isPowerOf2_64(Value: MulAmt - MulAmtLowBit) && !(CanSub && PreferSub())) {
18130 Op = ISD::ADD;
18131 ShiftAmt1 = MulAmt - MulAmtLowBit;
18132 } else if (CanSub) {
18133 Op = ISD::SUB;
18134 ShiftAmt1 = MulAmt + MulAmtLowBit;
18135 } else {
18136 return SDValue();
18137 }
18138 EVT VT = N->getValueType(ResNo: 0);
18139 SDLoc DL(N);
18140 SDValue Shift1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X,
18141 N2: DAG.getConstant(Val: Log2_64(Value: ShiftAmt1), DL, VT));
18142 SDValue Shift2 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X,
18143 N2: DAG.getConstant(Val: Log2_64(Value: MulAmtLowBit), DL, VT));
18144 return DAG.getNode(Opcode: Op, DL, VT, N1: Shift1, N2: Shift2);
18145}
18146
18147static SDValue getShlAddShlAdd(SDNode *N, SelectionDAG &DAG, unsigned ShX,
18148 unsigned ShY, bool AddX, unsigned Shift) {
18149 SDLoc DL(N);
18150 EVT VT = N->getValueType(ResNo: 0);
18151 SDValue X = N->getOperand(Num: 0);
18152 // Put the shift first if we can fold:
18153 // a. a zext into the shift forming a slli.uw
18154 // b. an exact shift right forming one shorter shift or no shift at all
18155 using namespace SDPatternMatch;
18156 if (Shift != 0 &&
18157 sd_match(N: X, P: m_AnyOf(preds: m_And(L: m_Value(), R: m_SpecificInt(UINT64_C(0xffffffff))),
18158 preds: m_ExactSr(L: m_Value(), R: m_ConstInt())))) {
18159 X = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: Shift, DL, VT));
18160 Shift = 0;
18161 }
18162 SDValue ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
18163 N2: DAG.getTargetConstant(Val: ShY, DL, VT), N3: X);
18164 if (ShX != 0)
18165 ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: ShlAdd,
18166 N2: DAG.getTargetConstant(Val: ShX, DL, VT), N3: AddX ? X : ShlAdd);
18167 if (Shift == 0)
18168 return ShlAdd;
18169 // Otherwise, put the shl last so that it can fold with following instructions
18170 // (e.g. sext or add).
18171 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShlAdd, N2: DAG.getConstant(Val: Shift, DL, VT));
18172}
18173
18174static SDValue expandMulToShlAddShlAdd(SDNode *N, SelectionDAG &DAG,
18175 uint64_t MulAmt, unsigned Shift) {
18176 switch (MulAmt) {
18177 // 3/5/9 -> (shYadd X, X)
18178 case 3:
18179 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 1, /*AddX=*/false, Shift);
18180 case 5:
18181 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 2, /*AddX=*/false, Shift);
18182 case 9:
18183 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 3, /*AddX=*/false, Shift);
18184 // 3/5/9 * 3/5/9 -> (shXadd (shYadd X, X), (shYadd X, X))
18185 case 5 * 3:
18186 return getShlAddShlAdd(N, DAG, ShX: 2, ShY: 1, /*AddX=*/false, Shift);
18187 case 9 * 3:
18188 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 1, /*AddX=*/false, Shift);
18189 case 5 * 5:
18190 return getShlAddShlAdd(N, DAG, ShX: 2, ShY: 2, /*AddX=*/false, Shift);
18191 case 9 * 5:
18192 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 2, /*AddX=*/false, Shift);
18193 case 9 * 9:
18194 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 3, /*AddX=*/false, Shift);
18195 default:
18196 break;
18197 }
18198
18199 int ShX;
18200 if (int ShY = isShifted359(Value: MulAmt - 1, Shift&: ShX)) {
18201 assert(ShX != 0 && "MulAmt=4,6,10 handled before");
18202 // 2/4/8 * 3/5/9 + 1 -> (shXadd (shYadd X, X), X)
18203 if (ShX <= 3)
18204 return getShlAddShlAdd(N, DAG, ShX, ShY, /*AddX=*/true, Shift);
18205 // 2^N * 3/5/9 + 1 -> (add (shYadd (shl X, N), (shl X, N)), X)
18206 if (Shift == 0) {
18207 SDLoc DL(N);
18208 EVT VT = N->getValueType(ResNo: 0);
18209 SDValue X = N->getOperand(Num: 0);
18210 SDValue Shl =
18211 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShX, DL, VT));
18212 SDValue ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: Shl,
18213 N2: DAG.getTargetConstant(Val: ShY, DL, VT), N3: Shl);
18214 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: ShlAdd, N2: X);
18215 }
18216 }
18217 return SDValue();
18218}
18219
18220// Try to expand a scalar multiply to a faster sequence.
18221static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
18222 TargetLowering::DAGCombinerInfo &DCI,
18223 const RISCVSubtarget &Subtarget) {
18224
18225 EVT VT = N->getValueType(ResNo: 0);
18226
18227 // LI + MUL is usually smaller than the alternative sequence.
18228 if (DAG.getMachineFunction().getFunction().hasMinSize())
18229 return SDValue();
18230
18231 if (VT != Subtarget.getXLenVT())
18232 return SDValue();
18233
18234 bool ShouldExpandMul =
18235 (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) ||
18236 !Subtarget.hasStdExtZmmul();
18237 if (!ShouldExpandMul)
18238 return SDValue();
18239
18240 ConstantSDNode *CNode = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
18241 if (!CNode)
18242 return SDValue();
18243 uint64_t MulAmt = CNode->getZExtValue();
18244
18245 // Don't do this if the Xqciac extension is enabled and the MulAmt in simm12.
18246 if (Subtarget.hasVendorXqciac() && isInt<12>(x: CNode->getSExtValue()))
18247 return SDValue();
18248
18249 // WARNING: The code below is knowingly incorrect with regards to undef
18250 // semantics. We're adding additional uses of X here, and in principle, we
18251 // should be freezing X before doing so. However, adding freeze here causes
18252 // real regressions, and no other target properly freezes X in these cases
18253 // either.
18254 if (Subtarget.hasShlAdd(ShAmt: 3)) {
18255 // 3/5/9 * 2^N -> (shl (shXadd X, X), N)
18256 // 3/5/9 * 3/5/9 * 2^N - In particular, this covers multiples
18257 // of 25 which happen to be quite common.
18258 // (2/4/8 * 3/5/9 + 1) * 2^N
18259 unsigned Shift = llvm::countr_zero(Val: MulAmt);
18260 if (SDValue V = expandMulToShlAddShlAdd(N, DAG, MulAmt: MulAmt >> Shift, Shift))
18261 return V;
18262
18263 // If this is a power 2 + 2/4/8, we can use a shift followed by a single
18264 // shXadd. First check if this a sum of two power of 2s because that's
18265 // easy. Then count how many zeros are up to the first bit.
18266 SDValue X = N->getOperand(Num: 0);
18267 if (Shift >= 1 && Shift <= 3 && isPowerOf2_64(Value: MulAmt & (MulAmt - 1))) {
18268 unsigned ShiftAmt = llvm::countr_zero(Val: (MulAmt & (MulAmt - 1)));
18269 SDLoc DL(N);
18270 SDValue Shift1 =
18271 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShiftAmt, DL, VT));
18272 return DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
18273 N2: DAG.getTargetConstant(Val: Shift, DL, VT), N3: Shift1);
18274 }
18275
18276 // TODO: 2^(C1>3) * 3/5/9 - 1
18277
18278 // 2^n + 2/4/8 + 1 -> (add (shl X, C1), (shXadd X, X))
18279 if (MulAmt > 2 && isPowerOf2_64(Value: (MulAmt - 1) & (MulAmt - 2))) {
18280 unsigned ScaleShift = llvm::countr_zero(Val: MulAmt - 1);
18281 if (ScaleShift >= 1 && ScaleShift < 4) {
18282 unsigned ShiftAmt = llvm::countr_zero(Val: (MulAmt - 1) & (MulAmt - 2));
18283 SDLoc DL(N);
18284 SDValue Shift1 =
18285 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShiftAmt, DL, VT));
18286 return DAG.getNode(
18287 Opcode: ISD::ADD, DL, VT, N1: Shift1,
18288 N2: DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
18289 N2: DAG.getTargetConstant(Val: ScaleShift, DL, VT), N3: X));
18290 }
18291 }
18292
18293 // 2^N - 3/5/9 --> (sub (shl X, C1), (shXadd X, x))
18294 for (uint64_t Offset : {3, 5, 9}) {
18295 if (isPowerOf2_64(Value: MulAmt + Offset)) {
18296 unsigned ShAmt = llvm::countr_zero(Val: MulAmt + Offset);
18297 if (ShAmt >= VT.getSizeInBits())
18298 continue;
18299 SDLoc DL(N);
18300 SDValue Shift1 =
18301 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShAmt, DL, VT));
18302 SDValue Mul359 =
18303 DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
18304 N2: DAG.getTargetConstant(Val: Log2_64(Value: Offset - 1), DL, VT), N3: X);
18305 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Shift1, N2: Mul359);
18306 }
18307 }
18308 }
18309
18310 if (SDValue V = expandMulToAddOrSubOfShl(N, DAG, MulAmt))
18311 return V;
18312
18313 if (!Subtarget.hasStdExtZmmul())
18314 return expandMulToNAFSequence(N, DAG, MulAmt);
18315
18316 return SDValue();
18317}
18318
18319// Combine vXi32 (mul (and (lshr X, 15), 0x10001), 0xffff) ->
18320// (bitcast (sra (v2Xi16 (bitcast X)), 15))
18321// Same for other equivalent types with other equivalent constants.
18322static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG) {
18323 EVT VT = N->getValueType(ResNo: 0);
18324 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
18325
18326 // Do this for legal vectors unless they are i1 or i8 vectors.
18327 if (!VT.isVector() || !TLI.isTypeLegal(VT) || VT.getScalarSizeInBits() < 16)
18328 return SDValue();
18329
18330 if (N->getOperand(Num: 0).getOpcode() != ISD::AND ||
18331 N->getOperand(Num: 0).getOperand(i: 0).getOpcode() != ISD::SRL)
18332 return SDValue();
18333
18334 SDValue And = N->getOperand(Num: 0);
18335 SDValue Srl = And.getOperand(i: 0);
18336
18337 APInt V1, V2, V3;
18338 if (!ISD::isConstantSplatVector(N: N->getOperand(Num: 1).getNode(), SplatValue&: V1) ||
18339 !ISD::isConstantSplatVector(N: And.getOperand(i: 1).getNode(), SplatValue&: V2) ||
18340 !ISD::isConstantSplatVector(N: Srl.getOperand(i: 1).getNode(), SplatValue&: V3))
18341 return SDValue();
18342
18343 unsigned HalfSize = VT.getScalarSizeInBits() / 2;
18344 if (!V1.isMask(numBits: HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
18345 V3 != (HalfSize - 1))
18346 return SDValue();
18347
18348 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(),
18349 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HalfSize),
18350 EC: VT.getVectorElementCount() * 2);
18351 SDLoc DL(N);
18352 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Srl.getOperand(i: 0));
18353 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT: HalfVT, N1: Cast,
18354 N2: DAG.getConstant(Val: HalfSize - 1, DL, VT: HalfVT));
18355 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Sra);
18356}
18357
18358static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
18359 TargetLowering::DAGCombinerInfo &DCI,
18360 const RISCVSubtarget &Subtarget) {
18361 EVT VT = N->getValueType(ResNo: 0);
18362 if (!VT.isVector())
18363 return expandMul(N, DAG, DCI, Subtarget);
18364
18365 SDLoc DL(N);
18366 SDValue N0 = N->getOperand(Num: 0);
18367 SDValue N1 = N->getOperand(Num: 1);
18368 SDValue MulOper;
18369 unsigned AddSubOpc;
18370
18371 // vmadd: (mul (add x, 1), y) -> (add (mul x, y), y)
18372 // (mul x, add (y, 1)) -> (add x, (mul x, y))
18373 // vnmsub: (mul (sub 1, x), y) -> (sub y, (mul x, y))
18374 // (mul x, (sub 1, y)) -> (sub x, (mul x, y))
18375 auto IsAddSubWith1 = [&](SDValue V) -> bool {
18376 AddSubOpc = V->getOpcode();
18377 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) {
18378 SDValue Opnd = V->getOperand(Num: 1);
18379 MulOper = V->getOperand(Num: 0);
18380 if (AddSubOpc == ISD::SUB)
18381 std::swap(a&: Opnd, b&: MulOper);
18382 if (isOneOrOneSplat(V: Opnd))
18383 return true;
18384 }
18385 return false;
18386 };
18387
18388 if (IsAddSubWith1(N0)) {
18389 SDValue MulVal = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1, N2: MulOper);
18390 return DAG.getNode(Opcode: AddSubOpc, DL, VT, N1, N2: MulVal);
18391 }
18392
18393 if (IsAddSubWith1(N1)) {
18394 SDValue MulVal = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N0, N2: MulOper);
18395 return DAG.getNode(Opcode: AddSubOpc, DL, VT, N1: N0, N2: MulVal);
18396 }
18397
18398 if (SDValue V = combineBinOpOfZExt(N, DAG))
18399 return V;
18400
18401 if (SDValue V = combineVectorMulToSraBitcast(N, DAG))
18402 return V;
18403
18404 return SDValue();
18405}
18406
18407/// According to the property that indexed load/store instructions zero-extend
18408/// their indices, try to narrow the type of index operand.
18409static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) {
18410 if (isIndexTypeSigned(IndexType))
18411 return false;
18412
18413 if (!N->hasOneUse())
18414 return false;
18415
18416 EVT VT = N.getValueType();
18417 SDLoc DL(N);
18418
18419 // In general, what we're doing here is seeing if we can sink a truncate to
18420 // a smaller element type into the expression tree building our index.
18421 // TODO: We can generalize this and handle a bunch more cases if useful.
18422
18423 // Narrow a buildvector to the narrowest element type. This requires less
18424 // work and less register pressure at high LMUL, and creates smaller constants
18425 // which may be cheaper to materialize.
18426 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode())) {
18427 KnownBits Known = DAG.computeKnownBits(Op: N);
18428 unsigned ActiveBits = std::max(a: 8u, b: Known.countMaxActiveBits());
18429 LLVMContext &C = *DAG.getContext();
18430 EVT ResultVT = EVT::getIntegerVT(Context&: C, BitWidth: ActiveBits).getRoundIntegerType(Context&: C);
18431 if (ResultVT.bitsLT(VT: VT.getVectorElementType())) {
18432 N = DAG.getNode(Opcode: ISD::TRUNCATE, DL,
18433 VT: VT.changeVectorElementType(Context&: C, EltVT: ResultVT), Operand: N);
18434 return true;
18435 }
18436 }
18437
18438 // Handle the pattern (shl (zext x to ty), C) and bits(x) + C < bits(ty).
18439 if (N.getOpcode() != ISD::SHL)
18440 return false;
18441
18442 SDValue N0 = N.getOperand(i: 0);
18443 if (N0.getOpcode() != ISD::ZERO_EXTEND &&
18444 N0.getOpcode() != RISCVISD::VZEXT_VL)
18445 return false;
18446 if (!N0->hasOneUse())
18447 return false;
18448
18449 APInt ShAmt;
18450 SDValue N1 = N.getOperand(i: 1);
18451 if (!ISD::isConstantSplatVector(N: N1.getNode(), SplatValue&: ShAmt))
18452 return false;
18453
18454 SDValue Src = N0.getOperand(i: 0);
18455 EVT SrcVT = Src.getValueType();
18456 unsigned SrcElen = SrcVT.getScalarSizeInBits();
18457
18458 // Consider any leading zeros in the source.
18459 SrcElen -= DAG.computeKnownBits(Op: Src).countMinLeadingZeros();
18460
18461 unsigned ShAmtV = ShAmt.getZExtValue();
18462 unsigned NewElen = PowerOf2Ceil(A: SrcElen + ShAmtV);
18463 NewElen = std::max(a: NewElen, b: 8U);
18464 // Make sure the new elen is at least as large as the original elen.
18465 NewElen = std::max<unsigned>(a: NewElen, b: SrcVT.getScalarSizeInBits());
18466
18467 // Skip if NewElen is not narrower than the original extended type.
18468 if (NewElen >= N0.getValueType().getScalarSizeInBits())
18469 return false;
18470
18471 EVT NewEltVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewElen);
18472 EVT NewVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: NewEltVT);
18473
18474 SDValue NewExt = DAG.getNode(Opcode: N0->getOpcode(), DL, VT: NewVT, Ops: N0->ops());
18475 SDValue NewShAmtVec = DAG.getConstant(Val: ShAmtV, DL, VT: NewVT);
18476 N = DAG.getNode(Opcode: ISD::SHL, DL, VT: NewVT, N1: NewExt, N2: NewShAmtVec);
18477 return true;
18478}
18479
18480/// Try to map an integer comparison with size > XLEN to vector instructions
18481/// before type legalization splits it up into chunks.
18482static SDValue
18483combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
18484 const SDLoc &DL, SelectionDAG &DAG,
18485 const RISCVSubtarget &Subtarget) {
18486 assert(ISD::isIntEqualitySetCC(CC) && "Bad comparison predicate");
18487
18488 if (!Subtarget.useRVVForFixedLengthVectors())
18489 return SDValue();
18490
18491 MVT XLenVT = Subtarget.getXLenVT();
18492 EVT OpVT = X.getValueType();
18493 // We're looking for an oversized integer equality comparison.
18494 if (!OpVT.isScalarInteger())
18495 return SDValue();
18496
18497 unsigned OpSize = OpVT.getSizeInBits();
18498 // The size should be larger than XLen and smaller than the maximum vector
18499 // size.
18500 if (OpSize <= Subtarget.getXLen() ||
18501 OpSize > Subtarget.getRealMinVLen() *
18502 Subtarget.getMaxLMULForFixedLengthVectors())
18503 return SDValue();
18504
18505 // Don't perform this combine if constructing the vector will be expensive.
18506 auto IsVectorBitCastCheap = [](SDValue X) {
18507 X = peekThroughBitcasts(V: X);
18508 return isa<ConstantSDNode>(Val: X) || X.getValueType().isVector() ||
18509 X.getOpcode() == ISD::LOAD;
18510 };
18511 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
18512 return SDValue();
18513
18514 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
18515 Kind: Attribute::NoImplicitFloat))
18516 return SDValue();
18517
18518 // Bail out for non-byte-sized types.
18519 if (!OpVT.isByteSized())
18520 return SDValue();
18521
18522 // Find a preferred vector element type by inspecting how the value is used.
18523 auto GetPreferredEltVT = [](SDValue V) -> MVT {
18524 // Look backwards: check if V itself is derived from a vector
18525 SDValue Peek = peekThroughBitcasts(V);
18526 EVT PeekVT = Peek.getValueType();
18527
18528 if (PeekVT.isVector() && PeekVT.isInteger()) {
18529 EVT EltVT = PeekVT.getVectorElementType();
18530 if (EltVT.isSimple())
18531 return EltVT.getSimpleVT();
18532 }
18533
18534 // Look forwards: check if V is bitcasted to a vector elsewhere in the DAG
18535 for (SDUse &Use : V->uses()) {
18536 // Ensure we are checking the use of the specific value result, not the
18537 // node's chain
18538 if (Use.getResNo() != V.getResNo())
18539 continue;
18540
18541 SDNode *User = Use.getUser();
18542 if (User->getOpcode() == ISD::BITCAST) {
18543 EVT CastVT = User->getValueType(ResNo: 0);
18544 if (CastVT.isVector() && CastVT.isInteger()) {
18545 EVT EltVT = CastVT.getVectorElementType();
18546 if (EltVT.isSimple())
18547 return EltVT.getSimpleVT();
18548 }
18549 }
18550 }
18551 return MVT::INVALID_SIMPLE_VALUE_TYPE;
18552 };
18553
18554 auto IsValidEltVT = [&](MVT VT) {
18555 if (VT == MVT::INVALID_SIMPLE_VALUE_TYPE || !VT.isInteger())
18556 return false;
18557
18558 // Make sure we don't try to create an impossible vector type where the
18559 // elements don't perfectly fill up the OpSize boundary.
18560 unsigned EltSize = VT.getSizeInBits();
18561 if (OpSize % EltSize != 0)
18562 return false;
18563
18564 // Construct the proposed vector type to check its legality
18565 unsigned NumElts = OpSize / EltSize;
18566 EVT TestVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: NumElts);
18567 return Subtarget.getTargetLowering()->isTypeLegal(VT: TestVecVT);
18568 };
18569
18570 // Get preferred VT from either X or Y.
18571 MVT EltVT = GetPreferredEltVT(X);
18572 if (!IsValidEltVT(EltVT))
18573 EltVT = GetPreferredEltVT(Y);
18574
18575 // If both are unsuitable, use the safe default (i8)
18576 if (!IsValidEltVT(EltVT))
18577 EltVT = MVT::i8;
18578
18579 unsigned EltSize = EltVT.getSizeInBits();
18580 unsigned NumElts = OpSize / EltSize;
18581 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: EltVT, NumElements: NumElts);
18582 EVT CmpVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: NumElts);
18583
18584 SDValue VecX = DAG.getBitcast(VT: VecVT, V: X);
18585 SDValue VecY;
18586 // Constant fold the common case of comparing with zero. Later optimizations
18587 // might not do this for us.
18588 if (isNullConstant(V: Y))
18589 VecY = DAG.getConstant(Val: 0, DL, VT: VecVT);
18590 else
18591 VecY = DAG.getBitcast(VT: VecVT, V: Y);
18592
18593 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: VecX, RHS: VecY, Cond: ISD::SETNE);
18594 return DAG.getSetCC(DL, VT, LHS: DAG.getNode(Opcode: ISD::VECREDUCE_OR, DL, VT: XLenVT, Operand: Cmp),
18595 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: CC);
18596}
18597
18598static SDValue performSETCCCombine(SDNode *N,
18599 TargetLowering::DAGCombinerInfo &DCI,
18600 const RISCVSubtarget &Subtarget) {
18601 SelectionDAG &DAG = DCI.DAG;
18602 SDLoc dl(N);
18603 SDValue N0 = N->getOperand(Num: 0);
18604 SDValue N1 = N->getOperand(Num: 1);
18605 EVT VT = N->getValueType(ResNo: 0);
18606 EVT OpVT = N0.getValueType();
18607
18608 ISD::CondCode Cond = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
18609 // Looking for an equality compare.
18610 if (!isIntEqualitySetCC(Code: Cond))
18611 return SDValue();
18612
18613 if (SDValue V =
18614 combineVectorSizedSetCCEquality(VT, X: N0, Y: N1, CC: Cond, DL: dl, DAG, Subtarget))
18615 return V;
18616
18617 if (DCI.isAfterLegalizeDAG() && isa<ConstantSDNode>(Val: N1) &&
18618 N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
18619 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
18620 const APInt &AndRHSC = N0.getConstantOperandAPInt(i: 1);
18621 // (X & -(1 << C)) == 0 -> (X >> C) == 0 if the AND constant can't use ANDI.
18622 if (isNullConstant(V: N1) && !isInt<12>(x: AndRHSC.getSExtValue()) &&
18623 AndRHSC.isNegatedPowerOf2()) {
18624 unsigned ShiftBits = AndRHSC.countr_zero();
18625 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: N0.getOperand(i: 0),
18626 N2: DAG.getConstant(Val: ShiftBits, DL: dl, VT: OpVT));
18627 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: N1, Cond);
18628 }
18629
18630 // Similar to above but handling the lower 32 bits by using sraiw. Allow
18631 // comparing with constants other than 0 if the constant can be folded into
18632 // addi or xori after shifting.
18633 uint64_t N1Int = cast<ConstantSDNode>(Val&: N1)->getZExtValue();
18634 uint64_t AndRHSInt = AndRHSC.getZExtValue();
18635 if (OpVT == MVT::i64 && isUInt<32>(x: AndRHSInt) &&
18636 isPowerOf2_32(Value: -uint32_t(AndRHSInt)) && (N1Int & AndRHSInt) == N1Int) {
18637 unsigned ShiftBits = llvm::countr_zero(Val: AndRHSInt);
18638 int64_t NewC = SignExtend64<32>(x: N1Int) >> ShiftBits;
18639 if (NewC >= -2048 && NewC <= 2048) {
18640 SDValue SExt =
18641 DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OpVT, N1: N0.getOperand(i: 0),
18642 N2: DAG.getValueType(MVT::i32));
18643 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: OpVT, N1: SExt,
18644 N2: DAG.getConstant(Val: ShiftBits, DL: dl, VT: OpVT));
18645 return DAG.getSetCC(DL: dl, VT, LHS: Shift,
18646 RHS: DAG.getSignedConstant(Val: NewC, DL: dl, VT: OpVT), Cond);
18647 }
18648 }
18649
18650 // Fold (and X, Mask) ==/!= C -> X ==/!= sext(C, countr_one(Mask)) if the
18651 // Mask is only clearing redundant sign bits.
18652 if (isMask_64(Value: AndRHSInt)) {
18653 unsigned TrailingOnes = llvm::countr_one(Value: AndRHSInt);
18654 unsigned N1Width = llvm::bit_width(Value: N1Int);
18655 int64_t N1SExt = SignExtend64(X: N1Int, B: TrailingOnes);
18656 if (N1Width <= TrailingOnes && isInt<12>(x: N1SExt) &&
18657 DAG.ComputeMaxSignificantBits(Op: N0.getOperand(i: 0)) <= TrailingOnes)
18658 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
18659 RHS: DAG.getSignedConstant(Val: N1SExt, DL: dl, VT: OpVT), Cond);
18660 }
18661 }
18662
18663 // Replace (seteq (i64 (and X, 0xffffffff)), C1) with
18664 // (seteq (i64 (sext_inreg (X, i32)), C1')) where C1' is C1 sign extended from
18665 // bit 31. Same for setne. C1' may be cheaper to materialize and the
18666 // sext_inreg can become a sext.w instead of a shift pair.
18667 if (OpVT != MVT::i64 || !Subtarget.is64Bit())
18668 return SDValue();
18669
18670 // RHS needs to be a constant.
18671 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
18672 if (!N1C)
18673 return SDValue();
18674
18675 // LHS needs to be (and X, 0xffffffff).
18676 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
18677 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) ||
18678 N0.getConstantOperandVal(i: 1) != UINT64_C(0xffffffff))
18679 return SDValue();
18680
18681 // Don't do this if the sign bit is provably zero, it will be turned back into
18682 // an AND.
18683 APInt SignMask = APInt::getOneBitSet(numBits: 64, BitNo: 31);
18684 if (DAG.MaskedValueIsZero(Op: N0.getOperand(i: 0), Mask: SignMask))
18685 return SDValue();
18686
18687 const APInt &C1 = N1C->getAPIntValue();
18688
18689 // If the constant is larger than 2^32 - 1 it is impossible for both sides
18690 // to be equal.
18691 if (C1.getActiveBits() > 32)
18692 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
18693
18694 SDValue SExtOp = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: N, VT: OpVT,
18695 N1: N0.getOperand(i: 0), N2: DAG.getValueType(MVT::i32));
18696 return DAG.getSetCC(DL: dl, VT, LHS: SExtOp, RHS: DAG.getConstant(Val: C1.trunc(width: 32).sext(width: 64),
18697 DL: dl, VT: OpVT), Cond);
18698}
18699
18700static SDValue
18701performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
18702 const RISCVSubtarget &Subtarget) {
18703 SelectionDAG &DAG = DCI.DAG;
18704 SDValue Src = N->getOperand(Num: 0);
18705 EVT VT = N->getValueType(ResNo: 0);
18706 EVT SrcVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
18707 unsigned Opc = Src.getOpcode();
18708 SDLoc DL(N);
18709
18710 // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
18711 // Don't do this with Zhinx. We need to explicitly sign extend the GPR.
18712 if (Opc == RISCVISD::FMV_X_ANYEXTH && SrcVT.bitsGE(VT: MVT::i16) &&
18713 Subtarget.hasStdExtZfhmin())
18714 return DAG.getNode(Opcode: RISCVISD::FMV_X_SIGNEXTH, DL, VT, Operand: Src.getOperand(i: 0));
18715
18716 // Fold (sext_inreg (shl X, Y), i32) -> (sllw X, Y) iff Y u< 32
18717 if (Opc == ISD::SHL && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
18718 VT == MVT::i64 && !isa<ConstantSDNode>(Val: Src.getOperand(i: 1)) &&
18719 DAG.computeKnownBits(Op: Src.getOperand(i: 1)).countMaxActiveBits() <= 5)
18720 return DAG.getNode(Opcode: RISCVISD::SLLW, DL, VT, N1: Src.getOperand(i: 0),
18721 N2: Src.getOperand(i: 1));
18722
18723 // Fold (sext_inreg (setcc), i1) -> (sub 0, (setcc))
18724 if (Opc == ISD::SETCC && SrcVT == MVT::i1 && DCI.isAfterLegalizeDAG())
18725 return DAG.getNegative(Val: Src, DL, VT);
18726
18727 // Fold (sext_inreg (xor (setcc), -1), i1) -> (add (setcc), -1)
18728 if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
18729 isAllOnesConstant(V: Src.getOperand(i: 1)) &&
18730 Src.getOperand(i: 0).getOpcode() == ISD::SETCC && DCI.isAfterLegalizeDAG())
18731 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Src.getOperand(i: 0),
18732 N2: DAG.getAllOnesConstant(DL, VT));
18733
18734 return SDValue();
18735}
18736
18737namespace {
18738// Forward declaration of the structure holding the necessary information to
18739// apply a combine.
18740struct CombineResult;
18741
18742enum ExtKind : uint8_t {
18743 ZExt = 1 << 0,
18744 SExt = 1 << 1,
18745 FPExt = 1 << 2,
18746 BF16Ext = 1 << 3
18747};
18748/// Helper class for folding sign/zero extensions.
18749/// In particular, this class is used for the following combines:
18750/// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
18751/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
18752/// mul | mul_vl -> vwmul(u) | vwmul_su
18753/// shl | shl_vl -> vwsll
18754/// fadd -> vfwadd | vfwadd_w
18755/// fsub -> vfwsub | vfwsub_w
18756/// fmul -> vfwmul
18757/// An object of this class represents an operand of the operation we want to
18758/// combine.
18759/// E.g., when trying to combine `mul_vl a, b`, we will have one instance of
18760/// NodeExtensionHelper for `a` and one for `b`.
18761///
18762/// This class abstracts away how the extension is materialized and
18763/// how its number of users affect the combines.
18764///
18765/// In particular:
18766/// - VWADD_W is conceptually == add(op0, sext(op1))
18767/// - VWADDU_W == add(op0, zext(op1))
18768/// - VWSUB_W == sub(op0, sext(op1))
18769/// - VWSUBU_W == sub(op0, zext(op1))
18770/// - VFWADD_W == fadd(op0, fpext(op1))
18771/// - VFWSUB_W == fsub(op0, fpext(op1))
18772/// And VMV_V_X_VL, depending on the value, is conceptually equivalent to
18773/// zext|sext(smaller_value).
18774struct NodeExtensionHelper {
18775 /// Records if this operand is like being zero extended.
18776 bool SupportsZExt;
18777 /// Records if this operand is like being sign extended.
18778 /// Note: SupportsZExt and SupportsSExt are not mutually exclusive. For
18779 /// instance, a splat constant (e.g., 3), would support being both sign and
18780 /// zero extended.
18781 bool SupportsSExt;
18782 /// Records if this operand is like being floating point extended.
18783 bool SupportsFPExt;
18784 /// Records if this operand is extended from bf16.
18785 bool SupportsBF16Ext;
18786 /// This boolean captures whether we care if this operand would still be
18787 /// around after the folding happens.
18788 bool EnforceOneUse;
18789 /// Original value that this NodeExtensionHelper represents.
18790 SDValue OrigOperand;
18791
18792 /// Get the value feeding the extension or the value itself.
18793 /// E.g., for zext(a), this would return a.
18794 SDValue getSource() const {
18795 switch (OrigOperand.getOpcode()) {
18796 case ISD::ZERO_EXTEND:
18797 case ISD::SIGN_EXTEND:
18798 case RISCVISD::VSEXT_VL:
18799 case RISCVISD::VZEXT_VL:
18800 case RISCVISD::FP_EXTEND_VL:
18801 return OrigOperand.getOperand(i: 0);
18802 default:
18803 return OrigOperand;
18804 }
18805 }
18806
18807 /// Check if this instance represents a splat.
18808 bool isSplat() const {
18809 return OrigOperand.getOpcode() == RISCVISD::VMV_V_X_VL ||
18810 OrigOperand.getOpcode() == ISD::SPLAT_VECTOR;
18811 }
18812
18813 /// Get the extended opcode.
18814 unsigned getExtOpc(ExtKind SupportsExt) const {
18815 switch (SupportsExt) {
18816 case ExtKind::SExt:
18817 return RISCVISD::VSEXT_VL;
18818 case ExtKind::ZExt:
18819 return RISCVISD::VZEXT_VL;
18820 case ExtKind::FPExt:
18821 case ExtKind::BF16Ext:
18822 return RISCVISD::FP_EXTEND_VL;
18823 }
18824 llvm_unreachable("Unknown ExtKind enum");
18825 }
18826
18827 /// Get or create a value that can feed \p Root with the given extension \p
18828 /// SupportsExt. If \p SExt is std::nullopt, this returns the source of this
18829 /// operand. \see ::getSource().
18830 SDValue getOrCreateExtendedOp(SDNode *Root, SelectionDAG &DAG,
18831 const RISCVSubtarget &Subtarget,
18832 std::optional<ExtKind> SupportsExt) const {
18833 if (!SupportsExt.has_value())
18834 return OrigOperand;
18835
18836 MVT NarrowVT = getNarrowType(Root, SupportsExt: *SupportsExt);
18837
18838 SDValue Source = getSource();
18839 assert(Subtarget.getTargetLowering()->isTypeLegal(Source.getValueType()));
18840 if (Source.getValueType() == NarrowVT)
18841 return Source;
18842
18843 unsigned ExtOpc = getExtOpc(SupportsExt: *SupportsExt);
18844
18845 // If we need an extension, we should be changing the type.
18846 SDLoc DL(OrigOperand);
18847 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
18848 switch (OrigOperand.getOpcode()) {
18849 case ISD::ZERO_EXTEND:
18850 case ISD::SIGN_EXTEND:
18851 case RISCVISD::VSEXT_VL:
18852 case RISCVISD::VZEXT_VL:
18853 case RISCVISD::FP_EXTEND_VL:
18854 return DAG.getNode(Opcode: ExtOpc, DL, VT: NarrowVT, N1: Source, N2: Mask, N3: VL);
18855 case ISD::SPLAT_VECTOR:
18856 return DAG.getSplat(VT: NarrowVT, DL, Op: Source.getOperand(i: 0));
18857 case RISCVISD::VMV_V_X_VL:
18858 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: NarrowVT,
18859 N1: DAG.getUNDEF(VT: NarrowVT), N2: Source.getOperand(i: 1), N3: VL);
18860 case RISCVISD::VFMV_V_F_VL:
18861 Source = Source.getOperand(i: 1);
18862 assert(Source.getOpcode() == ISD::FP_EXTEND && "Unexpected source");
18863 Source = Source.getOperand(i: 0);
18864 assert(Source.getValueType() == NarrowVT.getVectorElementType());
18865 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: NarrowVT,
18866 N1: DAG.getUNDEF(VT: NarrowVT), N2: Source, N3: VL);
18867 default:
18868 // Other opcodes can only come from the original LHS of VW(ADD|SUB)_W_VL
18869 // and that operand should already have the right NarrowVT so no
18870 // extension should be required at this point.
18871 llvm_unreachable("Unsupported opcode");
18872 }
18873 }
18874
18875 /// Helper function to get the narrow type for \p Root.
18876 /// The narrow type is the type of \p Root where we divided the size of each
18877 /// element by 2. E.g., if Root's type <2xi16> -> narrow type <2xi8>.
18878 /// \pre Both the narrow type and the original type should be legal.
18879 static MVT getNarrowType(const SDNode *Root, ExtKind SupportsExt) {
18880 MVT VT = Root->getSimpleValueType(ResNo: 0);
18881
18882 // Determine the narrow size.
18883 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
18884
18885 MVT EltVT = SupportsExt == ExtKind::BF16Ext ? MVT::bf16
18886 : SupportsExt == ExtKind::FPExt
18887 ? MVT::getFloatingPointVT(BitWidth: NarrowSize)
18888 : MVT::getIntegerVT(BitWidth: NarrowSize);
18889
18890 assert((int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
18891 "Trying to extend something we can't represent");
18892 MVT NarrowVT = MVT::getVectorVT(VT: EltVT, EC: VT.getVectorElementCount());
18893 return NarrowVT;
18894 }
18895
18896 /// Get the opcode to materialize:
18897 /// Opcode(sext(a), sext(b)) -> newOpcode(a, b)
18898 static unsigned getSExtOpcode(unsigned Opcode) {
18899 switch (Opcode) {
18900 case ISD::ADD:
18901 case RISCVISD::ADD_VL:
18902 case RISCVISD::VWADD_W_VL:
18903 case RISCVISD::VWADDU_W_VL:
18904 case ISD::OR:
18905 case RISCVISD::OR_VL:
18906 return RISCVISD::VWADD_VL;
18907 case ISD::SUB:
18908 case RISCVISD::SUB_VL:
18909 case RISCVISD::VWSUB_W_VL:
18910 case RISCVISD::VWSUBU_W_VL:
18911 return RISCVISD::VWSUB_VL;
18912 case ISD::MUL:
18913 case RISCVISD::MUL_VL:
18914 return RISCVISD::VWMUL_VL;
18915 default:
18916 llvm_unreachable("Unexpected opcode");
18917 }
18918 }
18919
18920 /// Get the opcode to materialize:
18921 /// Opcode(zext(a), zext(b)) -> newOpcode(a, b)
18922 static unsigned getZExtOpcode(unsigned Opcode) {
18923 switch (Opcode) {
18924 case ISD::ADD:
18925 case RISCVISD::ADD_VL:
18926 case RISCVISD::VWADD_W_VL:
18927 case RISCVISD::VWADDU_W_VL:
18928 case ISD::OR:
18929 case RISCVISD::OR_VL:
18930 return RISCVISD::VWADDU_VL;
18931 case ISD::SUB:
18932 case RISCVISD::SUB_VL:
18933 case RISCVISD::VWSUB_W_VL:
18934 case RISCVISD::VWSUBU_W_VL:
18935 return RISCVISD::VWSUBU_VL;
18936 case ISD::MUL:
18937 case RISCVISD::MUL_VL:
18938 return RISCVISD::VWMULU_VL;
18939 case ISD::SHL:
18940 case RISCVISD::SHL_VL:
18941 return RISCVISD::VWSLL_VL;
18942 default:
18943 llvm_unreachable("Unexpected opcode");
18944 }
18945 }
18946
18947 /// Get the opcode to materialize:
18948 /// Opcode(fpext(a), fpext(b)) -> newOpcode(a, b)
18949 static unsigned getFPExtOpcode(unsigned Opcode) {
18950 switch (Opcode) {
18951 case RISCVISD::FADD_VL:
18952 case RISCVISD::VFWADD_W_VL:
18953 return RISCVISD::VFWADD_VL;
18954 case RISCVISD::FSUB_VL:
18955 case RISCVISD::VFWSUB_W_VL:
18956 return RISCVISD::VFWSUB_VL;
18957 case RISCVISD::FMUL_VL:
18958 return RISCVISD::VFWMUL_VL;
18959 case RISCVISD::VFMADD_VL:
18960 return RISCVISD::VFWMADD_VL;
18961 case RISCVISD::VFMSUB_VL:
18962 return RISCVISD::VFWMSUB_VL;
18963 case RISCVISD::VFNMADD_VL:
18964 return RISCVISD::VFWNMADD_VL;
18965 case RISCVISD::VFNMSUB_VL:
18966 return RISCVISD::VFWNMSUB_VL;
18967 default:
18968 llvm_unreachable("Unexpected opcode");
18969 }
18970 }
18971
18972 /// Get the opcode to materialize \p Opcode(sext(a), zext(b)) ->
18973 /// newOpcode(a, b).
18974 static unsigned getSUOpcode(unsigned Opcode) {
18975 assert((Opcode == RISCVISD::MUL_VL || Opcode == ISD::MUL) &&
18976 "SU is only supported for MUL");
18977 return RISCVISD::VWMULSU_VL;
18978 }
18979
18980 /// Get the opcode to materialize
18981 /// \p Opcode(a, s|z|fpext(b)) -> newOpcode(a, b).
18982 static unsigned getWOpcode(unsigned Opcode, ExtKind SupportsExt) {
18983 switch (Opcode) {
18984 case ISD::ADD:
18985 case RISCVISD::ADD_VL:
18986 case ISD::OR:
18987 case RISCVISD::OR_VL:
18988 return SupportsExt == ExtKind::SExt ? RISCVISD::VWADD_W_VL
18989 : RISCVISD::VWADDU_W_VL;
18990 case ISD::SUB:
18991 case RISCVISD::SUB_VL:
18992 return SupportsExt == ExtKind::SExt ? RISCVISD::VWSUB_W_VL
18993 : RISCVISD::VWSUBU_W_VL;
18994 case RISCVISD::FADD_VL:
18995 return RISCVISD::VFWADD_W_VL;
18996 case RISCVISD::FSUB_VL:
18997 return RISCVISD::VFWSUB_W_VL;
18998 default:
18999 llvm_unreachable("Unexpected opcode");
19000 }
19001 }
19002
19003 using CombineToTry = std::function<std::optional<CombineResult>(
19004 SDNode * /*Root*/, const NodeExtensionHelper & /*LHS*/,
19005 const NodeExtensionHelper & /*RHS*/, SelectionDAG &,
19006 const RISCVSubtarget &)>;
19007
19008 /// Check if this node needs to be fully folded or extended for all users.
19009 bool needToPromoteOtherUsers() const { return EnforceOneUse; }
19010
19011 void fillUpExtensionSupportForSplat(SDNode *Root, SelectionDAG &DAG,
19012 const RISCVSubtarget &Subtarget) {
19013 unsigned Opc = OrigOperand.getOpcode();
19014 MVT VT = OrigOperand.getSimpleValueType();
19015
19016 assert((Opc == ISD::SPLAT_VECTOR || Opc == RISCVISD::VMV_V_X_VL) &&
19017 "Unexpected Opcode");
19018
19019 // The pasthru must be undef for tail agnostic.
19020 if (Opc == RISCVISD::VMV_V_X_VL && !OrigOperand.getOperand(i: 0).isUndef())
19021 return;
19022
19023 // Get the scalar value.
19024 SDValue Op = Opc == ISD::SPLAT_VECTOR ? OrigOperand.getOperand(i: 0)
19025 : OrigOperand.getOperand(i: 1);
19026
19027 // See if we have enough sign bits or zero bits in the scalar to use a
19028 // widening opcode by splatting to smaller element size.
19029 unsigned EltBits = VT.getScalarSizeInBits();
19030 unsigned ScalarBits = Op.getValueSizeInBits();
19031 // If we're not getting all bits from the element, we need special handling.
19032 if (ScalarBits < EltBits) {
19033 // This should only occur on RV32.
19034 assert(Opc == RISCVISD::VMV_V_X_VL && EltBits == 64 && ScalarBits == 32 &&
19035 !Subtarget.is64Bit() && "Unexpected splat");
19036 // vmv.v.x sign extends narrow inputs.
19037 SupportsSExt = true;
19038
19039 // If the input is positive, then sign extend is also zero extend.
19040 if (DAG.SignBitIsZero(Op))
19041 SupportsZExt = true;
19042
19043 EnforceOneUse = false;
19044 return;
19045 }
19046
19047 unsigned NarrowSize = EltBits / 2;
19048 // If the narrow type cannot be expressed with a legal VMV,
19049 // this is not a valid candidate.
19050 if (NarrowSize < 8)
19051 return;
19052
19053 if (DAG.ComputeMaxSignificantBits(Op) <= NarrowSize)
19054 SupportsSExt = true;
19055
19056 if (DAG.MaskedValueIsZero(Op,
19057 Mask: APInt::getBitsSetFrom(numBits: ScalarBits, loBit: NarrowSize)))
19058 SupportsZExt = true;
19059
19060 EnforceOneUse = false;
19061 }
19062
19063 bool isSupportedFPExtend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
19064 return (NarrowEltVT == MVT::f32 ||
19065 (NarrowEltVT == MVT::f16 && Subtarget.hasVInstructionsF16()));
19066 }
19067
19068 bool isSupportedBF16Extend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
19069 return NarrowEltVT == MVT::bf16 &&
19070 (Subtarget.hasStdExtZvfbfwma() || Subtarget.hasVInstructionsBF16());
19071 }
19072
19073 /// Helper method to set the various fields of this struct based on the
19074 /// type of \p Root.
19075 void fillUpExtensionSupport(SDNode *Root, SelectionDAG &DAG,
19076 const RISCVSubtarget &Subtarget) {
19077 SupportsZExt = false;
19078 SupportsSExt = false;
19079 SupportsFPExt = false;
19080 SupportsBF16Ext = false;
19081 EnforceOneUse = true;
19082 unsigned Opc = OrigOperand.getOpcode();
19083 // For the nodes we handle below, we end up using their inputs directly: see
19084 // getSource(). However since they either don't have a passthru or we check
19085 // that their passthru is undef, we can safely ignore their mask and VL.
19086 switch (Opc) {
19087 case ISD::ZERO_EXTEND:
19088 case ISD::SIGN_EXTEND: {
19089 MVT VT = OrigOperand.getSimpleValueType();
19090 if (!VT.isVector())
19091 break;
19092
19093 SDValue NarrowElt = OrigOperand.getOperand(i: 0);
19094 MVT NarrowVT = NarrowElt.getSimpleValueType();
19095 // i1 types are legal but we can't select V{S,Z}EXT_VLs with them.
19096 if (NarrowVT.getVectorElementType() == MVT::i1)
19097 break;
19098
19099 SupportsZExt = Opc == ISD::ZERO_EXTEND;
19100 SupportsSExt = Opc == ISD::SIGN_EXTEND;
19101 break;
19102 }
19103 case RISCVISD::VZEXT_VL:
19104 SupportsZExt = true;
19105 break;
19106 case RISCVISD::VSEXT_VL:
19107 SupportsSExt = true;
19108 break;
19109 case RISCVISD::FP_EXTEND_VL: {
19110 MVT NarrowEltVT =
19111 OrigOperand.getOperand(i: 0).getSimpleValueType().getVectorElementType();
19112 if (isSupportedFPExtend(NarrowEltVT, Subtarget))
19113 SupportsFPExt = true;
19114 if (isSupportedBF16Extend(NarrowEltVT, Subtarget))
19115 SupportsBF16Ext = true;
19116
19117 break;
19118 }
19119 case ISD::SPLAT_VECTOR:
19120 case RISCVISD::VMV_V_X_VL:
19121 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
19122 break;
19123 case RISCVISD::VFMV_V_F_VL: {
19124 MVT VT = OrigOperand.getSimpleValueType();
19125
19126 if (!OrigOperand.getOperand(i: 0).isUndef())
19127 break;
19128
19129 SDValue Op = OrigOperand.getOperand(i: 1);
19130 if (Op.getOpcode() != ISD::FP_EXTEND)
19131 break;
19132
19133 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
19134 unsigned ScalarBits = Op.getOperand(i: 0).getValueSizeInBits();
19135 if (NarrowSize != ScalarBits)
19136 break;
19137
19138 if (isSupportedFPExtend(NarrowEltVT: Op.getOperand(i: 0).getSimpleValueType(), Subtarget))
19139 SupportsFPExt = true;
19140 if (isSupportedBF16Extend(NarrowEltVT: Op.getOperand(i: 0).getSimpleValueType(),
19141 Subtarget))
19142 SupportsBF16Ext = true;
19143 break;
19144 }
19145 default:
19146 break;
19147 }
19148 }
19149
19150 /// Check if \p Root supports any extension folding combines.
19151 static bool isSupportedRoot(const SDNode *Root,
19152 const RISCVSubtarget &Subtarget) {
19153 switch (Root->getOpcode()) {
19154 case ISD::ADD:
19155 case ISD::SUB:
19156 case ISD::MUL: {
19157 return Root->getValueType(ResNo: 0).isScalableVector();
19158 }
19159 case ISD::OR: {
19160 return Root->getValueType(ResNo: 0).isScalableVector() &&
19161 Root->getFlags().hasDisjoint();
19162 }
19163 // Vector Widening Integer Add/Sub/Mul Instructions
19164 case RISCVISD::ADD_VL:
19165 case RISCVISD::MUL_VL:
19166 case RISCVISD::VWADD_W_VL:
19167 case RISCVISD::VWADDU_W_VL:
19168 case RISCVISD::SUB_VL:
19169 case RISCVISD::VWSUB_W_VL:
19170 case RISCVISD::VWSUBU_W_VL:
19171 // Vector Widening Floating-Point Add/Sub/Mul Instructions
19172 case RISCVISD::FADD_VL:
19173 case RISCVISD::FSUB_VL:
19174 case RISCVISD::FMUL_VL:
19175 case RISCVISD::VFWADD_W_VL:
19176 case RISCVISD::VFWSUB_W_VL:
19177 return true;
19178 case RISCVISD::OR_VL:
19179 return Root->getFlags().hasDisjoint();
19180 case ISD::SHL:
19181 return Root->getValueType(ResNo: 0).isScalableVector() &&
19182 Subtarget.hasStdExtZvbb();
19183 case RISCVISD::SHL_VL:
19184 return Subtarget.hasStdExtZvbb();
19185 case RISCVISD::VFMADD_VL:
19186 case RISCVISD::VFNMSUB_VL:
19187 case RISCVISD::VFNMADD_VL:
19188 case RISCVISD::VFMSUB_VL:
19189 return true;
19190 default:
19191 return false;
19192 }
19193 }
19194
19195 /// Build a NodeExtensionHelper for \p Root.getOperand(\p OperandIdx).
19196 NodeExtensionHelper(SDNode *Root, unsigned OperandIdx, SelectionDAG &DAG,
19197 const RISCVSubtarget &Subtarget) {
19198 assert(isSupportedRoot(Root, Subtarget) &&
19199 "Trying to build an helper with an "
19200 "unsupported root");
19201 assert(OperandIdx < 2 && "Requesting something else than LHS or RHS");
19202 assert(DAG.getTargetLoweringInfo().isTypeLegal(Root->getValueType(0)));
19203 OrigOperand = Root->getOperand(Num: OperandIdx);
19204
19205 unsigned Opc = Root->getOpcode();
19206 switch (Opc) {
19207 // We consider
19208 // VW<ADD|SUB>_W(LHS, RHS) -> <ADD|SUB>(LHS, SEXT(RHS))
19209 // VW<ADD|SUB>U_W(LHS, RHS) -> <ADD|SUB>(LHS, ZEXT(RHS))
19210 // VFW<ADD|SUB>_W(LHS, RHS) -> F<ADD|SUB>(LHS, FPEXT(RHS))
19211 case RISCVISD::VWADD_W_VL:
19212 case RISCVISD::VWADDU_W_VL:
19213 case RISCVISD::VWSUB_W_VL:
19214 case RISCVISD::VWSUBU_W_VL:
19215 case RISCVISD::VFWADD_W_VL:
19216 case RISCVISD::VFWSUB_W_VL:
19217 // Operand 1 can't be changed.
19218 if (OperandIdx == 1)
19219 break;
19220 [[fallthrough]];
19221 default:
19222 fillUpExtensionSupport(Root, DAG, Subtarget);
19223 break;
19224 }
19225 }
19226
19227 /// Helper function to get the Mask and VL from \p Root.
19228 static std::pair<SDValue, SDValue>
19229 getMaskAndVL(const SDNode *Root, SelectionDAG &DAG,
19230 const RISCVSubtarget &Subtarget) {
19231 assert(isSupportedRoot(Root, Subtarget) && "Unexpected root");
19232 switch (Root->getOpcode()) {
19233 case ISD::ADD:
19234 case ISD::SUB:
19235 case ISD::MUL:
19236 case ISD::OR:
19237 case ISD::SHL: {
19238 SDLoc DL(Root);
19239 MVT VT = Root->getSimpleValueType(ResNo: 0);
19240 return getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
19241 }
19242 default:
19243 return std::make_pair(x: Root->getOperand(Num: 3), y: Root->getOperand(Num: 4));
19244 }
19245 }
19246
19247 /// Helper function to check if \p N is commutative with respect to the
19248 /// foldings that are supported by this class.
19249 static bool isCommutative(const SDNode *N) {
19250 switch (N->getOpcode()) {
19251 case ISD::ADD:
19252 case ISD::MUL:
19253 case ISD::OR:
19254 case RISCVISD::ADD_VL:
19255 case RISCVISD::MUL_VL:
19256 case RISCVISD::OR_VL:
19257 case RISCVISD::FADD_VL:
19258 case RISCVISD::FMUL_VL:
19259 case RISCVISD::VFMADD_VL:
19260 case RISCVISD::VFNMSUB_VL:
19261 case RISCVISD::VFNMADD_VL:
19262 case RISCVISD::VFMSUB_VL:
19263 return true;
19264 case RISCVISD::VWADD_W_VL:
19265 case RISCVISD::VWADDU_W_VL:
19266 case ISD::SUB:
19267 case RISCVISD::SUB_VL:
19268 case RISCVISD::VWSUB_W_VL:
19269 case RISCVISD::VWSUBU_W_VL:
19270 case RISCVISD::VFWADD_W_VL:
19271 case RISCVISD::FSUB_VL:
19272 case RISCVISD::VFWSUB_W_VL:
19273 case ISD::SHL:
19274 case RISCVISD::SHL_VL:
19275 return false;
19276 default:
19277 llvm_unreachable("Unexpected opcode");
19278 }
19279 }
19280
19281 /// Get a list of combine to try for folding extensions in \p Root.
19282 /// Note that each returned CombineToTry function doesn't actually modify
19283 /// anything. Instead they produce an optional CombineResult that if not None,
19284 /// need to be materialized for the combine to be applied.
19285 /// \see CombineResult::materialize.
19286 /// If the related CombineToTry function returns std::nullopt, that means the
19287 /// combine didn't match.
19288 static SmallVector<CombineToTry>
19289 getSupportedFoldings(const SDNode *Root, const RISCVSubtarget &Subtarget);
19290};
19291
19292/// Helper structure that holds all the necessary information to materialize a
19293/// combine that does some extension folding.
19294struct CombineResult {
19295 /// Opcode to be generated when materializing the combine.
19296 unsigned TargetOpcode;
19297 // No value means no extension is needed.
19298 std::optional<ExtKind> LHSExt;
19299 std::optional<ExtKind> RHSExt;
19300 /// Root of the combine.
19301 SDNode *Root;
19302 /// LHS of the TargetOpcode.
19303 NodeExtensionHelper LHS;
19304 /// RHS of the TargetOpcode.
19305 NodeExtensionHelper RHS;
19306
19307 CombineResult(unsigned TargetOpcode, SDNode *Root,
19308 const NodeExtensionHelper &LHS, std::optional<ExtKind> LHSExt,
19309 const NodeExtensionHelper &RHS, std::optional<ExtKind> RHSExt)
19310 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
19311 LHS(LHS), RHS(RHS) {}
19312
19313 /// Return a value that uses TargetOpcode and that can be used to replace
19314 /// Root.
19315 /// The actual replacement is *not* done in that method.
19316 SDValue materialize(SelectionDAG &DAG,
19317 const RISCVSubtarget &Subtarget) const {
19318 SDValue Mask, VL, Passthru;
19319 std::tie(args&: Mask, args&: VL) =
19320 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
19321 switch (Root->getOpcode()) {
19322 default:
19323 Passthru = Root->getOperand(Num: 2);
19324 break;
19325 case ISD::ADD:
19326 case ISD::SUB:
19327 case ISD::MUL:
19328 case ISD::OR:
19329 case ISD::SHL:
19330 Passthru = DAG.getUNDEF(VT: Root->getValueType(ResNo: 0));
19331 break;
19332 }
19333 return DAG.getNode(Opcode: TargetOpcode, DL: SDLoc(Root), VT: Root->getValueType(ResNo: 0),
19334 N1: LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SupportsExt: LHSExt),
19335 N2: RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SupportsExt: RHSExt),
19336 N3: Passthru, N4: Mask, N5: VL);
19337 }
19338};
19339
19340/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
19341/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
19342/// are zext) and LHS and RHS can be folded into Root.
19343/// AllowExtMask define which form `ext` can take in this pattern.
19344///
19345/// \note If the pattern can match with both zext and sext, the returned
19346/// CombineResult will feature the zext result.
19347///
19348/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19349/// can be used to apply the pattern.
19350static std::optional<CombineResult>
19351canFoldToVWWithSameExtensionImpl(SDNode *Root, const NodeExtensionHelper &LHS,
19352 const NodeExtensionHelper &RHS,
19353 uint8_t AllowExtMask, SelectionDAG &DAG,
19354 const RISCVSubtarget &Subtarget) {
19355 if ((AllowExtMask & ExtKind::ZExt) && LHS.SupportsZExt && RHS.SupportsZExt)
19356 return CombineResult(NodeExtensionHelper::getZExtOpcode(Opcode: Root->getOpcode()),
19357 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
19358 /*RHSExt=*/{ExtKind::ZExt});
19359 if ((AllowExtMask & ExtKind::SExt) && LHS.SupportsSExt && RHS.SupportsSExt)
19360 return CombineResult(NodeExtensionHelper::getSExtOpcode(Opcode: Root->getOpcode()),
19361 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
19362 /*RHSExt=*/{ExtKind::SExt});
19363 if ((AllowExtMask & ExtKind::FPExt) && LHS.SupportsFPExt && RHS.SupportsFPExt)
19364 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
19365 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
19366 /*RHSExt=*/{ExtKind::FPExt});
19367 if ((AllowExtMask & ExtKind::BF16Ext) && LHS.SupportsBF16Ext &&
19368 RHS.SupportsBF16Ext)
19369 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
19370 Root, LHS, /*LHSExt=*/{ExtKind::BF16Ext}, RHS,
19371 /*RHSExt=*/{ExtKind::BF16Ext});
19372 return std::nullopt;
19373}
19374
19375/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
19376/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
19377/// are zext) and LHS and RHS can be folded into Root.
19378///
19379/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19380/// can be used to apply the pattern.
19381static std::optional<CombineResult>
19382canFoldToVWWithSameExtension(SDNode *Root, const NodeExtensionHelper &LHS,
19383 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19384 const RISCVSubtarget &Subtarget) {
19385 return canFoldToVWWithSameExtensionImpl(
19386 Root, LHS, RHS, AllowExtMask: ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
19387 Subtarget);
19388}
19389
19390/// Check if \p Root follows a pattern Root(zext(LHS), zext(RHS))
19391///
19392/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19393/// can be used to apply the pattern.
19394static std::optional<CombineResult>
19395canFoldToVWWithSameExtZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19396 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19397 const RISCVSubtarget &Subtarget) {
19398 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, AllowExtMask: ExtKind::ZExt, DAG,
19399 Subtarget);
19400}
19401
19402/// Check if \p Root follows a pattern Root(bf16ext(LHS), bf16ext(RHS))
19403///
19404/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19405/// can be used to apply the pattern.
19406static std::optional<CombineResult>
19407canFoldToVWWithSameExtBF16(SDNode *Root, const NodeExtensionHelper &LHS,
19408 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19409 const RISCVSubtarget &Subtarget) {
19410 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, AllowExtMask: ExtKind::BF16Ext, DAG,
19411 Subtarget);
19412}
19413
19414/// Check if \p Root follows a pattern Root(LHS, ext(RHS))
19415///
19416/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19417/// can be used to apply the pattern.
19418static std::optional<CombineResult>
19419canFoldToVW_W(SDNode *Root, const NodeExtensionHelper &LHS,
19420 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19421 const RISCVSubtarget &Subtarget) {
19422 if (RHS.SupportsFPExt)
19423 return CombineResult(
19424 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::FPExt),
19425 Root, LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::FPExt});
19426
19427 // FIXME: Is it useful to form a vwadd.wx or vwsub.wx if it removes a scalar
19428 // sext/zext?
19429 // Control this behavior behind an option (AllowSplatInVW_W) for testing
19430 // purposes.
19431 if (RHS.SupportsZExt && (!RHS.isSplat() || AllowSplatInVW_W))
19432 return CombineResult(
19433 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::ZExt), Root,
19434 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::ZExt});
19435 if (RHS.SupportsSExt && (!RHS.isSplat() || AllowSplatInVW_W))
19436 return CombineResult(
19437 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::SExt), Root,
19438 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::SExt});
19439 return std::nullopt;
19440}
19441
19442/// Check if \p Root follows a pattern Root(sext(LHS), RHS)
19443///
19444/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19445/// can be used to apply the pattern.
19446static std::optional<CombineResult>
19447canFoldToVWWithSEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19448 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19449 const RISCVSubtarget &Subtarget) {
19450 if (LHS.SupportsSExt)
19451 return CombineResult(NodeExtensionHelper::getSExtOpcode(Opcode: Root->getOpcode()),
19452 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
19453 /*RHSExt=*/std::nullopt);
19454 return std::nullopt;
19455}
19456
19457/// Check if \p Root follows a pattern Root(zext(LHS), RHS)
19458///
19459/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19460/// can be used to apply the pattern.
19461static std::optional<CombineResult>
19462canFoldToVWWithZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19463 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19464 const RISCVSubtarget &Subtarget) {
19465 if (LHS.SupportsZExt)
19466 return CombineResult(NodeExtensionHelper::getZExtOpcode(Opcode: Root->getOpcode()),
19467 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
19468 /*RHSExt=*/std::nullopt);
19469 return std::nullopt;
19470}
19471
19472/// Check if \p Root follows a pattern Root(fpext(LHS), RHS)
19473///
19474/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19475/// can be used to apply the pattern.
19476static std::optional<CombineResult>
19477canFoldToVWWithFPEXT(SDNode *Root, const NodeExtensionHelper &LHS,
19478 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19479 const RISCVSubtarget &Subtarget) {
19480 if (LHS.SupportsFPExt)
19481 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
19482 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
19483 /*RHSExt=*/std::nullopt);
19484 return std::nullopt;
19485}
19486
19487/// Check if \p Root follows a pattern Root(sext(LHS), zext(RHS))
19488///
19489/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
19490/// can be used to apply the pattern.
19491static std::optional<CombineResult>
19492canFoldToVW_SU(SDNode *Root, const NodeExtensionHelper &LHS,
19493 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
19494 const RISCVSubtarget &Subtarget) {
19495
19496 if (!LHS.SupportsSExt || !RHS.SupportsZExt)
19497 return std::nullopt;
19498 return CombineResult(NodeExtensionHelper::getSUOpcode(Opcode: Root->getOpcode()),
19499 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
19500 /*RHSExt=*/{ExtKind::ZExt});
19501}
19502
19503SmallVector<NodeExtensionHelper::CombineToTry>
19504NodeExtensionHelper::getSupportedFoldings(const SDNode *Root,
19505 const RISCVSubtarget &Subtarget) {
19506 SmallVector<CombineToTry> Strategies;
19507 switch (Root->getOpcode()) {
19508 case ISD::ADD:
19509 case ISD::SUB:
19510 case ISD::OR:
19511 case RISCVISD::ADD_VL:
19512 case RISCVISD::SUB_VL:
19513 case RISCVISD::OR_VL:
19514 case RISCVISD::FADD_VL:
19515 case RISCVISD::FSUB_VL:
19516 // add|sub|fadd|fsub-> vwadd(u)|vwsub(u)|vfwadd|vfwsub
19517 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
19518 if (Subtarget.hasVInstructionsBF16())
19519 Strategies.push_back(Elt: canFoldToVWWithSameExtBF16);
19520 // add|sub|fadd|fsub -> vwadd(u)_w|vwsub(u)_w}|vfwadd_w|vfwsub_w
19521 Strategies.push_back(Elt: canFoldToVW_W);
19522 break;
19523 case RISCVISD::FMUL_VL:
19524 case RISCVISD::VFMADD_VL:
19525 case RISCVISD::VFMSUB_VL:
19526 case RISCVISD::VFNMADD_VL:
19527 case RISCVISD::VFNMSUB_VL:
19528 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
19529 if (Subtarget.hasVInstructionsBF16() ||
19530 (Subtarget.hasStdExtZvfbfwma() &&
19531 Root->getOpcode() == RISCVISD::VFMADD_VL))
19532 Strategies.push_back(Elt: canFoldToVWWithSameExtBF16);
19533 break;
19534 case ISD::MUL:
19535 case RISCVISD::MUL_VL:
19536 // mul -> vwmul(u)
19537 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
19538 // mul -> vwmulsu
19539 Strategies.push_back(Elt: canFoldToVW_SU);
19540 break;
19541 case ISD::SHL:
19542 case RISCVISD::SHL_VL:
19543 // shl -> vwsll
19544 Strategies.push_back(Elt: canFoldToVWWithSameExtZEXT);
19545 break;
19546 case RISCVISD::VWADD_W_VL:
19547 case RISCVISD::VWSUB_W_VL:
19548 // vwadd_w|vwsub_w -> vwadd|vwsub
19549 Strategies.push_back(Elt: canFoldToVWWithSEXT);
19550 break;
19551 case RISCVISD::VWADDU_W_VL:
19552 case RISCVISD::VWSUBU_W_VL:
19553 // vwaddu_w|vwsubu_w -> vwaddu|vwsubu
19554 Strategies.push_back(Elt: canFoldToVWWithZEXT);
19555 break;
19556 case RISCVISD::VFWADD_W_VL:
19557 case RISCVISD::VFWSUB_W_VL:
19558 // vfwadd_w|vfwsub_w -> vfwadd|vfwsub
19559 Strategies.push_back(Elt: canFoldToVWWithFPEXT);
19560 break;
19561 default:
19562 llvm_unreachable("Unexpected opcode");
19563 }
19564 return Strategies;
19565}
19566} // End anonymous namespace.
19567
19568static SDValue simplifyOp_VL(SDNode *N) {
19569 // TODO: Extend this to other binops using generic identity logic
19570 assert(N->getOpcode() == RISCVISD::ADD_VL);
19571 SDValue A = N->getOperand(Num: 0);
19572 SDValue B = N->getOperand(Num: 1);
19573 SDValue Passthru = N->getOperand(Num: 2);
19574 if (!Passthru.isUndef())
19575 // TODO:This could be a vmerge instead
19576 return SDValue();
19577 ;
19578 if (ISD::isConstantSplatVectorAllZeros(N: B.getNode()))
19579 return A;
19580 // Peek through fixed to scalable
19581 if (B.getOpcode() == ISD::INSERT_SUBVECTOR && B.getOperand(i: 0).isUndef() &&
19582 ISD::isConstantSplatVectorAllZeros(N: B.getOperand(i: 1).getNode()))
19583 return A;
19584 return SDValue();
19585}
19586
19587/// Combine a binary or FMA operation to its equivalent VW or VW_W form.
19588/// The supported combines are:
19589/// add | add_vl | or disjoint | or_vl disjoint -> vwadd(u) | vwadd(u)_w
19590/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
19591/// mul | mul_vl -> vwmul(u) | vwmul_su
19592/// shl | shl_vl -> vwsll
19593/// fadd_vl -> vfwadd | vfwadd_w
19594/// fsub_vl -> vfwsub | vfwsub_w
19595/// fmul_vl -> vfwmul
19596/// vwadd_w(u) -> vwadd(u)
19597/// vwsub_w(u) -> vwsub(u)
19598/// vfwadd_w -> vfwadd
19599/// vfwsub_w -> vfwsub
19600static SDValue combineOp_VLToVWOp_VL(SDNode *N,
19601 TargetLowering::DAGCombinerInfo &DCI,
19602 const RISCVSubtarget &Subtarget) {
19603 SelectionDAG &DAG = DCI.DAG;
19604 if (DCI.isBeforeLegalize())
19605 return SDValue();
19606
19607 if (!NodeExtensionHelper::isSupportedRoot(Root: N, Subtarget))
19608 return SDValue();
19609
19610 SmallVector<SDNode *> Worklist;
19611 SmallPtrSet<SDNode *, 8> Inserted;
19612 SmallPtrSet<SDNode *, 8> ExtensionsToRemove;
19613 Worklist.push_back(Elt: N);
19614 Inserted.insert(Ptr: N);
19615 SmallVector<CombineResult> CombinesToApply;
19616
19617 while (!Worklist.empty()) {
19618 SDNode *Root = Worklist.pop_back_val();
19619
19620 NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
19621 NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
19622 auto AppendUsersIfNeeded =
19623 [&Worklist, &Subtarget, &Inserted,
19624 &ExtensionsToRemove](const NodeExtensionHelper &Op) {
19625 if (Op.needToPromoteOtherUsers()) {
19626 // Remember that we're supposed to remove this extension.
19627 ExtensionsToRemove.insert(Ptr: Op.OrigOperand.getNode());
19628 for (SDUse &Use : Op.OrigOperand->uses()) {
19629 SDNode *TheUser = Use.getUser();
19630 if (!NodeExtensionHelper::isSupportedRoot(Root: TheUser, Subtarget))
19631 return false;
19632 // We only support the first 2 operands of FMA.
19633 if (Use.getOperandNo() >= 2)
19634 return false;
19635 if (Inserted.insert(Ptr: TheUser).second)
19636 Worklist.push_back(Elt: TheUser);
19637 }
19638 }
19639 return true;
19640 };
19641
19642 // Control the compile time by limiting the number of node we look at in
19643 // total.
19644 if (Inserted.size() > ExtensionMaxWebSize)
19645 return SDValue();
19646
19647 SmallVector<NodeExtensionHelper::CombineToTry> FoldingStrategies =
19648 NodeExtensionHelper::getSupportedFoldings(Root, Subtarget);
19649
19650 assert(!FoldingStrategies.empty() && "Nothing to be folded");
19651 bool Matched = false;
19652 for (int Attempt = 0;
19653 (Attempt != 1 + NodeExtensionHelper::isCommutative(N: Root)) && !Matched;
19654 ++Attempt) {
19655
19656 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
19657 FoldingStrategies) {
19658 std::optional<CombineResult> Res =
19659 FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
19660 if (Res) {
19661 // If this strategy wouldn't remove an extension we're supposed to
19662 // remove, reject it.
19663 if (!Res->LHSExt.has_value() &&
19664 ExtensionsToRemove.contains(Ptr: LHS.OrigOperand.getNode()))
19665 continue;
19666 if (!Res->RHSExt.has_value() &&
19667 ExtensionsToRemove.contains(Ptr: RHS.OrigOperand.getNode()))
19668 continue;
19669
19670 Matched = true;
19671 CombinesToApply.push_back(Elt: *Res);
19672 // All the inputs that are extended need to be folded, otherwise
19673 // we would be leaving the old input (since it is may still be used),
19674 // and the new one.
19675 if (Res->LHSExt.has_value())
19676 if (!AppendUsersIfNeeded(LHS))
19677 return SDValue();
19678 if (Res->RHSExt.has_value())
19679 if (!AppendUsersIfNeeded(RHS))
19680 return SDValue();
19681 break;
19682 }
19683 }
19684 std::swap(a&: LHS, b&: RHS);
19685 }
19686 // Right now we do an all or nothing approach.
19687 if (!Matched)
19688 return SDValue();
19689 }
19690 // Store the value for the replacement of the input node separately.
19691 SDValue InputRootReplacement;
19692 // We do the RAUW after we materialize all the combines, because some replaced
19693 // nodes may be feeding some of the yet-to-be-replaced nodes. Put differently,
19694 // some of these nodes may appear in the NodeExtensionHelpers of some of the
19695 // yet-to-be-visited CombinesToApply roots.
19696 SmallVector<std::pair<SDValue, SDValue>> ValuesToReplace;
19697 ValuesToReplace.reserve(N: CombinesToApply.size());
19698 for (CombineResult Res : CombinesToApply) {
19699 SDValue NewValue = Res.materialize(DAG, Subtarget);
19700 if (!InputRootReplacement) {
19701 assert(Res.Root == N &&
19702 "First element is expected to be the current node");
19703 InputRootReplacement = NewValue;
19704 } else {
19705 ValuesToReplace.emplace_back(Args: SDValue(Res.Root, 0), Args&: NewValue);
19706 }
19707 }
19708 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
19709 DCI.CombineTo(N: OldNewValues.first.getNode(), Res: OldNewValues.second);
19710 }
19711 return InputRootReplacement;
19712}
19713
19714// Fold (vwadd(u).wv y, (vmerge cond, x, 0)) -> vwadd(u).wv y, x, y, cond
19715// (vwsub(u).wv y, (vmerge cond, x, 0)) -> vwsub(u).wv y, x, y, cond
19716// y will be the Passthru and cond will be the Mask.
19717static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG) {
19718 unsigned Opc = N->getOpcode();
19719 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
19720 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
19721
19722 SDValue Y = N->getOperand(Num: 0);
19723 SDValue MergeOp = N->getOperand(Num: 1);
19724 unsigned MergeOpc = MergeOp.getOpcode();
19725
19726 if (MergeOpc != RISCVISD::VMERGE_VL && MergeOpc != ISD::VSELECT)
19727 return SDValue();
19728
19729 SDValue X = MergeOp->getOperand(Num: 1);
19730
19731 if (!MergeOp.hasOneUse())
19732 return SDValue();
19733
19734 // Passthru should be undef
19735 SDValue Passthru = N->getOperand(Num: 2);
19736 if (!Passthru.isUndef())
19737 return SDValue();
19738
19739 // Mask should be all ones
19740 SDValue Mask = N->getOperand(Num: 3);
19741 if (Mask.getOpcode() != RISCVISD::VMSET_VL)
19742 return SDValue();
19743
19744 // False value of MergeOp should be all zeros
19745 SDValue Z = MergeOp->getOperand(Num: 2);
19746
19747 if (Z.getOpcode() == ISD::INSERT_SUBVECTOR &&
19748 (isNullOrNullSplat(V: Z.getOperand(i: 0)) || Z.getOperand(i: 0).isUndef()))
19749 Z = Z.getOperand(i: 1);
19750
19751 if (!ISD::isConstantSplatVectorAllZeros(N: Z.getNode()))
19752 return SDValue();
19753
19754 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
19755 Ops: {Y, X, Y, MergeOp->getOperand(Num: 0), N->getOperand(Num: 4)},
19756 Flags: N->getFlags());
19757}
19758
19759// vwaddu C (vabd A B) -> vwabda(A B C)
19760// vwaddu C (vabdu A B) -> vwabdau(A B C)
19761static SDValue performVWABDACombine(SDNode *N, SelectionDAG &DAG,
19762 const RISCVSubtarget &Subtarget) {
19763 if (!Subtarget.hasStdExtZvabd())
19764 return SDValue();
19765
19766 MVT VT = N->getSimpleValueType(ResNo: 0);
19767 if (VT.getVectorElementType() != MVT::i8 &&
19768 VT.getVectorElementType() != MVT::i16)
19769 return SDValue();
19770
19771 SDValue Op0 = N->getOperand(Num: 0);
19772 SDValue Op1 = N->getOperand(Num: 1);
19773 SDValue Passthru = N->getOperand(Num: 2);
19774 if (!Passthru->isUndef())
19775 return SDValue();
19776
19777 SDValue Mask = N->getOperand(Num: 3);
19778 SDValue VL = N->getOperand(Num: 4);
19779 auto IsABD = [](SDValue Op) {
19780 if (Op->getOpcode() != RISCVISD::ABDS_VL &&
19781 Op->getOpcode() != RISCVISD::ABDU_VL)
19782 return SDValue();
19783 return Op;
19784 };
19785
19786 SDValue Diff = IsABD(Op0);
19787 Diff = Diff ? Diff : IsABD(Op1);
19788 if (!Diff)
19789 return SDValue();
19790 SDValue Acc = Diff == Op0 ? Op1 : Op0;
19791
19792 SDLoc DL(N);
19793 Acc = DAG.getNode(Opcode: RISCVISD::VZEXT_VL, DL, VT, N1: Acc, N2: Mask, N3: VL);
19794 SDValue Result = DAG.getNode(
19795 Opcode: Diff.getOpcode() == RISCVISD::ABDS_VL ? RISCVISD::VWABDA_VL
19796 : RISCVISD::VWABDAU_VL,
19797 DL, VT, N1: Diff.getOperand(i: 0), N2: Diff.getOperand(i: 1), N3: Acc, N4: Mask, N5: VL);
19798 return Result;
19799}
19800
19801// vwaddu_wv C (vabd A B) -> vwabda(A B C)
19802// vwaddu_wv C (zext (vabd A B)) -> vwabda(A (sext B) (sext C))
19803// vwaddu_wv C (vabdu A B) -> vwabdau(A B C)
19804// vwaddu_wv C (zext (vabdu A B)) -> vwabdau(A (zext B) (zext C))
19805static SDValue performVWABDACombineWV(SDNode *N, SelectionDAG &DAG,
19806 const RISCVSubtarget &Subtarget) {
19807 if (!Subtarget.hasStdExtZvabd())
19808 return SDValue();
19809
19810 MVT VT = N->getSimpleValueType(ResNo: 0);
19811 // The result is widened, so we can accept i16/i32 here.
19812 if (VT.getVectorElementType() != MVT::i16 &&
19813 VT.getVectorElementType() != MVT::i32)
19814 return SDValue();
19815
19816 SDValue Op0 = N->getOperand(Num: 0);
19817 SDValue Op1 = N->getOperand(Num: 1);
19818 SDValue Passthru = N->getOperand(Num: 2);
19819 if (!Passthru->isUndef())
19820 return SDValue();
19821
19822 SDValue Mask = N->getOperand(Num: 3);
19823 SDValue VL = N->getOperand(Num: 4);
19824 unsigned ExtOpc = 0;
19825 MVT ExtVT;
19826 auto GetDiff = [&](SDValue Op) {
19827 unsigned Opc = Op.getOpcode();
19828 if (Opc == RISCVISD::VZEXT_VL) {
19829 SDValue Src = Op->getOperand(Num: 0);
19830 unsigned SrcOpc = Src.getOpcode();
19831 switch (SrcOpc) {
19832 default:
19833 return SDValue();
19834 case ISD::ABDS:
19835 case RISCVISD::ABDS_VL:
19836 ExtOpc = RISCVISD::VSEXT_VL;
19837 break;
19838 case ISD::ABDU:
19839 case RISCVISD::ABDU_VL:
19840 ExtOpc = RISCVISD::VZEXT_VL;
19841 break;
19842 }
19843 ExtVT = Op->getSimpleValueType(ResNo: 0);
19844 return Src;
19845 }
19846
19847 if (Opc != ISD::ABDS && Opc != ISD::ABDU && Opc != RISCVISD::ABDS_VL &&
19848 Opc != RISCVISD::ABDU_VL)
19849 return SDValue();
19850 return Op;
19851 };
19852
19853 SDValue Diff = GetDiff(Op0);
19854 if (!Diff) {
19855 std::swap(a&: Op0, b&: Op1);
19856 Diff = GetDiff(Op0);
19857 if (!Diff)
19858 return SDValue();
19859 }
19860 SDValue Acc = Op1;
19861
19862 SDLoc DL(N);
19863 SDValue DiffA = Diff.getOperand(i: 0);
19864 SDValue DiffB = Diff.getOperand(i: 1);
19865 if (ExtOpc) {
19866 DiffA = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtVT, N1: DiffA, N2: Mask, N3: VL);
19867 DiffB = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtVT, N1: DiffB, N2: Mask, N3: VL);
19868 }
19869 SDValue Result = DAG.getNode(Opcode: Diff.getOpcode() == ISD::ABDS ||
19870 Diff.getOpcode() == RISCVISD::ABDS_VL
19871 ? RISCVISD::VWABDA_VL
19872 : RISCVISD::VWABDAU_VL,
19873 DL, VT, N1: DiffA, N2: DiffB, N3: Acc, N4: Mask, N5: VL);
19874 return Result;
19875}
19876
19877static SDValue performVWADDSUBW_VLCombine(SDNode *N,
19878 TargetLowering::DAGCombinerInfo &DCI,
19879 const RISCVSubtarget &Subtarget) {
19880 [[maybe_unused]] unsigned Opc = N->getOpcode();
19881 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
19882 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
19883
19884 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
19885 return V;
19886
19887 return combineVWADDSUBWSelect(N, DAG&: DCI.DAG);
19888}
19889
19890// Helper function for performMemPairCombine.
19891// Try to combine the memory loads/stores LSNode1 and LSNode2
19892// into a single memory pair operation.
19893static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1,
19894 LSBaseSDNode *LSNode2, SDValue BasePtr,
19895 uint64_t Imm) {
19896 SmallPtrSet<const SDNode *, 32> Visited;
19897 SmallVector<const SDNode *, 8> Worklist = {LSNode1, LSNode2};
19898
19899 if (SDNode::hasPredecessorHelper(N: LSNode1, Visited, Worklist) ||
19900 SDNode::hasPredecessorHelper(N: LSNode2, Visited, Worklist))
19901 return SDValue();
19902
19903 MachineFunction &MF = DAG.getMachineFunction();
19904 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
19905
19906 // The new operation has twice the width.
19907 MVT XLenVT = Subtarget.getXLenVT();
19908 EVT MemVT = LSNode1->getMemoryVT();
19909 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
19910 MachineMemOperand *MMO = LSNode1->getMemOperand();
19911 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
19912 MMO, PtrInfo: MMO->getPointerInfo(), Size: MemVT == MVT::i32 ? 8 : 16);
19913
19914 if (LSNode1->getOpcode() == ISD::LOAD) {
19915 auto Ext = cast<LoadSDNode>(Val: LSNode1)->getExtensionType();
19916 unsigned Opcode;
19917 if (MemVT == MVT::i32)
19918 Opcode = (Ext == ISD::ZEXTLOAD) ? RISCVISD::TH_LWUD : RISCVISD::TH_LWD;
19919 else
19920 Opcode = RISCVISD::TH_LDD;
19921
19922 SDValue Res = DAG.getMemIntrinsicNode(
19923 Opcode, dl: SDLoc(LSNode1), VTList: DAG.getVTList(VTs: {XLenVT, XLenVT, MVT::Other}),
19924 Ops: {LSNode1->getChain(), BasePtr,
19925 DAG.getConstant(Val: Imm, DL: SDLoc(LSNode1), VT: XLenVT)},
19926 MemVT: NewMemVT, MMO: NewMMO);
19927
19928 SDValue Node1 =
19929 DAG.getMergeValues(Ops: {Res.getValue(R: 0), Res.getValue(R: 2)}, dl: SDLoc(LSNode1));
19930 SDValue Node2 =
19931 DAG.getMergeValues(Ops: {Res.getValue(R: 1), Res.getValue(R: 2)}, dl: SDLoc(LSNode2));
19932
19933 DAG.ReplaceAllUsesWith(From: LSNode2, To: Node2.getNode());
19934 return Node1;
19935 } else {
19936 unsigned Opcode = (MemVT == MVT::i32) ? RISCVISD::TH_SWD : RISCVISD::TH_SDD;
19937
19938 SDValue Res = DAG.getMemIntrinsicNode(
19939 Opcode, dl: SDLoc(LSNode1), VTList: DAG.getVTList(VT: MVT::Other),
19940 Ops: {LSNode1->getChain(), LSNode1->getOperand(Num: 1), LSNode2->getOperand(Num: 1),
19941 BasePtr, DAG.getConstant(Val: Imm, DL: SDLoc(LSNode1), VT: XLenVT)},
19942 MemVT: NewMemVT, MMO: NewMMO);
19943
19944 DAG.ReplaceAllUsesWith(From: LSNode2, To: Res.getNode());
19945 return Res;
19946 }
19947}
19948
19949// Try to combine two adjacent loads/stores to a single pair instruction from
19950// the XTHeadMemPair vendor extension.
19951static SDValue performMemPairCombine(SDNode *N,
19952 TargetLowering::DAGCombinerInfo &DCI) {
19953 SelectionDAG &DAG = DCI.DAG;
19954 MachineFunction &MF = DAG.getMachineFunction();
19955 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
19956
19957 // Target does not support load/store pair.
19958 if (!Subtarget.hasVendorXTHeadMemPair())
19959 return SDValue();
19960
19961 LSBaseSDNode *LSNode1 = cast<LSBaseSDNode>(Val: N);
19962 EVT MemVT = LSNode1->getMemoryVT();
19963 unsigned OpNum = LSNode1->getOpcode() == ISD::LOAD ? 1 : 2;
19964
19965 // No volatile, indexed or atomic loads/stores.
19966 if (!LSNode1->isSimple() || LSNode1->isIndexed())
19967 return SDValue();
19968
19969 // Function to get a base + constant representation from a memory value.
19970 auto ExtractBaseAndOffset = [](SDValue Ptr) -> std::pair<SDValue, uint64_t> {
19971 if (Ptr->getOpcode() == ISD::ADD)
19972 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1)))
19973 return {Ptr->getOperand(Num: 0), C1->getZExtValue()};
19974 return {Ptr, 0};
19975 };
19976
19977 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->getOperand(Num: OpNum));
19978
19979 SDValue Chain = N->getOperand(Num: 0);
19980 for (SDUse &Use : Chain->uses()) {
19981 if (Use.getUser() != N && Use.getResNo() == 0 &&
19982 Use.getUser()->getOpcode() == N->getOpcode()) {
19983 LSBaseSDNode *LSNode2 = cast<LSBaseSDNode>(Val: Use.getUser());
19984
19985 // No volatile, indexed or atomic loads/stores.
19986 if (!LSNode2->isSimple() || LSNode2->isIndexed())
19987 continue;
19988
19989 // Check if LSNode1 and LSNode2 have the same type and extension.
19990 if (LSNode1->getOpcode() == ISD::LOAD)
19991 if (cast<LoadSDNode>(Val: LSNode2)->getExtensionType() !=
19992 cast<LoadSDNode>(Val: LSNode1)->getExtensionType())
19993 continue;
19994
19995 if (LSNode1->getMemoryVT() != LSNode2->getMemoryVT())
19996 continue;
19997
19998 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->getOperand(Num: OpNum));
19999
20000 // Check if the base pointer is the same for both instruction.
20001 if (Base1 != Base2)
20002 continue;
20003
20004 // Check if the offsets match the XTHeadMemPair encoding constraints.
20005 bool Valid = false;
20006 if (MemVT == MVT::i32) {
20007 // Check for adjacent i32 values and a 2-bit index.
20008 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(x: Offset1))
20009 Valid = true;
20010 } else if (MemVT == MVT::i64) {
20011 // Check for adjacent i64 values and a 2-bit index.
20012 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(x: Offset1))
20013 Valid = true;
20014 }
20015
20016 if (!Valid)
20017 continue;
20018
20019 // Try to combine.
20020 if (SDValue Res =
20021 tryMemPairCombine(DAG, LSNode1, LSNode2, BasePtr: Base1, Imm: Offset1))
20022 return Res;
20023 }
20024 }
20025
20026 return SDValue();
20027}
20028
20029// Fold
20030// (fp_to_int (froundeven X)) -> fcvt X, rne
20031// (fp_to_int (ftrunc X)) -> fcvt X, rtz
20032// (fp_to_int (ffloor X)) -> fcvt X, rdn
20033// (fp_to_int (fceil X)) -> fcvt X, rup
20034// (fp_to_int (fround X)) -> fcvt X, rmm
20035// (fp_to_int (frint X)) -> fcvt X
20036static SDValue performFP_TO_INTCombine(SDNode *N,
20037 TargetLowering::DAGCombinerInfo &DCI,
20038 const RISCVSubtarget &Subtarget) {
20039 SelectionDAG &DAG = DCI.DAG;
20040 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20041 MVT XLenVT = Subtarget.getXLenVT();
20042
20043 SDValue Src = N->getOperand(Num: 0);
20044
20045 // Don't do this for strict-fp Src.
20046 if (Src->isStrictFPOpcode())
20047 return SDValue();
20048
20049 // Ensure the FP type is legal.
20050 if (!TLI.isTypeLegal(VT: Src.getValueType()))
20051 return SDValue();
20052
20053 // Don't do this for f16 with Zfhmin and not Zfh.
20054 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
20055 return SDValue();
20056
20057 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Src.getOpcode());
20058 // If the result is invalid, we didn't find a foldable instruction.
20059 if (FRM == RISCVFPRndMode::Invalid)
20060 return SDValue();
20061
20062 SDLoc DL(N);
20063 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
20064 EVT VT = N->getValueType(ResNo: 0);
20065
20066 if (VT.isVector() && TLI.isTypeLegal(VT)) {
20067 MVT SrcVT = Src.getSimpleValueType();
20068 MVT SrcContainerVT = SrcVT;
20069 MVT ContainerVT = VT.getSimpleVT();
20070 SDValue XVal = Src.getOperand(i: 0);
20071
20072 // For widening and narrowing conversions we just combine it into a
20073 // VFCVT_..._VL node, as there are no specific VFWCVT/VFNCVT VL nodes. They
20074 // end up getting lowered to their appropriate pseudo instructions based on
20075 // their operand types
20076 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits() * 2 ||
20077 VT.getScalarSizeInBits() * 2 < SrcVT.getScalarSizeInBits())
20078 return SDValue();
20079
20080 // Make fixed-length vectors scalable first
20081 if (SrcVT.isFixedLengthVector()) {
20082 SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT, Subtarget);
20083 XVal = convertToScalableVector(VT: SrcContainerVT, V: XVal, DAG, Subtarget);
20084 ContainerVT = getContainerForFixedLengthVector(VT: ContainerVT, Subtarget);
20085 }
20086
20087 auto [Mask, VL] =
20088 getDefaultVLOps(VecVT: SrcVT, ContainerVT: SrcContainerVT, DL, DAG, Subtarget);
20089
20090 SDValue FpToInt;
20091 if (FRM == RISCVFPRndMode::RTZ) {
20092 // Use the dedicated trunc static rounding mode if we're truncating so we
20093 // don't need to generate calls to fsrmi/fsrm
20094 unsigned Opc =
20095 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
20096 FpToInt = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: XVal, N2: Mask, N3: VL);
20097 } else {
20098 unsigned Opc =
20099 IsSigned ? RISCVISD::VFCVT_RM_X_F_VL : RISCVISD::VFCVT_RM_XU_F_VL;
20100 FpToInt = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: XVal, N2: Mask,
20101 N3: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), N4: VL);
20102 }
20103
20104 // If converted from fixed-length to scalable, convert back
20105 if (VT.isFixedLengthVector())
20106 FpToInt = convertFromScalableVector(VT, V: FpToInt, DAG, Subtarget);
20107
20108 return FpToInt;
20109 }
20110
20111 // Only handle XLen or i32 types. Other types narrower than XLen will
20112 // eventually be legalized to XLenVT.
20113 if (VT != MVT::i32 && VT != XLenVT)
20114 return SDValue();
20115
20116 unsigned Opc;
20117 if (VT == XLenVT)
20118 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
20119 else
20120 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
20121
20122 SDValue FpToInt = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Src.getOperand(i: 0),
20123 N2: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT));
20124 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FpToInt);
20125}
20126
20127// Fold
20128// (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
20129// (fp_to_int_sat (ftrunc X)) -> (select X == nan, 0, (fcvt X, rtz))
20130// (fp_to_int_sat (ffloor X)) -> (select X == nan, 0, (fcvt X, rdn))
20131// (fp_to_int_sat (fceil X)) -> (select X == nan, 0, (fcvt X, rup))
20132// (fp_to_int_sat (fround X)) -> (select X == nan, 0, (fcvt X, rmm))
20133// (fp_to_int_sat (frint X)) -> (select X == nan, 0, (fcvt X, dyn))
20134static SDValue performFP_TO_INT_SATCombine(SDNode *N,
20135 TargetLowering::DAGCombinerInfo &DCI,
20136 const RISCVSubtarget &Subtarget) {
20137 SelectionDAG &DAG = DCI.DAG;
20138 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
20139 MVT XLenVT = Subtarget.getXLenVT();
20140
20141 // Only handle XLen types. Other types narrower than XLen will eventually be
20142 // legalized to XLenVT.
20143 EVT DstVT = N->getValueType(ResNo: 0);
20144 if (DstVT != XLenVT)
20145 return SDValue();
20146
20147 SDValue Src = N->getOperand(Num: 0);
20148
20149 // Don't do this for strict-fp Src.
20150 if (Src->isStrictFPOpcode())
20151 return SDValue();
20152
20153 // Ensure the FP type is also legal.
20154 if (!TLI.isTypeLegal(VT: Src.getValueType()))
20155 return SDValue();
20156
20157 // Don't do this for f16 with Zfhmin and not Zfh.
20158 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
20159 return SDValue();
20160
20161 EVT SatVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
20162
20163 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Src.getOpcode());
20164 if (FRM == RISCVFPRndMode::Invalid)
20165 return SDValue();
20166
20167 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
20168
20169 unsigned Opc;
20170 if (SatVT == DstVT)
20171 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
20172 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
20173 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
20174 else
20175 return SDValue();
20176 // FIXME: Support other SatVTs by clamping before or after the conversion.
20177
20178 Src = Src.getOperand(i: 0);
20179
20180 SDLoc DL(N);
20181 SDValue FpToInt = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Src,
20182 N2: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT));
20183
20184 // fcvt.wu.* sign extends bit 31 on RV64. FP_TO_UINT_SAT expects to zero
20185 // extend.
20186 if (Opc == RISCVISD::FCVT_WU_RV64)
20187 FpToInt = DAG.getZeroExtendInReg(Op: FpToInt, DL, VT: MVT::i32);
20188
20189 // RISC-V FP-to-int conversions saturate to the destination register size, but
20190 // don't produce 0 for nan.
20191 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: DstVT);
20192 return DAG.getSelectCC(DL, LHS: Src, RHS: Src, True: ZeroInt, False: FpToInt, Cond: ISD::CondCode::SETUO);
20193}
20194
20195// Combine (bitreverse (bswap X)) to the BREV8 GREVI encoding if the type is
20196// smaller than XLenVT.
20197static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG,
20198 const RISCVSubtarget &Subtarget) {
20199 assert(Subtarget.hasStdExtZbkb() && "Unexpected extension");
20200
20201 SDValue Src = N->getOperand(Num: 0);
20202 if (Src.getOpcode() != ISD::BSWAP)
20203 return SDValue();
20204
20205 EVT VT = N->getValueType(ResNo: 0);
20206 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() ||
20207 !llvm::has_single_bit<uint32_t>(Value: VT.getSizeInBits()))
20208 return SDValue();
20209
20210 SDLoc DL(N);
20211 return DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT, Operand: Src.getOperand(i: 0));
20212}
20213
20214/// Matches a reverse shifted right EVL elements, or a vp.reverse.
20215// TODO: Remove vp.reverse
20216static auto m_ReverseEVL = [](auto X, auto EVL) {
20217 using namespace SDPatternMatch;
20218 return m_AnyOf(m_SpliceRight(m_OneUse(m_VectorReverse(X)), m_Poison(), EVL),
20219 m_Node(ISD::EXPERIMENTAL_VP_REVERSE, X, m_Value(), EVL));
20220};
20221
20222// TODO: A vlse.v is not necessarily faster than a vrgather.vv on all uarchs.
20223// Remove once a cost model driven transform is implemented in the loop
20224// vectorizer.
20225static SDValue performReverseEVLCombine(SDNode *N,
20226 TargetLowering::DAGCombinerInfo &DCI,
20227 const RISCVSubtarget &Subtarget) {
20228 SelectionDAG &DAG = DCI.DAG;
20229 // Fold:
20230 // vp.reverse(vp.load(ADDR, REVMASK, EVL), EVL)
20231 // -> vp.strided.load(ADDR, -1, MASK, EVL)
20232 //
20233 // splice.right(reverse(vp.load(ADDR, REVMASK, EVL)), poison, EVL)
20234 // -> vp.strided.load(ADDR, -1, MASK, EVL)
20235 //
20236 // vp.reverse(binop(vp.load(ADDR, REVMASK, EVL), splat), EVL)
20237 // -> binop(vp.strided.load(ADDR, -1, MASK, EVL), splat)
20238 using namespace SDPatternMatch;
20239 SDValue Op, EVL;
20240 if (!sd_match(N, P: m_ReverseEVL(m_Value(N&: Op), m_Value(N&: EVL))))
20241 return SDValue();
20242
20243 VPLoadSDNode *VPLoad = nullptr;
20244 // Find the single vp_load and check all other leaves are splats.
20245 SmallVector<SDValue> Worklist = {Op};
20246 while (!Worklist.empty()) {
20247 SDValue X = Worklist.pop_back_val();
20248 if (DAG.isSplatValue(V: X))
20249 continue;
20250 if (!X.hasOneUser())
20251 return SDValue();
20252 if (auto *VPL = dyn_cast<VPLoadSDNode>(Val&: X)) {
20253 if (VPLoad && VPLoad != VPL)
20254 return SDValue();
20255 VPLoad = VPL;
20256 } else if (DAG.getTargetLoweringInfo().isBinOp(Opcode: X.getOpcode()) &&
20257 X->getNumValues() == 1) {
20258 append_range(C&: Worklist, R: X->op_values());
20259 } else {
20260 return SDValue();
20261 }
20262 }
20263 if (!VPLoad)
20264 return SDValue();
20265
20266 EVT LoadVT = VPLoad->getValueType(ResNo: 0);
20267 // We do not have a strided_load version for masks, and the evl of vp.reverse
20268 // and vp.load should always be the same.
20269 if (!LoadVT.getVectorElementType().isByteSized() ||
20270 EVL != VPLoad->getVectorLength())
20271 return SDValue();
20272
20273 SDValue LoadMask = VPLoad->getMask();
20274 // If Mask is all ones, then load is unmasked and can be reversed.
20275 if (!isOneOrOneSplat(V: LoadMask)) {
20276 // If the mask is not all ones, we can reverse the load if the mask was also
20277 // reversed by a vp.reverse with the same EVL.
20278 SDValue OrigMask;
20279 if (!sd_match(N: LoadMask, P: m_ReverseEVL(m_Value(N&: OrigMask), m_Specific(N: EVL))))
20280 return SDValue();
20281 LoadMask = OrigMask;
20282 }
20283
20284 // Base = LoadAddr + (NumElem - 1) * ElemWidthByte
20285 SDLoc DL(N);
20286 MVT XLenVT = Subtarget.getXLenVT();
20287 SDValue NumElem = VPLoad->getVectorLength();
20288 uint64_t ElemWidthByte = VPLoad->getValueType(ResNo: 0).getScalarSizeInBits() / 8;
20289
20290 SDValue Temp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: NumElem,
20291 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
20292 SDValue Temp2 = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Temp1,
20293 N2: DAG.getConstant(Val: ElemWidthByte, DL, VT: XLenVT));
20294 SDValue Base = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: VPLoad->getBasePtr(), N2: Temp2);
20295 SDValue Stride = DAG.getSignedConstant(Val: -ElemWidthByte, DL, VT: XLenVT);
20296
20297 MachineFunction &MF = DAG.getMachineFunction();
20298 MachinePointerInfo PtrInfo(VPLoad->getAddressSpace());
20299 MachineMemOperand *MMO = MF.getMachineMemOperand(
20300 PtrInfo, F: VPLoad->getMemOperand()->getFlags(),
20301 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: VPLoad->getAlign());
20302
20303 SDValue Ret = DAG.getStridedLoadVP(
20304 VT: LoadVT, DL, Chain: VPLoad->getChain(), Ptr: Base, Stride, Mask: LoadMask,
20305 EVL: VPLoad->getVectorLength(), MMO, IsExpanding: VPLoad->isExpandingLoad());
20306
20307 DCI.CombineTo(N: VPLoad, Res0: Ret.getValue(R: 0), Res1: Ret.getValue(R: 1));
20308
20309 // Remove the top level reverse.
20310 (void)sd_match(N, P: m_ReverseEVL(m_Value(N&: Op), m_Value()));
20311 return Op;
20312}
20313
20314// Fold (i32 (bitcast (v4i8/v2i16 const_splat))) to a scalar i32 constant
20315// on RV64.
20316static SDValue performP_BITCASTCombine(SDNode *N, SelectionDAG &DAG,
20317 const RISCVSubtarget &Subtarget) {
20318 SDValue N0 = N->getOperand(Num: 0);
20319 EVT VT = N->getValueType(ResNo: 0);
20320 EVT SrcVT = N0.getValueType();
20321 if (!Subtarget.is64Bit() || VT != MVT::i32 ||
20322 (SrcVT != MVT::v4i8 && SrcVT != MVT::v2i16))
20323 return SDValue();
20324
20325 APInt SplatVal;
20326 if (!ISD::isConstantSplatVector(N: N0.getNode(), SplatValue&: SplatVal))
20327 return SDValue();
20328 return DAG.getConstant(Val: APInt::getSplat(NewLen: VT.getSizeInBits(), V: SplatVal),
20329 DL: SDLoc(N), VT);
20330}
20331
20332static SDValue performVP_STORECombine(SDNode *N, SelectionDAG &DAG,
20333 const RISCVSubtarget &Subtarget) {
20334 // Fold:
20335 // vp.store(vp.reverse(VAL, EVL), ADDR, REVMASK, EVL)
20336 // -> vp.strided.store(VAL, NEW_ADDR, -1, MASK, EVL)
20337 //
20338 // vp.store(splice.right(reverse(VAL), poison, EVL), ADDR, REVMASK, EVL)
20339 // -> vp.strided.store(VAL, NEW_ADDR, -1, MASK, EVL)
20340 auto *VPStore = cast<VPStoreSDNode>(Val: N);
20341 SDValue EVL = VPStore->getVectorLength();
20342
20343 using namespace SDPatternMatch;
20344 SDValue Val;
20345 if (!sd_match(N: VPStore->getValue(),
20346 P: m_OneUse(P: m_ReverseEVL(m_Value(N&: Val), m_Specific(N: EVL)))))
20347 return SDValue();
20348
20349 EVT ReverseVT = VPStore->getValue()->getValueType(ResNo: 0);
20350
20351 // We do not have a strided_store version for masks.
20352 if (!ReverseVT.getVectorElementType().isByteSized())
20353 return SDValue();
20354
20355 SDValue StoreMask = VPStore->getMask();
20356 // If Mask is all ones, then load is unmasked and can be reversed.
20357 if (!isOneOrOneSplat(V: StoreMask)) {
20358 // If the mask is not all ones, we can reverse the store if the mask was
20359 // also reversed by a vp.reverse with the same EVL.
20360 SDValue OrigMask;
20361 if (!sd_match(N: StoreMask, P: m_ReverseEVL(m_Value(N&: OrigMask), m_Specific(N: EVL))))
20362 return SDValue();
20363 StoreMask = OrigMask;
20364 }
20365
20366 // Base = StoreAddr + (NumElem - 1) * ElemWidthByte
20367 SDLoc DL(N);
20368 MVT XLenVT = Subtarget.getXLenVT();
20369 SDValue NumElem = VPStore->getVectorLength();
20370 uint64_t ElemWidthByte = ReverseVT.getScalarSizeInBits() / 8;
20371
20372 SDValue Temp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: NumElem,
20373 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
20374 SDValue Temp2 = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Temp1,
20375 N2: DAG.getConstant(Val: ElemWidthByte, DL, VT: XLenVT));
20376 SDValue Base =
20377 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: VPStore->getBasePtr(), N2: Temp2);
20378 SDValue Stride = DAG.getSignedConstant(Val: -ElemWidthByte, DL, VT: XLenVT);
20379
20380 MachineFunction &MF = DAG.getMachineFunction();
20381 MachinePointerInfo PtrInfo(VPStore->getAddressSpace());
20382 MachineMemOperand *MMO = MF.getMachineMemOperand(
20383 PtrInfo, F: VPStore->getMemOperand()->getFlags(),
20384 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: VPStore->getAlign());
20385
20386 return DAG.getStridedStoreVP(
20387 Chain: VPStore->getChain(), DL, Val, Ptr: Base, Offset: VPStore->getOffset(), Stride,
20388 Mask: StoreMask, EVL: VPStore->getVectorLength(), MemVT: VPStore->getMemoryVT(), MMO,
20389 AM: VPStore->getAddressingMode(), IsTruncating: VPStore->isTruncatingStore(),
20390 IsCompressing: VPStore->isCompressingStore());
20391}
20392
20393// Convert from one FMA opcode to another based on whether we are negating the
20394// multiply result and/or the accumulator.
20395// NOTE: Only supports RVV operations with VL.
20396static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc) {
20397 // Negating the multiply result changes ADD<->SUB and toggles 'N'.
20398 if (NegMul) {
20399 // clang-format off
20400 switch (Opcode) {
20401 default: llvm_unreachable("Unexpected opcode");
20402 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
20403 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
20404 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
20405 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
20406 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
20407 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
20408 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
20409 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
20410 }
20411 // clang-format on
20412 }
20413
20414 // Negating the accumulator changes ADD<->SUB.
20415 if (NegAcc) {
20416 // clang-format off
20417 switch (Opcode) {
20418 default: llvm_unreachable("Unexpected opcode");
20419 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
20420 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
20421 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
20422 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
20423 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
20424 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
20425 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
20426 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
20427 }
20428 // clang-format on
20429 }
20430
20431 return Opcode;
20432}
20433
20434static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG) {
20435 // Fold FNEG_VL into FMA opcodes.
20436 // The first operand of strict-fp is chain.
20437 bool IsStrict =
20438 DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(Opcode: N->getOpcode());
20439 unsigned Offset = IsStrict ? 1 : 0;
20440 SDValue A = N->getOperand(Num: 0 + Offset);
20441 SDValue B = N->getOperand(Num: 1 + Offset);
20442 SDValue C = N->getOperand(Num: 2 + Offset);
20443 SDValue Mask = N->getOperand(Num: 3 + Offset);
20444 SDValue VL = N->getOperand(Num: 4 + Offset);
20445
20446 auto invertIfNegative = [&Mask, &VL](SDValue &V) {
20447 if (V.getOpcode() == RISCVISD::FNEG_VL && V.getOperand(i: 1) == Mask &&
20448 V.getOperand(i: 2) == VL) {
20449 // Return the negated input.
20450 V = V.getOperand(i: 0);
20451 return true;
20452 }
20453
20454 return false;
20455 };
20456
20457 bool NegA = invertIfNegative(A);
20458 bool NegB = invertIfNegative(B);
20459 bool NegC = invertIfNegative(C);
20460
20461 // If no operands are negated, we're done.
20462 if (!NegA && !NegB && !NegC)
20463 return SDValue();
20464
20465 unsigned NewOpcode = negateFMAOpcode(Opcode: N->getOpcode(), NegMul: NegA != NegB, NegAcc: NegC);
20466 if (IsStrict)
20467 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(N), VTList: N->getVTList(),
20468 Ops: {N->getOperand(Num: 0), A, B, C, Mask, VL});
20469 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: A, N2: B, N3: C, N4: Mask,
20470 N5: VL);
20471}
20472
20473static SDValue performVFMADD_VLCombine(SDNode *N,
20474 TargetLowering::DAGCombinerInfo &DCI,
20475 const RISCVSubtarget &Subtarget) {
20476 SelectionDAG &DAG = DCI.DAG;
20477
20478 if (SDValue V = combineVFMADD_VLWithVFNEG_VL(N, DAG))
20479 return V;
20480
20481 // FIXME: Ignore strict opcodes for now.
20482 if (DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(Opcode: N->getOpcode()))
20483 return SDValue();
20484
20485 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
20486}
20487
20488static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
20489 const RISCVSubtarget &Subtarget) {
20490 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode");
20491
20492 EVT VT = N->getValueType(ResNo: 0);
20493
20494 if (VT != Subtarget.getXLenVT())
20495 return SDValue();
20496
20497 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
20498 return SDValue();
20499 uint64_t ShAmt = N->getConstantOperandVal(Num: 1);
20500
20501 SDValue N0 = N->getOperand(Num: 0);
20502
20503 // Combine (sra (sext_inreg (shl X, C1), iX), C2) ->
20504 // (sra (shl X, C1+(XLen-iX)), C2+(XLen-iX)) so it gets selected as SLLI+SRAI.
20505 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && N0.hasOneUse()) {
20506 unsigned ExtSize =
20507 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
20508 if (ShAmt < ExtSize && N0.getOperand(i: 0).getOpcode() == ISD::SHL &&
20509 N0.getOperand(i: 0).hasOneUse() &&
20510 isa<ConstantSDNode>(Val: N0.getOperand(i: 0).getOperand(i: 1))) {
20511 uint64_t LShAmt = N0.getOperand(i: 0).getConstantOperandVal(i: 1);
20512 if (LShAmt < ExtSize) {
20513 unsigned Size = VT.getSizeInBits();
20514 SDLoc ShlDL(N0.getOperand(i: 0));
20515 SDValue Shl =
20516 DAG.getNode(Opcode: ISD::SHL, DL: ShlDL, VT, N1: N0.getOperand(i: 0).getOperand(i: 0),
20517 N2: DAG.getConstant(Val: LShAmt + (Size - ExtSize), DL: ShlDL, VT));
20518 SDLoc DL(N);
20519 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Shl,
20520 N2: DAG.getConstant(Val: ShAmt + (Size - ExtSize), DL, VT));
20521 }
20522 }
20523 }
20524
20525 if (ShAmt > 32 || VT != MVT::i64)
20526 return SDValue();
20527
20528 // Combine (sra (shl X, 32), 32 - C) -> (shl (sext_inreg X, i32), C)
20529 // FIXME: Should this be a generic combine? There's a similar combine on X86.
20530 //
20531 // Also try these folds where an add or sub is in the middle.
20532 // (sra (add (shl X, 32), C1), 32 - C) -> (shl (sext_inreg (add X, C1), C)
20533 // (sra (sub C1, (shl X, 32)), 32 - C) -> (shl (sext_inreg (sub C1, X), C)
20534 SDValue Shl;
20535 ConstantSDNode *AddC = nullptr;
20536
20537 // We might have an ADD or SUB between the SRA and SHL.
20538 bool IsAdd = N0.getOpcode() == ISD::ADD;
20539 if ((IsAdd || N0.getOpcode() == ISD::SUB)) {
20540 // Other operand needs to be a constant we can modify.
20541 AddC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: IsAdd ? 1 : 0));
20542 if (!AddC)
20543 return SDValue();
20544
20545 // AddC needs to have at least 32 trailing zeros.
20546 if (llvm::countr_zero(Val: AddC->getZExtValue()) < 32)
20547 return SDValue();
20548
20549 // All users should be a shift by constant less than or equal to 32. This
20550 // ensures we'll do this optimization for each of them to produce an
20551 // add/sub+sext_inreg they can all share.
20552 for (SDNode *U : N0->users()) {
20553 if (U->getOpcode() != ISD::SRA ||
20554 !isa<ConstantSDNode>(Val: U->getOperand(Num: 1)) ||
20555 U->getConstantOperandVal(Num: 1) > 32)
20556 return SDValue();
20557 }
20558
20559 Shl = N0.getOperand(i: IsAdd ? 0 : 1);
20560 } else {
20561 // Not an ADD or SUB.
20562 Shl = N0;
20563 }
20564
20565 // Look for a shift left by 32.
20566 if (Shl.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: Shl.getOperand(i: 1)) ||
20567 Shl.getConstantOperandVal(i: 1) != 32)
20568 return SDValue();
20569
20570 // We if we didn't look through an add/sub, then the shl should have one use.
20571 // If we did look through an add/sub, the sext_inreg we create is free so
20572 // we're only creating 2 new instructions. It's enough to only remove the
20573 // original sra+add/sub.
20574 if (!AddC && !Shl.hasOneUse())
20575 return SDValue();
20576
20577 SDLoc DL(N);
20578 SDValue In = Shl.getOperand(i: 0);
20579
20580 // If we looked through an ADD or SUB, we need to rebuild it with the shifted
20581 // constant.
20582 if (AddC) {
20583 SDValue ShiftedAddC =
20584 DAG.getConstant(Val: AddC->getZExtValue() >> 32, DL, VT: MVT::i64);
20585 if (IsAdd)
20586 In = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: In, N2: ShiftedAddC);
20587 else
20588 In = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: ShiftedAddC, N2: In);
20589 }
20590
20591 SDValue SExt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: In,
20592 N2: DAG.getValueType(MVT::i32));
20593 if (ShAmt == 32)
20594 return SExt;
20595
20596 return DAG.getNode(
20597 Opcode: ISD::SHL, DL, VT: MVT::i64, N1: SExt,
20598 N2: DAG.getConstant(Val: 32 - ShAmt, DL, VT: MVT::i64));
20599}
20600
20601// Invert (and/or (set cc X, Y), (xor Z, 1)) to (or/and (set !cc X, Y)), Z) if
20602// the result is used as the condition of a br_cc or select_cc we can invert,
20603// inverting the setcc is free, and Z is 0/1. Caller will invert the
20604// br_cc/select_cc.
20605static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG) {
20606 bool IsAnd = Cond.getOpcode() == ISD::AND;
20607 if (!IsAnd && Cond.getOpcode() != ISD::OR)
20608 return SDValue();
20609
20610 if (!Cond.hasOneUse())
20611 return SDValue();
20612
20613 SDValue Setcc = Cond.getOperand(i: 0);
20614 SDValue Xor = Cond.getOperand(i: 1);
20615 // Canonicalize setcc to LHS.
20616 if (Setcc.getOpcode() != ISD::SETCC)
20617 std::swap(a&: Setcc, b&: Xor);
20618 // LHS should be a setcc and RHS should be an xor.
20619 if (Setcc.getOpcode() != ISD::SETCC || !Setcc.hasOneUse() ||
20620 Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
20621 return SDValue();
20622
20623 // If the condition is an And, SimplifyDemandedBits may have changed
20624 // (xor Z, 1) to (not Z).
20625 SDValue Xor1 = Xor.getOperand(i: 1);
20626 if (!isOneConstant(V: Xor1) && !(IsAnd && isAllOnesConstant(V: Xor1)))
20627 return SDValue();
20628
20629 EVT VT = Cond.getValueType();
20630 SDValue Xor0 = Xor.getOperand(i: 0);
20631
20632 // The LHS of the xor needs to be 0/1.
20633 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
20634 if (!DAG.MaskedValueIsZero(Op: Xor0, Mask))
20635 return SDValue();
20636
20637 // We can only invert integer setccs.
20638 EVT SetCCOpVT = Setcc.getOperand(i: 0).getValueType();
20639 if (!SetCCOpVT.isScalarInteger())
20640 return SDValue();
20641
20642 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Setcc.getOperand(i: 2))->get();
20643 if (ISD::isIntEqualitySetCC(Code: CCVal)) {
20644 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: SetCCOpVT);
20645 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT, LHS: Setcc.getOperand(i: 0),
20646 RHS: Setcc.getOperand(i: 1), Cond: CCVal);
20647 } else if (CCVal == ISD::SETLT && isNullConstant(V: Setcc.getOperand(i: 0))) {
20648 // Invert (setlt 0, X) by converting to (setlt X, 1).
20649 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT, LHS: Setcc.getOperand(i: 1),
20650 RHS: DAG.getConstant(Val: 1, DL: SDLoc(Setcc), VT), Cond: CCVal);
20651 } else if (CCVal == ISD::SETLT && isOneConstant(V: Setcc.getOperand(i: 1))) {
20652 // (setlt X, 1) by converting to (setlt 0, X).
20653 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT,
20654 LHS: DAG.getConstant(Val: 0, DL: SDLoc(Setcc), VT),
20655 RHS: Setcc.getOperand(i: 0), Cond: CCVal);
20656 } else
20657 return SDValue();
20658
20659 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
20660 return DAG.getNode(Opcode: Opc, DL: SDLoc(Cond), VT, N1: Setcc, N2: Xor.getOperand(i: 0));
20661}
20662
20663// Perform common combines for BR_CC and SELECT_CC conditions.
20664static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
20665 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
20666 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
20667
20668 // As far as arithmetic right shift always saves the sign,
20669 // shift can be omitted.
20670 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
20671 // setge (sra X, N), 0 -> setge X, 0
20672 if (isNullConstant(V: RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
20673 LHS.getOpcode() == ISD::SRA) {
20674 LHS = LHS.getOperand(i: 0);
20675 return true;
20676 }
20677
20678 if (!ISD::isIntEqualitySetCC(Code: CCVal))
20679 return false;
20680
20681 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
20682 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
20683 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(V: RHS) &&
20684 LHS.getOperand(i: 0).getValueType() == Subtarget.getXLenVT()) {
20685 // If we're looking for eq 0 instead of ne 0, we need to invert the
20686 // condition.
20687 bool Invert = CCVal == ISD::SETEQ;
20688 CCVal = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
20689 if (Invert)
20690 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
20691
20692 RHS = LHS.getOperand(i: 1);
20693 LHS = LHS.getOperand(i: 0);
20694 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
20695
20696 CC = DAG.getCondCode(Cond: CCVal);
20697 return true;
20698 }
20699
20700 auto isFoldableXorEq = [&DAG](SDValue LHS, SDValue RHS) -> bool {
20701 if (LHS.getOpcode() != ISD::XOR || !isNullConstant(V: RHS))
20702 return false;
20703
20704 // If XOR cannot be an XORI, allow the fold.
20705 const auto *XorCnst = dyn_cast<ConstantSDNode>(Val: LHS.getOperand(i: 1));
20706 if (!XorCnst || !isInt<12>(x: XorCnst->getSExtValue()))
20707 return true;
20708
20709 // Fold (X(i1) ^ 1) == 0 -> X != 0
20710 SDValue VarOp = LHS.getOperand(i: 0);
20711 const APInt Mask = APInt::getBitsSetFrom(numBits: VarOp.getValueSizeInBits(), loBit: 1);
20712 if (XorCnst->getSExtValue() == 1 && DAG.MaskedValueIsZero(Op: VarOp, Mask))
20713 return true;
20714
20715 // If the Xor is only used by select or br_cc, allow the fold.
20716 return all_of(Range: LHS->users(), P: [](const SDNode *UserNode) {
20717 const unsigned Opcode = UserNode->getOpcode();
20718 return Opcode == RISCVISD::SELECT_CC || Opcode == RISCVISD::BR_CC;
20719 });
20720 };
20721 // Fold ((xor X, Y), 0, eq/ne) -> (X, Y, eq/ne)
20722 if (isFoldableXorEq(LHS, RHS)) {
20723 RHS = LHS.getOperand(i: 1);
20724 LHS = LHS.getOperand(i: 0);
20725 return true;
20726 }
20727 // Fold ((sext (xor X, C)), 0, eq/ne) -> ((sext(X), C, eq/ne)
20728 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG) {
20729 const SDValue LHS0 = LHS.getOperand(i: 0);
20730 if (isFoldableXorEq(LHS0, RHS) && isa<ConstantSDNode>(Val: LHS0.getOperand(i: 1))) {
20731 // SEXT(XOR(X, Y)) -> XOR(SEXT(X), SEXT(Y)))
20732 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: LHS.getValueType(),
20733 N1: LHS0.getOperand(i: 1), N2: LHS.getOperand(i: 1));
20734 LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: LHS.getValueType(),
20735 N1: LHS0.getOperand(i: 0), N2: LHS.getOperand(i: 1));
20736 return true;
20737 }
20738 }
20739
20740 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, XLen-1-C), 0, ge/lt)
20741 if (isNullConstant(V: RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
20742 LHS.getOperand(i: 1).getOpcode() == ISD::Constant) {
20743 SDValue LHS0 = LHS.getOperand(i: 0);
20744 if (LHS0.getOpcode() == ISD::AND &&
20745 LHS0.getOperand(i: 1).getOpcode() == ISD::Constant) {
20746 uint64_t Mask = LHS0.getConstantOperandVal(i: 1);
20747 uint64_t ShAmt = LHS.getConstantOperandVal(i: 1);
20748 if (isPowerOf2_64(Value: Mask) && Log2_64(Value: Mask) == ShAmt) {
20749 // XAndesPerf supports branch on test bit.
20750 if (Subtarget.hasVendorXAndesPerf()) {
20751 LHS =
20752 DAG.getNode(Opcode: ISD::AND, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
20753 N2: DAG.getConstant(Val: Mask, DL, VT: LHS.getValueType()));
20754 return true;
20755 }
20756
20757 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
20758 CC = DAG.getCondCode(Cond: CCVal);
20759
20760 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
20761 LHS = LHS0.getOperand(i: 0);
20762 if (ShAmt != 0)
20763 LHS =
20764 DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
20765 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
20766 return true;
20767 }
20768 }
20769 }
20770
20771 // (X, 1, setne) -> // (X, 0, seteq) if we can prove X is 0/1.
20772 // This can occur when legalizing some floating point comparisons.
20773 APInt Mask = APInt::getBitsSetFrom(numBits: LHS.getValueSizeInBits(), loBit: 1);
20774 if (isOneConstant(V: RHS) && DAG.MaskedValueIsZero(Op: LHS, Mask)) {
20775 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
20776 CC = DAG.getCondCode(Cond: CCVal);
20777 RHS = DAG.getConstant(Val: 0, DL, VT: LHS.getValueType());
20778 return true;
20779 }
20780
20781 if (isNullConstant(V: RHS)) {
20782 if (SDValue NewCond = tryDemorganOfBooleanCondition(Cond: LHS, DAG)) {
20783 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
20784 CC = DAG.getCondCode(Cond: CCVal);
20785 LHS = NewCond;
20786 return true;
20787 }
20788 }
20789
20790 return false;
20791}
20792
20793// Fold
20794// (select C, (add Y, X), Y) -> (add Y, (select C, X, 0)).
20795// (select C, (sub Y, X), Y) -> (sub Y, (select C, X, 0)).
20796// (select C, (or Y, X), Y) -> (or Y, (select C, X, 0)).
20797// (select C, (xor Y, X), Y) -> (xor Y, (select C, X, 0)).
20798// (select C, (rotl Y, X), Y) -> (rotl Y, (select C, X, 0)).
20799// (select C, (rotr Y, X), Y) -> (rotr Y, (select C, X, 0)).
20800static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
20801 SDValue TrueVal, SDValue FalseVal,
20802 bool Swapped) {
20803 bool Commutative = true;
20804 unsigned Opc = TrueVal.getOpcode();
20805 switch (Opc) {
20806 default:
20807 return SDValue();
20808 case ISD::SHL:
20809 case ISD::SRA:
20810 case ISD::SRL:
20811 case ISD::SUB:
20812 case ISD::ROTL:
20813 case ISD::ROTR:
20814 Commutative = false;
20815 break;
20816 case ISD::ADD:
20817 case ISD::OR:
20818 case ISD::XOR:
20819 case ISD::UMIN:
20820 case ISD::UMAX:
20821 break;
20822 }
20823
20824 if (!TrueVal.hasOneUse())
20825 return SDValue();
20826
20827 unsigned OpToFold;
20828 if (FalseVal == TrueVal.getOperand(i: 0))
20829 OpToFold = 0;
20830 else if (Commutative && FalseVal == TrueVal.getOperand(i: 1))
20831 OpToFold = 1;
20832 else
20833 return SDValue();
20834
20835 EVT VT = N->getValueType(ResNo: 0);
20836 SDLoc DL(N);
20837 SDValue OtherOp = TrueVal.getOperand(i: 1 - OpToFold);
20838 EVT OtherOpVT = OtherOp.getValueType();
20839 SDValue IdentityOperand =
20840 DAG.getIdentityElement(Opcode: Opc, DL, VT: OtherOpVT, Flags: N->getFlags());
20841 if (!Commutative)
20842 IdentityOperand = DAG.getConstant(Val: 0, DL, VT: OtherOpVT);
20843 assert(IdentityOperand && "No identity operand!");
20844
20845 if (Swapped)
20846 std::swap(a&: OtherOp, b&: IdentityOperand);
20847 SDValue NewSel =
20848 DAG.getSelect(DL, VT: OtherOpVT, Cond: N->getOperand(Num: 0), LHS: OtherOp, RHS: IdentityOperand);
20849 return DAG.getNode(Opcode: TrueVal.getOpcode(), DL, VT, N1: FalseVal, N2: NewSel);
20850}
20851
20852// This tries to get rid of `select` and `icmp` that are being used to handle
20853// `Targets` that do not support `cttz(0)`/`ctlz(0)`.
20854static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG) {
20855 SDValue Cond = N->getOperand(Num: 0);
20856
20857 // This represents either CTTZ or CTLZ instruction.
20858 SDValue CountZeroes;
20859
20860 SDValue ValOnZero;
20861
20862 if (Cond.getOpcode() != ISD::SETCC)
20863 return SDValue();
20864
20865 if (!isNullConstant(V: Cond->getOperand(Num: 1)))
20866 return SDValue();
20867
20868 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond->getOperand(Num: 2))->get();
20869 if (CCVal == ISD::CondCode::SETEQ) {
20870 CountZeroes = N->getOperand(Num: 2);
20871 ValOnZero = N->getOperand(Num: 1);
20872 } else if (CCVal == ISD::CondCode::SETNE) {
20873 CountZeroes = N->getOperand(Num: 1);
20874 ValOnZero = N->getOperand(Num: 2);
20875 } else {
20876 return SDValue();
20877 }
20878
20879 if (CountZeroes.getOpcode() == ISD::TRUNCATE ||
20880 CountZeroes.getOpcode() == ISD::ZERO_EXTEND)
20881 CountZeroes = CountZeroes.getOperand(i: 0);
20882
20883 if (CountZeroes.getOpcode() != ISD::CTTZ &&
20884 CountZeroes.getOpcode() != ISD::CTTZ_ZERO_POISON &&
20885 CountZeroes.getOpcode() != ISD::CTLZ &&
20886 CountZeroes.getOpcode() != ISD::CTLZ_ZERO_POISON)
20887 return SDValue();
20888
20889 if (!isNullConstant(V: ValOnZero))
20890 return SDValue();
20891
20892 SDValue CountZeroesArgument = CountZeroes->getOperand(Num: 0);
20893 if (Cond->getOperand(Num: 0) != CountZeroesArgument)
20894 return SDValue();
20895
20896 unsigned BitWidth = CountZeroes.getValueSizeInBits();
20897 if (!isPowerOf2_32(Value: BitWidth))
20898 return SDValue();
20899
20900 if (CountZeroes.getOpcode() == ISD::CTTZ_ZERO_POISON) {
20901 CountZeroes = DAG.getNode(Opcode: ISD::CTTZ, DL: SDLoc(CountZeroes),
20902 VT: CountZeroes.getValueType(), Operand: CountZeroesArgument);
20903 } else if (CountZeroes.getOpcode() == ISD::CTLZ_ZERO_POISON) {
20904 CountZeroes = DAG.getNode(Opcode: ISD::CTLZ, DL: SDLoc(CountZeroes),
20905 VT: CountZeroes.getValueType(), Operand: CountZeroesArgument);
20906 }
20907
20908 SDValue BitWidthMinusOne =
20909 DAG.getConstant(Val: BitWidth - 1, DL: SDLoc(N), VT: CountZeroes.getValueType());
20910
20911 auto AndNode = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CountZeroes.getValueType(),
20912 N1: CountZeroes, N2: BitWidthMinusOne);
20913 return DAG.getZExtOrTrunc(Op: AndNode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
20914}
20915
20916static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG,
20917 const RISCVSubtarget &Subtarget) {
20918 SDValue Cond = N->getOperand(Num: 0);
20919 SDValue True = N->getOperand(Num: 1);
20920 SDValue False = N->getOperand(Num: 2);
20921 SDLoc DL(N);
20922 EVT VT = N->getValueType(ResNo: 0);
20923 EVT CondVT = Cond.getValueType();
20924
20925 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
20926 return SDValue();
20927
20928 // Replace (setcc eq (and x, C)) with (setcc ne (and x, C))) to generate
20929 // BEXTI, where C is power of 2.
20930 if (Subtarget.hasBEXTILike() && VT.isScalarInteger() &&
20931 (Subtarget.hasCZEROLike() || Subtarget.hasVendorXTHeadCondMov())) {
20932 SDValue LHS = Cond.getOperand(i: 0);
20933 SDValue RHS = Cond.getOperand(i: 1);
20934 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
20935 if (CC == ISD::SETEQ && LHS.getOpcode() == ISD::AND &&
20936 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) && isNullConstant(V: RHS)) {
20937 const APInt &MaskVal = LHS.getConstantOperandAPInt(i: 1);
20938 if (MaskVal.isPowerOf2() && !MaskVal.isSignedIntN(N: 12))
20939 return DAG.getSelect(DL, VT,
20940 Cond: DAG.getSetCC(DL, VT: CondVT, LHS, RHS, Cond: ISD::SETNE),
20941 LHS: False, RHS: True);
20942 }
20943 }
20944 return SDValue();
20945}
20946
20947static SDValue
20948canonicalizeVSelectTrueToOneUse(SDNode *N, SelectionDAG &DAG,
20949 const RISCVSubtarget &Subtarget) {
20950 SDValue CC = N->getOperand(Num: 0);
20951 SDValue TrueVal = N->getOperand(Num: 1);
20952 SDValue FalseVal = N->getOperand(Num: 2);
20953
20954 if (CC.getOpcode() != ISD::SETCC || !CC.hasOneUse() || TrueVal.hasOneUse() ||
20955 !FalseVal.hasOneUse())
20956 return SDValue();
20957
20958 // Only handles ISD::SETEQ and ISD::SETNE; no extra RVV introduced.
20959 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CC.getOperand(i: 2))->get();
20960 if (!isIntEqualitySetCC(Code: CCVal))
20961 return SDValue();
20962
20963 if (DAG.isSplatValue(V: TrueVal) || DAG.isSplatValue(V: FalseVal) ||
20964 TrueVal.getOpcode() == ISD::SPLAT_VECTOR_PARTS ||
20965 FalseVal.getOpcode() == ISD::SPLAT_VECTOR_PARTS ||
20966 TrueVal.getOpcode() == RISCVISD::VMV_V_X_VL ||
20967 FalseVal.getOpcode() == RISCVISD::VMV_V_X_VL)
20968 return SDValue();
20969
20970 SDLoc DL(N);
20971 EVT CVT = CC.getValueType();
20972 SDValue InvertedCC = DAG.getSetCC(DL, VT: CVT, LHS: CC.getOperand(i: 0), RHS: CC.getOperand(i: 1),
20973 Cond: ISD::getSetCCInverse(Operation: CCVal, Type: CVT));
20974 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: N->getValueType(ResNo: 0), N1: InvertedCC, N2: FalseVal,
20975 N3: TrueVal);
20976}
20977
20978static bool matchSelectAddSub(SDValue TrueVal, SDValue FalseVal, bool &SwapCC) {
20979 if (!TrueVal.hasOneUse() || !FalseVal.hasOneUse())
20980 return false;
20981
20982 SwapCC = false;
20983 if (TrueVal.getOpcode() == ISD::SUB && FalseVal.getOpcode() == ISD::ADD) {
20984 std::swap(a&: TrueVal, b&: FalseVal);
20985 SwapCC = true;
20986 }
20987
20988 if (TrueVal.getOpcode() != ISD::ADD || FalseVal.getOpcode() != ISD::SUB)
20989 return false;
20990
20991 SDValue A = FalseVal.getOperand(i: 0);
20992 SDValue B = FalseVal.getOperand(i: 1);
20993 // Add is commutative, so check both orders
20994 return ((TrueVal.getOperand(i: 0) == A && TrueVal.getOperand(i: 1) == B) ||
20995 (TrueVal.getOperand(i: 1) == A && TrueVal.getOperand(i: 0) == B));
20996}
20997
20998static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG,
20999 const RISCVSubtarget &Subtarget) {
21000 SDLoc DL(N);
21001 EVT VT = N->getValueType(ResNo: 0);
21002 SDValue CC = N->getOperand(Num: 0);
21003 SDValue TrueVal = N->getOperand(Num: 1);
21004 SDValue FalseVal = N->getOperand(Num: 2);
21005
21006 // Convert (vselect CC, true, false) to (vselect InvertCC, false, true) when
21007 // false has one use and true has multiple use.
21008 // It relies on RISCVVectorPeephole.cpp foldVMergeToMask to eliminate
21009 // vmerge.vv
21010 if (SDValue V = canonicalizeVSelectTrueToOneUse(N, DAG, Subtarget))
21011 return V;
21012
21013 // Convert vselect CC, (add a, b), (sub a, b) to add a, (vselect CC, -b, b).
21014 // This allows us match a vadd.vv fed by a masked vrsub, which reduces
21015 // register pressure over the add followed by masked vsub sequence.
21016 bool SwapCC;
21017 if (!matchSelectAddSub(TrueVal, FalseVal, SwapCC))
21018 return SDValue();
21019
21020 SDValue Sub = SwapCC ? TrueVal : FalseVal;
21021 SDValue A = Sub.getOperand(i: 0);
21022 SDValue B = Sub.getOperand(i: 1);
21023
21024 // Arrange the select such that we can match a masked
21025 // vrsub.vi to perform the conditional negate
21026 SDValue NegB = DAG.getNegative(Val: B, DL, VT);
21027 if (!SwapCC)
21028 CC = DAG.getLogicalNOT(DL, Val: CC, VT: CC->getValueType(ResNo: 0));
21029 SDValue NewB = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CC, N2: NegB, N3: B);
21030 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: NewB);
21031}
21032
21033// Fold (iN (select (src >u ((1 << N) - 1)), sext(src >s -1), trunc(src))) to
21034// USATI. This pattern saturates a signed value to an unsigned N-bit range
21035// [0, 2^N-1]:
21036// - If src < 0: result = 0
21037// (via the inner comparison src > -1 = false, sext to 0)
21038// - If src > ((1 << C) - 1): result = all 1s
21039// (via sext(true) = -1 = 0xFF...)
21040// - Otherwise: result = src (via trunc(src))
21041// The outer comparison is unsigned, so negative values appear as large
21042// unsigned values and trigger the saturation to MaxVal path, where the
21043// inner signed comparison then produces 0.
21044// TODO: Support (select (src <=u ((1 << C) - 1)), trunc(src), sext(src >s -1)).
21045static SDValue foldSelectToUSATI(SDNode *N, SelectionDAG &DAG,
21046 const RISCVSubtarget &Subtarget) {
21047 if (!Subtarget.hasStdExtP())
21048 return SDValue();
21049
21050 EVT VT = N->getValueType(ResNo: 0);
21051 MVT XLenVT = Subtarget.getXLenVT();
21052
21053 // Only support scalar integer types smaller than XLenVT
21054 if (!VT.isScalarInteger() || VT.bitsGE(VT: XLenVT))
21055 return SDValue();
21056
21057 unsigned SatWidth = VT.getSizeInBits();
21058 uint64_t MaxVal = (1ULL << SatWidth) - 1;
21059
21060 using namespace SDPatternMatch;
21061
21062 SDValue Src, InnerSetCC, FalseSrc;
21063 if (!sd_match(N, P: m_Select(Cond: m_SetCC(LHS: m_Value(N&: Src), RHS: m_SpecificInt(V: MaxVal),
21064 CC: m_SpecificCondCode(CC: ISD::SETUGT)),
21065 T: m_SExt(Op: m_Value(N&: InnerSetCC)),
21066 F: m_Trunc(Op: m_Value(N&: FalseSrc)))))
21067 return SDValue();
21068
21069 // Src can't be larger than XLenVT.
21070 if (Src.getValueType().bitsGT(VT: XLenVT))
21071 return SDValue();
21072
21073 // Check inner setcc: src > -1 (signed comparison)
21074 if (!sd_match(N: InnerSetCC,
21075 P: m_SpecificVT(RefVT: MVT::i1, P: m_SetCC(LHS: m_Specific(N: Src), RHS: m_AllOnes(),
21076 CC: m_SpecificCondCode(CC: ISD::SETGT)))))
21077 return SDValue();
21078
21079 // It's possible that the input to the setccs is also a truncate, in that
21080 // case the input to the truncate on the select's false operand may be the
21081 // same as the input to this setcc truncate. We need to look through the
21082 // setcc truncate to make sure CmpSrc and FalseSrc come from the same value.
21083 SDValue CmpSrc = Src;
21084 if (CmpSrc != FalseSrc && CmpSrc.getOpcode() == ISD::TRUNCATE)
21085 CmpSrc = CmpSrc.getOperand(i: 0);
21086
21087 if (CmpSrc != FalseSrc)
21088 return SDValue();
21089
21090 // We found a USATI pattern.
21091 SDLoc DL(N);
21092 Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: XLenVT, Operand: Src);
21093 SDValue USATI = DAG.getNode(Opcode: RISCVISD::USATI, DL, VT: XLenVT, N1: Src,
21094 N2: DAG.getTargetConstant(Val: SatWidth, DL, VT: XLenVT));
21095 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: USATI);
21096}
21097
21098static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
21099 const RISCVSubtarget &Subtarget) {
21100 if (SDValue Folded = foldSelectOfCTTZOrCTLZ(N, DAG))
21101 return Folded;
21102
21103 if (SDValue V = foldSelectToUSATI(N, DAG, Subtarget))
21104 return V;
21105
21106 if (SDValue V = useInversedSetcc(N, DAG, Subtarget))
21107 return V;
21108
21109 if (Subtarget.hasConditionalMoveFusion())
21110 return SDValue();
21111
21112 SDValue TrueVal = N->getOperand(Num: 1);
21113 SDValue FalseVal = N->getOperand(Num: 2);
21114 if (SDValue V = tryFoldSelectIntoOp(N, DAG, TrueVal, FalseVal, /*Swapped*/false))
21115 return V;
21116 return tryFoldSelectIntoOp(N, DAG, TrueVal: FalseVal, FalseVal: TrueVal, /*Swapped*/true);
21117}
21118
21119/// If we have a build_vector where each lane is binop X, C, where C
21120/// is a constant (but not necessarily the same constant on all lanes),
21121/// form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).
21122/// We assume that materializing a constant build vector will be no more
21123/// expensive that performing O(n) binops.
21124static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
21125 const RISCVSubtarget &Subtarget,
21126 const RISCVTargetLowering &TLI) {
21127 SDLoc DL(N);
21128 EVT VT = N->getValueType(ResNo: 0);
21129
21130 assert(!VT.isScalableVector() && "unexpected build vector");
21131
21132 if (VT.getVectorNumElements() == 1)
21133 return SDValue();
21134
21135 const unsigned Opcode = N->op_begin()->getNode()->getOpcode();
21136 if (!TLI.isBinOp(Opcode))
21137 return SDValue();
21138
21139 if (!TLI.isOperationLegalOrCustom(Op: Opcode, VT) || !TLI.isTypeLegal(VT))
21140 return SDValue();
21141
21142 // This BUILD_VECTOR involves an implicit truncation, and sinking
21143 // truncates through binops is non-trivial.
21144 if (N->op_begin()->getValueType() != VT.getVectorElementType())
21145 return SDValue();
21146
21147 SmallVector<SDValue> LHSOps;
21148 SmallVector<SDValue> RHSOps;
21149 for (SDValue Op : N->ops()) {
21150 if (Op.isUndef()) {
21151 // We can't form a divide or remainder from undef.
21152 if (!DAG.isSafeToSpeculativelyExecute(Opcode))
21153 return SDValue();
21154
21155 LHSOps.push_back(Elt: Op);
21156 RHSOps.push_back(Elt: Op);
21157 continue;
21158 }
21159
21160 // TODO: We can handle operations which have an neutral rhs value
21161 // (e.g. x + 0, a * 1 or a << 0), but we then have to keep track
21162 // of profit in a more explicit manner.
21163 if (Op.getOpcode() != Opcode || !Op.hasOneUse())
21164 return SDValue();
21165
21166 LHSOps.push_back(Elt: Op.getOperand(i: 0));
21167 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
21168 !isa<ConstantFPSDNode>(Val: Op.getOperand(i: 1)))
21169 return SDValue();
21170 // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
21171 // have different LHS and RHS types.
21172 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
21173 return SDValue();
21174
21175 RHSOps.push_back(Elt: Op.getOperand(i: 1));
21176 }
21177
21178 return DAG.getNode(Opcode, DL, VT, N1: DAG.getBuildVector(VT, DL, Ops: LHSOps),
21179 N2: DAG.getBuildVector(VT, DL, Ops: RHSOps));
21180}
21181
21182static MVT getQDOTXResultType(MVT OpVT) {
21183 ElementCount OpEC = OpVT.getVectorElementCount();
21184 assert(OpEC.isKnownMultipleOf(4) && OpVT.getVectorElementType() == MVT::i8);
21185 return MVT::getVectorVT(VT: MVT::i32, EC: OpEC.divideCoefficientBy(RHS: 4));
21186}
21187
21188/// Given fixed length vectors A and B with equal element types, but possibly
21189/// different number of elements, return A + B where either A or B is zero
21190/// padded to the larger number of elements.
21191static SDValue getZeroPaddedAdd(const SDLoc &DL, SDValue A, SDValue B,
21192 SelectionDAG &DAG) {
21193 // NOTE: Manually doing the extract/add/insert scheme produces
21194 // significantly better codegen than the naive pad with zeros
21195 // and add scheme.
21196 EVT AVT = A.getValueType();
21197 EVT BVT = B.getValueType();
21198 assert(AVT.getVectorElementType() == BVT.getVectorElementType());
21199 if (AVT.getVectorMinNumElements() > BVT.getVectorMinNumElements()) {
21200 std::swap(a&: A, b&: B);
21201 std::swap(a&: AVT, b&: BVT);
21202 }
21203
21204 SDValue BPart = DAG.getExtractSubvector(DL, VT: AVT, Vec: B, Idx: 0);
21205 SDValue Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: AVT, N1: A, N2: BPart);
21206 return DAG.getInsertSubvector(DL, Vec: B, SubVec: Res, Idx: 0);
21207}
21208
21209static SDValue foldReduceOperandViaVDOT4A(SDValue InVec, const SDLoc &DL,
21210 SelectionDAG &DAG,
21211 const RISCVSubtarget &Subtarget,
21212 const RISCVTargetLowering &TLI) {
21213 using namespace SDPatternMatch;
21214 // Note: We intentionally do not check the legality of the reduction type.
21215 // We want to handle the m4/m8 *src* types, and thus need to let illegal
21216 // intermediate types flow through here.
21217 if (InVec.getValueType().getVectorElementType() != MVT::i32 ||
21218 !InVec.getValueType().getVectorElementCount().isKnownMultipleOf(RHS: 4))
21219 return SDValue();
21220
21221 // Recurse through adds/disjoint ors (since generic dag canonicalizes to that
21222 // form).
21223 SDValue A, B;
21224 if (sd_match(N: InVec, P: m_AddLike(L: m_Value(N&: A), R: m_Value(N&: B)))) {
21225 SDValue AOpt = foldReduceOperandViaVDOT4A(InVec: A, DL, DAG, Subtarget, TLI);
21226 SDValue BOpt = foldReduceOperandViaVDOT4A(InVec: B, DL, DAG, Subtarget, TLI);
21227 if (AOpt || BOpt) {
21228 if (AOpt)
21229 A = AOpt;
21230 if (BOpt)
21231 B = BOpt;
21232 // From here, we're doing A + B with mixed types, implicitly zero
21233 // padded to the wider type. Note that we *don't* need the result
21234 // type to be the original VT, and in fact prefer narrower ones
21235 // if possible.
21236 return getZeroPaddedAdd(DL, A, B, DAG);
21237 }
21238 }
21239
21240 // zext a <--> partial_reduce_umla 0, a, 1
21241 // sext a <--> partial_reduce_smla 0, a, 1
21242 if (InVec.getOpcode() == ISD::ZERO_EXTEND ||
21243 InVec.getOpcode() == ISD::SIGN_EXTEND) {
21244 SDValue A = InVec.getOperand(i: 0);
21245 EVT OpVT = A.getValueType();
21246 if (OpVT.getVectorElementType() != MVT::i8 || !TLI.isTypeLegal(VT: OpVT))
21247 return SDValue();
21248
21249 MVT ResVT = getQDOTXResultType(OpVT: A.getSimpleValueType());
21250 SDValue B = DAG.getConstant(Val: 0x1, DL, VT: OpVT);
21251 bool IsSigned = InVec.getOpcode() == ISD::SIGN_EXTEND;
21252 unsigned Opc =
21253 IsSigned ? ISD::PARTIAL_REDUCE_SMLA : ISD::PARTIAL_REDUCE_UMLA;
21254 return DAG.getNode(Opcode: Opc, DL, VT: ResVT, Ops: {DAG.getConstant(Val: 0, DL, VT: ResVT), A, B});
21255 }
21256
21257 // mul (sext a, sext b) -> partial_reduce_smla 0, a, b
21258 // mul (zext a, zext b) -> partial_reduce_umla 0, a, b
21259 // mul (sext a, zext b) -> partial_reduce_ssmla 0, a, b
21260 // mul (zext a, sext b) -> partial_reduce_smla 0, b, a (swapped)
21261 if (!sd_match(N: InVec, P: m_Mul(L: m_Value(N&: A), R: m_Value(N&: B))))
21262 return SDValue();
21263
21264 if (!ISD::isExtOpcode(Opcode: A.getOpcode()))
21265 return SDValue();
21266
21267 EVT OpVT = A.getOperand(i: 0).getValueType();
21268 if (OpVT.getVectorElementType() != MVT::i8 ||
21269 OpVT != B.getOperand(i: 0).getValueType() ||
21270 !TLI.isTypeLegal(VT: A.getValueType()))
21271 return SDValue();
21272
21273 unsigned Opc;
21274 if (A.getOpcode() == ISD::SIGN_EXTEND && B.getOpcode() == ISD::SIGN_EXTEND)
21275 Opc = ISD::PARTIAL_REDUCE_SMLA;
21276 else if (A.getOpcode() == ISD::ZERO_EXTEND &&
21277 B.getOpcode() == ISD::ZERO_EXTEND)
21278 Opc = ISD::PARTIAL_REDUCE_UMLA;
21279 else if (A.getOpcode() == ISD::SIGN_EXTEND &&
21280 B.getOpcode() == ISD::ZERO_EXTEND)
21281 Opc = ISD::PARTIAL_REDUCE_SUMLA;
21282 else if (A.getOpcode() == ISD::ZERO_EXTEND &&
21283 B.getOpcode() == ISD::SIGN_EXTEND) {
21284 Opc = ISD::PARTIAL_REDUCE_SUMLA;
21285 std::swap(a&: A, b&: B);
21286 } else
21287 return SDValue();
21288
21289 MVT ResVT = getQDOTXResultType(OpVT: OpVT.getSimpleVT());
21290 return DAG.getNode(
21291 Opcode: Opc, DL, VT: ResVT,
21292 Ops: {DAG.getConstant(Val: 0, DL, VT: ResVT), A.getOperand(i: 0), B.getOperand(i: 0)});
21293}
21294
21295static SDValue performVECREDUCECombine(SDNode *N, SelectionDAG &DAG,
21296 const RISCVSubtarget &Subtarget,
21297 const RISCVTargetLowering &TLI) {
21298 if (!Subtarget.hasStdExtZvdot4a8i())
21299 return SDValue();
21300
21301 SDLoc DL(N);
21302 EVT VT = N->getValueType(ResNo: 0);
21303 SDValue InVec = N->getOperand(Num: 0);
21304 if (SDValue V = foldReduceOperandViaVDOT4A(InVec, DL, DAG, Subtarget, TLI))
21305 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT, Operand: V);
21306 return SDValue();
21307}
21308
21309static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
21310 const RISCVSubtarget &Subtarget,
21311 const RISCVTargetLowering &TLI) {
21312 SDValue InVec = N->getOperand(Num: 0);
21313 SDValue InVal = N->getOperand(Num: 1);
21314 SDValue EltNo = N->getOperand(Num: 2);
21315 SDLoc DL(N);
21316
21317 EVT VT = InVec.getValueType();
21318 if (VT.isScalableVector())
21319 return SDValue();
21320
21321 if (!InVec.hasOneUse())
21322 return SDValue();
21323
21324 // Given insert_vector_elt (binop a, VecC), (same_binop b, C2), Elt
21325 // move the insert_vector_elts into the arms of the binop. Note that
21326 // the new RHS must be a constant.
21327 const unsigned InVecOpcode = InVec->getOpcode();
21328 if (InVecOpcode == InVal->getOpcode() && TLI.isBinOp(Opcode: InVecOpcode) &&
21329 InVal.hasOneUse()) {
21330 SDValue InVecLHS = InVec->getOperand(Num: 0);
21331 SDValue InVecRHS = InVec->getOperand(Num: 1);
21332 SDValue InValLHS = InVal->getOperand(Num: 0);
21333 SDValue InValRHS = InVal->getOperand(Num: 1);
21334
21335 if (!ISD::isBuildVectorOfConstantSDNodes(N: InVecRHS.getNode()))
21336 return SDValue();
21337 if (!isa<ConstantSDNode>(Val: InValRHS) && !isa<ConstantFPSDNode>(Val: InValRHS))
21338 return SDValue();
21339 // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
21340 // have different LHS and RHS types.
21341 if (InVec.getOperand(i: 0).getValueType() != InVec.getOperand(i: 1).getValueType())
21342 return SDValue();
21343 SDValue LHS = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT,
21344 N1: InVecLHS, N2: InValLHS, N3: EltNo);
21345 SDValue RHS = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT,
21346 N1: InVecRHS, N2: InValRHS, N3: EltNo);
21347 return DAG.getNode(Opcode: InVecOpcode, DL, VT, N1: LHS, N2: RHS);
21348 }
21349
21350 // Given insert_vector_elt (concat_vectors ...), InVal, Elt
21351 // move the insert_vector_elt to the source operand of the concat_vector.
21352 if (InVec.getOpcode() != ISD::CONCAT_VECTORS)
21353 return SDValue();
21354
21355 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
21356 if (!IndexC)
21357 return SDValue();
21358 unsigned Elt = IndexC->getZExtValue();
21359
21360 EVT ConcatVT = InVec.getOperand(i: 0).getValueType();
21361 if (ConcatVT.getVectorElementType() != InVal.getValueType())
21362 return SDValue();
21363 unsigned ConcatNumElts = ConcatVT.getVectorNumElements();
21364 unsigned NewIdx = Elt % ConcatNumElts;
21365
21366 unsigned ConcatOpIdx = Elt / ConcatNumElts;
21367 SDValue ConcatOp = InVec.getOperand(i: ConcatOpIdx);
21368 ConcatOp = DAG.getInsertVectorElt(DL, Vec: ConcatOp, Elt: InVal, Idx: NewIdx);
21369
21370 SmallVector<SDValue> ConcatOps(InVec->ops());
21371 ConcatOps[ConcatOpIdx] = ConcatOp;
21372 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps);
21373}
21374
21375// If we're concatenating a series of vector loads like
21376// concat_vectors (load v4i8, p+0), (load v4i8, p+n), (load v4i8, p+n*2) ...
21377// Then we can turn this into a strided load by widening the vector elements
21378// vlse32 p, stride=n
21379static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
21380 const RISCVSubtarget &Subtarget,
21381 const RISCVTargetLowering &TLI) {
21382 SDLoc DL(N);
21383 EVT VT = N->getValueType(ResNo: 0);
21384
21385 // Only perform this combine on legal MVTs.
21386 if (!TLI.isTypeLegal(VT))
21387 return SDValue();
21388
21389 // TODO: Potentially extend this to scalable vectors
21390 if (VT.isScalableVector())
21391 return SDValue();
21392
21393 auto *BaseLd = dyn_cast<LoadSDNode>(Val: N->getOperand(Num: 0));
21394 if (!BaseLd || !BaseLd->isSimple() || !ISD::isNormalLoad(N: BaseLd) ||
21395 !SDValue(BaseLd, 0).hasOneUse())
21396 return SDValue();
21397
21398 EVT BaseLdVT = BaseLd->getValueType(ResNo: 0);
21399
21400 // Go through the loads and check that they're strided
21401 SmallVector<LoadSDNode *> Lds;
21402 Lds.push_back(Elt: BaseLd);
21403 Align Align = BaseLd->getAlign();
21404 for (SDValue Op : N->ops().drop_front()) {
21405 auto *Ld = dyn_cast<LoadSDNode>(Val&: Op);
21406 if (!Ld || !Ld->isSimple() || !Op.hasOneUse() ||
21407 Ld->getChain() != BaseLd->getChain() || !ISD::isNormalLoad(N: Ld) ||
21408 Ld->getValueType(ResNo: 0) != BaseLdVT)
21409 return SDValue();
21410
21411 Lds.push_back(Elt: Ld);
21412
21413 // The common alignment is the most restrictive (smallest) of all the loads
21414 Align = std::min(a: Align, b: Ld->getAlign());
21415 }
21416
21417 using PtrDiff = std::pair<std::variant<int64_t, SDValue>, bool>;
21418 auto GetPtrDiff = [&DAG](LoadSDNode *Ld1,
21419 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
21420 // If the load ptrs can be decomposed into a common (Base + Index) with a
21421 // common constant stride, then return the constant stride.
21422 BaseIndexOffset BIO1 = BaseIndexOffset::match(N: Ld1, DAG);
21423 BaseIndexOffset BIO2 = BaseIndexOffset::match(N: Ld2, DAG);
21424 if (BIO1.equalBaseIndex(Other: BIO2, DAG))
21425 return {{BIO2.getOffset() - BIO1.getOffset(), false}};
21426
21427 // Otherwise try to match (add LastPtr, Stride) or (add NextPtr, Stride)
21428 SDValue P1 = Ld1->getBasePtr();
21429 SDValue P2 = Ld2->getBasePtr();
21430 if (P2.getOpcode() == ISD::ADD && P2.getOperand(i: 0) == P1)
21431 return {{P2.getOperand(i: 1), false}};
21432 if (P1.getOpcode() == ISD::ADD && P1.getOperand(i: 0) == P2)
21433 return {{P1.getOperand(i: 1), true}};
21434
21435 return std::nullopt;
21436 };
21437
21438 // Get the distance between the first and second loads
21439 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
21440 if (!BaseDiff)
21441 return SDValue();
21442
21443 // Check all the loads are the same distance apart
21444 for (auto *It = Lds.begin() + 1; It != Lds.end() - 1; It++)
21445 if (GetPtrDiff(*It, *std::next(x: It)) != BaseDiff)
21446 return SDValue();
21447
21448 // TODO: At this point, we've successfully matched a generalized gather
21449 // load. Maybe we should emit that, and then move the specialized
21450 // matchers above and below into a DAG combine?
21451
21452 // Get the widened scalar type, e.g. v4i8 -> i64
21453 unsigned WideScalarBitWidth =
21454 BaseLdVT.getScalarSizeInBits() * BaseLdVT.getVectorNumElements();
21455 MVT WideScalarVT = MVT::getIntegerVT(BitWidth: WideScalarBitWidth);
21456
21457 // Get the vector type for the strided load, e.g. 4 x v4i8 -> v4i64
21458 MVT WideVecVT = MVT::getVectorVT(VT: WideScalarVT, NumElements: N->getNumOperands());
21459 if (!TLI.isTypeLegal(VT: WideVecVT))
21460 return SDValue();
21461
21462 // Check that the operation is legal
21463 if (!TLI.isLegalStridedLoadStore(DataType: WideVecVT, Alignment: Align))
21464 return SDValue();
21465
21466 auto [StrideVariant, MustNegateStride] = *BaseDiff;
21467 SDValue Stride =
21468 std::holds_alternative<SDValue>(v: StrideVariant)
21469 ? std::get<SDValue>(v&: StrideVariant)
21470 : DAG.getSignedConstant(Val: std::get<int64_t>(v&: StrideVariant), DL,
21471 VT: Lds[0]->getOffset().getValueType());
21472 if (MustNegateStride)
21473 Stride = DAG.getNegative(Val: Stride, DL, VT: Stride.getValueType());
21474
21475 SDValue AllOneMask =
21476 DAG.getSplat(VT: WideVecVT.changeVectorElementType(EltVT: MVT::i1), DL,
21477 Op: DAG.getConstant(Val: 1, DL, VT: MVT::i1));
21478
21479 uint64_t MemSize;
21480 if (auto *ConstStride = dyn_cast<ConstantSDNode>(Val&: Stride);
21481 ConstStride && ConstStride->getSExtValue() >= 0)
21482 // total size = (elsize * n) + (stride - elsize) * (n-1)
21483 // = elsize + stride * (n-1)
21484 MemSize = WideScalarVT.getSizeInBits() +
21485 ConstStride->getSExtValue() * (N->getNumOperands() - 1);
21486 else
21487 // If Stride isn't constant, then we can't know how much it will load
21488 MemSize = MemoryLocation::UnknownSize;
21489
21490 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
21491 PtrInfo: BaseLd->getPointerInfo(), F: BaseLd->getMemOperand()->getFlags(), Size: MemSize,
21492 BaseAlignment: Align);
21493
21494 SDValue StridedLoad = DAG.getStridedLoadVP(
21495 VT: WideVecVT, DL, Chain: BaseLd->getChain(), Ptr: BaseLd->getBasePtr(), Stride,
21496 Mask: AllOneMask,
21497 EVL: DAG.getConstant(Val: N->getNumOperands(), DL, VT: Subtarget.getXLenVT()), MMO);
21498
21499 for (SDValue Ld : N->ops())
21500 DAG.makeEquivalentMemoryOrdering(OldLoad: cast<LoadSDNode>(Val&: Ld), NewMemOp: StridedLoad);
21501
21502 return DAG.getBitcast(VT: VT.getSimpleVT(), V: StridedLoad);
21503}
21504
21505static SDValue performVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG,
21506 const RISCVSubtarget &Subtarget,
21507 const RISCVTargetLowering &TLI) {
21508 SDLoc DL(N);
21509 EVT VT = N->getValueType(ResNo: 0);
21510 const unsigned ElementSize = VT.getScalarSizeInBits();
21511 const unsigned NumElts = VT.getVectorNumElements();
21512 SDValue V1 = N->getOperand(Num: 0);
21513 SDValue V2 = N->getOperand(Num: 1);
21514 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
21515 ArrayRef<int> Mask = SVN->getMask();
21516 MVT XLenVT = Subtarget.getXLenVT();
21517
21518 // Recognized a disguised select of add/sub.
21519 bool SwapCC;
21520 if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts: NumElts) &&
21521 matchSelectAddSub(TrueVal: V1, FalseVal: V2, SwapCC)) {
21522 SDValue Sub = SwapCC ? V1 : V2;
21523 SDValue A = Sub.getOperand(i: 0);
21524 SDValue B = Sub.getOperand(i: 1);
21525
21526 SmallVector<SDValue> MaskVals;
21527 for (int MaskIndex : Mask) {
21528 bool SelectMaskVal = (MaskIndex < (int)NumElts);
21529 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
21530 }
21531 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
21532 EVT MaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: NumElts);
21533 SDValue CC = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
21534
21535 // Arrange the select such that we can match a masked
21536 // vrsub.vi to perform the conditional negate
21537 SDValue NegB = DAG.getNegative(Val: B, DL, VT);
21538 if (!SwapCC)
21539 CC = DAG.getLogicalNOT(DL, Val: CC, VT: CC->getValueType(ResNo: 0));
21540 SDValue NewB = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CC, N2: NegB, N3: B);
21541 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: NewB);
21542 }
21543
21544 if (SDValue V = compressShuffleOfShuffles(SVN, Subtarget, DAG))
21545 return V;
21546
21547 // Custom legalize <N x i128> or <N x i256> to <M x ELEN>. This runs
21548 // during the combine phase before type legalization, and relies on
21549 // DAGCombine not undoing the transform if isShuffleMaskLegal returns false
21550 // for the source mask.
21551 if (TLI.isTypeLegal(VT) || ElementSize <= Subtarget.getELen() ||
21552 !isPowerOf2_64(Value: ElementSize) || VT.getVectorNumElements() % 2 != 0 ||
21553 VT.isFloatingPoint() || TLI.isShuffleMaskLegal(M: Mask, VT))
21554 return SDValue();
21555
21556 SmallVector<int, 8> NewMask;
21557 narrowShuffleMaskElts(Scale: 2, Mask, ScaledMask&: NewMask);
21558
21559 LLVMContext &C = *DAG.getContext();
21560 EVT NewEltVT = EVT::getIntegerVT(Context&: C, BitWidth: ElementSize / 2);
21561 EVT NewVT = EVT::getVectorVT(Context&: C, VT: NewEltVT, NumElements: VT.getVectorNumElements() * 2);
21562 SDValue Res = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: DAG.getBitcast(VT: NewVT, V: V1),
21563 N2: DAG.getBitcast(VT: NewVT, V: V2), Mask: NewMask);
21564 return DAG.getBitcast(VT, V: Res);
21565}
21566
21567static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG,
21568 const RISCVSubtarget &Subtarget) {
21569 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
21570
21571 if (N->getValueType(ResNo: 0).isFixedLengthVector())
21572 return SDValue();
21573
21574 SDValue Addend = N->getOperand(Num: 0);
21575 SDValue MulOp = N->getOperand(Num: 1);
21576
21577 if (N->getOpcode() == RISCVISD::ADD_VL) {
21578 SDValue AddPassthruOp = N->getOperand(Num: 2);
21579 if (!AddPassthruOp.isUndef())
21580 return SDValue();
21581 }
21582
21583 auto IsVWMulOpc = [](unsigned Opc) {
21584 switch (Opc) {
21585 case RISCVISD::VWMUL_VL:
21586 case RISCVISD::VWMULU_VL:
21587 case RISCVISD::VWMULSU_VL:
21588 return true;
21589 default:
21590 return false;
21591 }
21592 };
21593
21594 if (!IsVWMulOpc(MulOp.getOpcode()))
21595 std::swap(a&: Addend, b&: MulOp);
21596
21597 if (!IsVWMulOpc(MulOp.getOpcode()))
21598 return SDValue();
21599
21600 SDValue MulPassthruOp = MulOp.getOperand(i: 2);
21601
21602 if (!MulPassthruOp.isUndef())
21603 return SDValue();
21604
21605 auto [AddMask, AddVL] = [](SDNode *N, SelectionDAG &DAG,
21606 const RISCVSubtarget &Subtarget) {
21607 if (N->getOpcode() == ISD::ADD) {
21608 SDLoc DL(N);
21609 return getDefaultScalableVLOps(VecVT: N->getSimpleValueType(ResNo: 0), DL, DAG,
21610 Subtarget);
21611 }
21612 return std::make_pair(x: N->getOperand(Num: 3), y: N->getOperand(Num: 4));
21613 }(N, DAG, Subtarget);
21614
21615 SDValue MulMask = MulOp.getOperand(i: 3);
21616 SDValue MulVL = MulOp.getOperand(i: 4);
21617
21618 if (AddMask != MulMask || AddVL != MulVL)
21619 return SDValue();
21620
21621 const auto &TSInfo =
21622 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
21623 unsigned Opc = TSInfo.getMAccOpcode(MulOpcode: MulOp.getOpcode());
21624
21625 SDLoc DL(N);
21626 EVT VT = N->getValueType(ResNo: 0);
21627 SDValue Ops[] = {MulOp.getOperand(i: 0), MulOp.getOperand(i: 1), Addend, AddMask,
21628 AddVL};
21629 return DAG.getNode(Opcode: Opc, DL, VT, Ops);
21630}
21631
21632static SDValue combineVdot4aAccum(SDNode *N, SelectionDAG &DAG,
21633 const RISCVSubtarget &Subtarget) {
21634
21635 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
21636
21637 if (!N->getValueType(ResNo: 0).isVector())
21638 return SDValue();
21639
21640 SDValue Addend = N->getOperand(Num: 0);
21641 SDValue DotOp = N->getOperand(Num: 1);
21642
21643 if (N->getOpcode() == RISCVISD::ADD_VL) {
21644 SDValue AddPassthruOp = N->getOperand(Num: 2);
21645 if (!AddPassthruOp.isUndef())
21646 return SDValue();
21647 }
21648
21649 auto IsVdot4aOpc = [](unsigned Opc) {
21650 switch (Opc) {
21651 case RISCVISD::VDOT4A_VL:
21652 case RISCVISD::VDOT4AU_VL:
21653 case RISCVISD::VDOT4ASU_VL:
21654 return true;
21655 default:
21656 return false;
21657 }
21658 };
21659
21660 if (!IsVdot4aOpc(DotOp.getOpcode()))
21661 std::swap(a&: Addend, b&: DotOp);
21662
21663 if (!IsVdot4aOpc(DotOp.getOpcode()))
21664 return SDValue();
21665
21666 auto [AddMask, AddVL] = [](SDNode *N, SelectionDAG &DAG,
21667 const RISCVSubtarget &Subtarget) {
21668 if (N->getOpcode() == ISD::ADD) {
21669 SDLoc DL(N);
21670 return getDefaultScalableVLOps(VecVT: N->getSimpleValueType(ResNo: 0), DL, DAG,
21671 Subtarget);
21672 }
21673 return std::make_pair(x: N->getOperand(Num: 3), y: N->getOperand(Num: 4));
21674 }(N, DAG, Subtarget);
21675
21676 SDValue MulVL = DotOp.getOperand(i: 4);
21677 if (AddVL != MulVL)
21678 return SDValue();
21679
21680 if (AddMask.getOpcode() != RISCVISD::VMSET_VL ||
21681 AddMask.getOperand(i: 0) != MulVL)
21682 return SDValue();
21683
21684 SDValue AccumOp = DotOp.getOperand(i: 2);
21685 SDLoc DL(N);
21686 EVT VT = N->getValueType(ResNo: 0);
21687 Addend = DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT, N1: Addend, N2: AccumOp,
21688 N3: DAG.getUNDEF(VT), N4: AddMask, N5: AddVL);
21689
21690 SDValue Ops[] = {DotOp.getOperand(i: 0), DotOp.getOperand(i: 1), Addend,
21691 DotOp.getOperand(i: 3), DotOp->getOperand(Num: 4)};
21692 return DAG.getNode(Opcode: DotOp->getOpcode(), DL, VT, Ops);
21693}
21694
21695static bool
21696legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index,
21697 ISD::MemIndexType &IndexType,
21698 RISCVTargetLowering::DAGCombinerInfo &DCI) {
21699 if (!DCI.isBeforeLegalize())
21700 return false;
21701
21702 SelectionDAG &DAG = DCI.DAG;
21703 const MVT XLenVT =
21704 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>().getXLenVT();
21705
21706 const EVT IndexVT = Index.getValueType();
21707
21708 // RISC-V indexed loads only support the "unsigned unscaled" addressing
21709 // mode, so anything else must be manually legalized.
21710 if (!isIndexTypeSigned(IndexType))
21711 return false;
21712
21713 if (IndexVT.getVectorElementType().bitsLT(VT: XLenVT)) {
21714 // Any index legalization should first promote to XLenVT, so we don't lose
21715 // bits when scaling. This may create an illegal index type so we let
21716 // LLVM's legalization take care of the splitting.
21717 // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
21718 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL,
21719 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: XLenVT,
21720 EC: IndexVT.getVectorElementCount()),
21721 Operand: Index);
21722 }
21723 IndexType = ISD::UNSIGNED_SCALED;
21724 return true;
21725}
21726
21727/// Match the index vector of a scatter or gather node as the shuffle mask
21728/// which performs the rearrangement if possible. Will only match if
21729/// all lanes are touched, and thus replacing the scatter or gather with
21730/// a unit strided access and shuffle is legal.
21731static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask,
21732 SmallVector<int> &ShuffleMask) {
21733 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
21734 return false;
21735 if (!ISD::isBuildVectorOfConstantSDNodes(N: Index.getNode()))
21736 return false;
21737
21738 const unsigned ElementSize = VT.getScalarStoreSize();
21739 const unsigned NumElems = VT.getVectorNumElements();
21740
21741 // Create the shuffle mask and check all bits active
21742 assert(ShuffleMask.empty());
21743 BitVector ActiveLanes(NumElems);
21744 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
21745 // TODO: We've found an active bit of UB, and could be
21746 // more aggressive here if desired.
21747 if (Index->getOperand(Num: i)->isUndef())
21748 return false;
21749 uint64_t C = Index->getConstantOperandVal(Num: i);
21750 if (C % ElementSize != 0)
21751 return false;
21752 C = C / ElementSize;
21753 if (C >= NumElems)
21754 return false;
21755 ShuffleMask.push_back(Elt: C);
21756 ActiveLanes.set(C);
21757 }
21758 return ActiveLanes.all();
21759}
21760
21761/// Match the index of a gather or scatter operation as an operation
21762/// with twice the element width and half the number of elements. This is
21763/// generally profitable (if legal) because these operations are linear
21764/// in VL, so even if we cause some extract VTYPE/VL toggles, we still
21765/// come out ahead.
21766static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask,
21767 Align BaseAlign, const RISCVSubtarget &ST) {
21768 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
21769 return false;
21770 if (!ISD::isBuildVectorOfConstantSDNodes(N: Index.getNode()))
21771 return false;
21772
21773 // Attempt a doubling. If we can use a element type 4x or 8x in
21774 // size, this will happen via multiply iterations of the transform.
21775 const unsigned NumElems = VT.getVectorNumElements();
21776 if (NumElems % 2 != 0)
21777 return false;
21778
21779 const unsigned ElementSize = VT.getScalarStoreSize();
21780 const unsigned WiderElementSize = ElementSize * 2;
21781 if (WiderElementSize > ST.getELen()/8)
21782 return false;
21783
21784 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
21785 return false;
21786
21787 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
21788 // TODO: We've found an active bit of UB, and could be
21789 // more aggressive here if desired.
21790 if (Index->getOperand(Num: i)->isUndef())
21791 return false;
21792 // TODO: This offset check is too strict if we support fully
21793 // misaligned memory operations.
21794 uint64_t C = Index->getConstantOperandVal(Num: i);
21795 if (i % 2 == 0) {
21796 if (C % WiderElementSize != 0)
21797 return false;
21798 continue;
21799 }
21800 uint64_t Last = Index->getConstantOperandVal(Num: i-1);
21801 if (C != Last + ElementSize)
21802 return false;
21803 }
21804 return true;
21805}
21806
21807// trunc (sra sext (X), zext (Y)) -> sra (X, smin (Y, scalarsize(Y) - 1))
21808// This would be benefit for the cases where X and Y are both the same value
21809// type of low precision vectors. Since the truncate would be lowered into
21810// n-levels TRUNCATE_VECTOR_VL to satisfy RVV's SEW*2->SEW truncate
21811// restriction, such pattern would be expanded into a series of "vsetvli"
21812// and "vnsrl" instructions later to reach this point.
21813static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG) {
21814 SDValue Mask = N->getOperand(Num: 1);
21815 SDValue VL = N->getOperand(Num: 2);
21816
21817 bool IsVLMAX = isAllOnesConstant(V: VL) ||
21818 (isa<RegisterSDNode>(Val: VL) &&
21819 cast<RegisterSDNode>(Val&: VL)->getReg() == RISCV::X0);
21820 if (!IsVLMAX || Mask.getOpcode() != RISCVISD::VMSET_VL ||
21821 Mask.getOperand(i: 0) != VL)
21822 return SDValue();
21823
21824 auto IsTruncNode = [&](SDValue V) {
21825 return V.getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL &&
21826 V.getOperand(i: 1) == Mask && V.getOperand(i: 2) == VL;
21827 };
21828
21829 SDValue Op = N->getOperand(Num: 0);
21830
21831 // We need to first find the inner level of TRUNCATE_VECTOR_VL node
21832 // to distinguish such pattern.
21833 while (IsTruncNode(Op)) {
21834 if (!Op.hasOneUse())
21835 return SDValue();
21836 Op = Op.getOperand(i: 0);
21837 }
21838
21839 if (Op.getOpcode() != ISD::SRA || !Op.hasOneUse())
21840 return SDValue();
21841
21842 SDValue N0 = Op.getOperand(i: 0);
21843 SDValue N1 = Op.getOperand(i: 1);
21844 if (N0.getOpcode() != ISD::SIGN_EXTEND || !N0.hasOneUse() ||
21845 N1.getOpcode() != ISD::ZERO_EXTEND || !N1.hasOneUse())
21846 return SDValue();
21847
21848 SDValue N00 = N0.getOperand(i: 0);
21849 SDValue N10 = N1.getOperand(i: 0);
21850 if (!N00.getValueType().isVector() ||
21851 N00.getValueType() != N10.getValueType() ||
21852 N->getValueType(ResNo: 0) != N10.getValueType())
21853 return SDValue();
21854
21855 unsigned MaxShAmt = N10.getValueType().getScalarSizeInBits() - 1;
21856 SDValue SMin =
21857 DAG.getNode(Opcode: ISD::SMIN, DL: SDLoc(N1), VT: N->getValueType(ResNo: 0), N1: N10,
21858 N2: DAG.getConstant(Val: MaxShAmt, DL: SDLoc(N1), VT: N->getValueType(ResNo: 0)));
21859 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: N00, N2: SMin);
21860}
21861
21862// Combine (truncate_vector_vl (umin X, C)) -> (vnclipu_vl X) if C is the
21863// maximum value for the truncated type.
21864// Combine (truncate_vector_vl (smin (smax X, C2), C1)) -> (vnclip_vl X) if C1
21865// is the signed maximum value for the truncated type and C2 is the signed
21866// minimum value.
21867static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG,
21868 const RISCVSubtarget &Subtarget) {
21869 assert(N->getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL);
21870
21871 MVT VT = N->getSimpleValueType(ResNo: 0);
21872
21873 SDValue Mask = N->getOperand(Num: 1);
21874 SDValue VL = N->getOperand(Num: 2);
21875
21876 auto MatchMinMax = [&VL, &Mask](SDValue V, unsigned Opc, unsigned OpcVL,
21877 APInt &SplatVal) {
21878 if (V.getOpcode() != Opc &&
21879 !(V.getOpcode() == OpcVL && V.getOperand(i: 2).isUndef() &&
21880 V.getOperand(i: 3) == Mask && V.getOperand(i: 4) == VL))
21881 return SDValue();
21882
21883 SDValue Op = V.getOperand(i: 1);
21884
21885 // Peek through conversion between fixed and scalable vectors.
21886 if (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op.getOperand(i: 0).isUndef() &&
21887 isNullConstant(V: Op.getOperand(i: 2)) &&
21888 Op.getOperand(i: 1).getValueType().isFixedLengthVector() &&
21889 Op.getOperand(i: 1).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
21890 Op.getOperand(i: 1).getOperand(i: 0).getValueType() == Op.getValueType() &&
21891 isNullConstant(V: Op.getOperand(i: 1).getOperand(i: 1)))
21892 Op = Op.getOperand(i: 1).getOperand(i: 0);
21893
21894 if (ISD::isConstantSplatVector(N: Op.getNode(), SplatValue&: SplatVal))
21895 return V.getOperand(i: 0);
21896
21897 if (Op.getOpcode() == RISCVISD::VMV_V_X_VL && Op.getOperand(i: 0).isUndef() &&
21898 Op.getOperand(i: 2) == VL) {
21899 if (auto *Op1 = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
21900 SplatVal =
21901 Op1->getAPIntValue().sextOrTrunc(width: Op.getScalarValueSizeInBits());
21902 return V.getOperand(i: 0);
21903 }
21904 }
21905
21906 return SDValue();
21907 };
21908
21909 SDLoc DL(N);
21910
21911 auto DetectUSatPattern = [&](SDValue V) {
21912 APInt LoC, HiC;
21913
21914 // Simple case, V is a UMIN.
21915 if (SDValue UMinOp = MatchMinMax(V, ISD::UMIN, RISCVISD::UMIN_VL, HiC))
21916 if (HiC.isMask(numBits: VT.getScalarSizeInBits()))
21917 return UMinOp;
21918
21919 // If we have an SMAX that removes negative numbers first, then we can match
21920 // SMIN instead of UMIN.
21921 if (SDValue SMinOp = MatchMinMax(V, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21922 if (SDValue SMaxOp =
21923 MatchMinMax(SMinOp, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21924 if (LoC.isNonNegative() && HiC.isMask(numBits: VT.getScalarSizeInBits()))
21925 return SMinOp;
21926
21927 // If we have an SMIN before an SMAX and the SMAX constant is less than or
21928 // equal to the SMIN constant, we can use vnclipu if we insert a new SMAX
21929 // first.
21930 if (SDValue SMaxOp = MatchMinMax(V, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21931 if (SDValue SMinOp =
21932 MatchMinMax(SMaxOp, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21933 if (LoC.isNonNegative() && HiC.isMask(numBits: VT.getScalarSizeInBits()) &&
21934 HiC.uge(RHS: LoC))
21935 return DAG.getNode(Opcode: RISCVISD::SMAX_VL, DL, VT: V.getValueType(), N1: SMinOp,
21936 N2: V.getOperand(i: 1), N3: DAG.getUNDEF(VT: V.getValueType()),
21937 N4: Mask, N5: VL);
21938
21939 return SDValue();
21940 };
21941
21942 auto DetectSSatPattern = [&](SDValue V) {
21943 unsigned NumDstBits = VT.getScalarSizeInBits();
21944 unsigned NumSrcBits = V.getScalarValueSizeInBits();
21945 APInt SignedMax = APInt::getSignedMaxValue(numBits: NumDstBits).sext(width: NumSrcBits);
21946 APInt SignedMin = APInt::getSignedMinValue(numBits: NumDstBits).sext(width: NumSrcBits);
21947
21948 APInt HiC, LoC;
21949 if (SDValue SMinOp = MatchMinMax(V, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21950 if (SDValue SMaxOp =
21951 MatchMinMax(SMinOp, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21952 if (HiC == SignedMax && LoC == SignedMin)
21953 return SMaxOp;
21954
21955 if (SDValue SMaxOp = MatchMinMax(V, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21956 if (SDValue SMinOp =
21957 MatchMinMax(SMaxOp, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21958 if (HiC == SignedMax && LoC == SignedMin)
21959 return SMinOp;
21960
21961 return SDValue();
21962 };
21963
21964 SDValue Src = N->getOperand(Num: 0);
21965
21966 // Look through multiple layers of truncates.
21967 while (Src.getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL &&
21968 Src.getOperand(i: 1) == Mask && Src.getOperand(i: 2) == VL &&
21969 Src.hasOneUse())
21970 Src = Src.getOperand(i: 0);
21971
21972 SDValue Val;
21973 unsigned ClipOpc;
21974 if ((Val = DetectUSatPattern(Src)))
21975 ClipOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
21976 else if ((Val = DetectSSatPattern(Src)))
21977 ClipOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
21978 else
21979 return SDValue();
21980
21981 MVT ValVT = Val.getSimpleValueType();
21982
21983 do {
21984 MVT ValEltVT = MVT::getIntegerVT(BitWidth: ValVT.getScalarSizeInBits() / 2);
21985 ValVT = ValVT.changeVectorElementType(EltVT: ValEltVT);
21986 Val = DAG.getNode(Opcode: ClipOpc, DL, VT: ValVT, N1: Val, N2: Mask, N3: VL);
21987 } while (ValVT != VT);
21988
21989 return Val;
21990}
21991
21992// Convert
21993// (iX ctpop (bitcast (vXi1 A)))
21994// ->
21995// (zext (vcpop.m (nxvYi1 (insert_subvec (vXi1 A)))))
21996// and
21997// (iN reduce.add (zext (vXi1 A to vXiN))
21998// ->
21999// (zext (vcpop.m (nxvYi1 (insert_subvec (vXi1 A)))))
22000// FIXME: It's complicated to match all the variations of this after type
22001// legalization so we only handle the pre-type legalization pattern, but that
22002// requires the fixed vector type to be legal.
22003static SDValue combineToVCPOP(SDNode *N, SelectionDAG &DAG,
22004 const RISCVSubtarget &Subtarget) {
22005 unsigned Opc = N->getOpcode();
22006 assert((Opc == ISD::CTPOP || Opc == ISD::VECREDUCE_ADD) &&
22007 "Unexpected opcode");
22008 EVT VT = N->getValueType(ResNo: 0);
22009 if (!VT.isScalarInteger())
22010 return SDValue();
22011
22012 SDValue Src = N->getOperand(Num: 0);
22013
22014 if (Opc == ISD::CTPOP) {
22015 // Peek through zero_extend. It doesn't change the count.
22016 if (Src.getOpcode() == ISD::ZERO_EXTEND)
22017 Src = Src.getOperand(i: 0);
22018
22019 if (Src.getOpcode() != ISD::BITCAST)
22020 return SDValue();
22021 Src = Src.getOperand(i: 0);
22022 } else if (Opc == ISD::VECREDUCE_ADD) {
22023 if (Src.getOpcode() != ISD::ZERO_EXTEND)
22024 return SDValue();
22025 Src = Src.getOperand(i: 0);
22026 }
22027
22028 EVT SrcEVT = Src.getValueType();
22029 if (!SrcEVT.isSimple())
22030 return SDValue();
22031
22032 MVT SrcMVT = SrcEVT.getSimpleVT();
22033 // Make sure the input is an i1 vector.
22034 if (!SrcMVT.isVectorOf(EltVT: MVT::i1))
22035 return SDValue();
22036
22037 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
22038 if (!TLI.isTypeLegal(VT: SrcMVT))
22039 return SDValue();
22040
22041 // Check that destination type is large enough to hold result without
22042 // overflow.
22043 if (Opc == ISD::VECREDUCE_ADD) {
22044 unsigned EltSize = SrcMVT.getScalarSizeInBits();
22045 unsigned MinSize = SrcMVT.getSizeInBits().getKnownMinValue();
22046 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
22047 unsigned MaxVLMAX = SrcMVT.isFixedLengthVector()
22048 ? SrcMVT.getVectorNumElements()
22049 : RISCVTargetLowering::computeVLMAX(
22050 VectorBits: VectorBitsMax, EltSize, MinSize);
22051 if (VT.getFixedSizeInBits() < Log2_32(Value: MaxVLMAX) + 1)
22052 return SDValue();
22053 }
22054
22055 MVT ContainerVT = SrcMVT;
22056 if (SrcMVT.isFixedLengthVector()) {
22057 ContainerVT = getContainerForFixedLengthVector(VT: SrcMVT, Subtarget);
22058 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
22059 }
22060
22061 SDLoc DL(N);
22062 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcMVT, ContainerVT, DL, DAG, Subtarget);
22063
22064 MVT XLenVT = Subtarget.getXLenVT();
22065 SDValue Pop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Src, N2: Mask, N3: VL);
22066 return DAG.getZExtOrTrunc(Op: Pop, DL, VT);
22067}
22068
22069static SDValue performSHLCombine(SDNode *N,
22070 TargetLowering::DAGCombinerInfo &DCI,
22071 const RISCVSubtarget &Subtarget) {
22072 // (shl (zext x), y) -> (vwsll x, y)
22073 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
22074 return V;
22075
22076 // (shl (sext x), C) -> (vwmulsu x, 1u << C)
22077 // (shl (zext x), C) -> (vwmulu x, 1u << C)
22078
22079 if (!DCI.isAfterLegalizeDAG())
22080 return SDValue();
22081
22082 SDValue LHS = N->getOperand(Num: 0);
22083 if (!LHS.hasOneUse())
22084 return SDValue();
22085 unsigned Opcode;
22086 switch (LHS.getOpcode()) {
22087 case ISD::SIGN_EXTEND:
22088 case RISCVISD::VSEXT_VL:
22089 Opcode = RISCVISD::VWMULSU_VL;
22090 break;
22091 case ISD::ZERO_EXTEND:
22092 case RISCVISD::VZEXT_VL:
22093 Opcode = RISCVISD::VWMULU_VL;
22094 break;
22095 default:
22096 return SDValue();
22097 }
22098
22099 SDValue RHS = N->getOperand(Num: 1);
22100 APInt ShAmt;
22101 uint64_t ShAmtInt;
22102 if (ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: ShAmt))
22103 ShAmtInt = ShAmt.getZExtValue();
22104 else if (RHS.getOpcode() == RISCVISD::VMV_V_X_VL &&
22105 RHS.getOperand(i: 1).getOpcode() == ISD::Constant)
22106 ShAmtInt = RHS.getConstantOperandVal(i: 1);
22107 else
22108 return SDValue();
22109
22110 // Better foldings:
22111 // (shl (sext x), 1) -> (vwadd x, x)
22112 // (shl (zext x), 1) -> (vwaddu x, x)
22113 if (ShAmtInt <= 1)
22114 return SDValue();
22115
22116 SDValue NarrowOp = LHS.getOperand(i: 0);
22117 MVT NarrowVT = NarrowOp.getSimpleValueType();
22118 uint64_t NarrowBits = NarrowVT.getScalarSizeInBits();
22119 if (ShAmtInt >= NarrowBits)
22120 return SDValue();
22121 MVT VT = N->getSimpleValueType(ResNo: 0);
22122 if (NarrowBits * 2 != VT.getScalarSizeInBits())
22123 return SDValue();
22124
22125 SelectionDAG &DAG = DCI.DAG;
22126 SDLoc DL(N);
22127 SDValue Passthru, Mask, VL;
22128 switch (N->getOpcode()) {
22129 case ISD::SHL:
22130 Passthru = DAG.getUNDEF(VT);
22131 std::tie(args&: Mask, args&: VL) = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
22132 break;
22133 case RISCVISD::SHL_VL:
22134 Passthru = N->getOperand(Num: 2);
22135 Mask = N->getOperand(Num: 3);
22136 VL = N->getOperand(Num: 4);
22137 break;
22138 default:
22139 llvm_unreachable("Expected SHL");
22140 }
22141 return DAG.getNode(Opcode, DL, VT, N1: NarrowOp,
22142 N2: DAG.getConstant(Val: 1ULL << ShAmtInt, DL: SDLoc(RHS), VT: NarrowVT),
22143 N3: Passthru, N4: Mask, N5: VL);
22144}
22145
22146// Fold (smax (smin X, (1 << C) - 1), -(1 << C)) -> riscv_sati X, C.
22147// Fold (smin (smax X, -(1 << C)), (1 << C) - 1) -> riscv_sati X, C.
22148// Fold (smax (smin X, (1 << C) - 1), 0) -> riscv_usati X, C.
22149// Fold (smin (smax X, 0, (1 << C) - 1) -> riscv_usati X, C.
22150static SDValue combineMinMaxToSat(SDNode *N,
22151 TargetLowering::DAGCombinerInfo &DCI,
22152 const RISCVSubtarget &Subtarget) {
22153 if (!DCI.isAfterLegalizeDAG())
22154 return SDValue();
22155
22156 if (!Subtarget.hasStdExtP())
22157 return SDValue();
22158
22159 EVT VT = N->getValueType(ResNo: 0);
22160
22161 if (VT != Subtarget.getXLenVT())
22162 return SDValue();
22163
22164 SDValue N0 = N->getOperand(Num: 0);
22165
22166 if ((N0.getOpcode() != ISD::SMIN && N0.getOpcode() != ISD::SMAX) ||
22167 !isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) ||
22168 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)))
22169 return SDValue();
22170
22171 SDValue Min = SDValue(N, 0);
22172 SDValue Max = N0;
22173 SDValue Input = N0.getOperand(i: 0);
22174 if (Min.getOpcode() == ISD::SMAX)
22175 std::swap(a&: Min, b&: Max);
22176
22177 APInt MinC = Min.getConstantOperandAPInt(i: 1);
22178 APInt MaxC = Max.getConstantOperandAPInt(i: 1);
22179
22180 if (Min.getOpcode() != ISD::SMIN || Max.getOpcode() != ISD::SMAX ||
22181 !(MinC + 1).isPowerOf2())
22182 return SDValue();
22183
22184 SelectionDAG &DAG = DCI.DAG;
22185
22186 SDLoc DL(N);
22187 if (MinC == ~MaxC)
22188 return DAG.getNode(Opcode: RISCVISD::SATI, DL, VT, N1: Input,
22189 N2: DAG.getTargetConstant(Val: MinC.countr_one(), DL, VT));
22190 if (MaxC == 0)
22191 return DAG.getNode(Opcode: RISCVISD::USATI, DL, VT, N1: Input,
22192 N2: DAG.getTargetConstant(Val: MinC.countr_one(), DL, VT));
22193
22194 return SDValue();
22195}
22196
22197// Returns true if the i32 pair (Lo, Hi) is the 64-bit sign-extension of the
22198// i32 value Lo, i.e. Hi == (sra Lo, 31). Used to fold ADDD/SUBD of a
22199// sign-extended operand into the WADDA/WSUBA widening accumulate nodes.
22200static bool isI32SignExtended(SDValue Lo, SDValue Hi) {
22201 return Hi.getOpcode() == ISD::SRA && Hi.getOperand(i: 0) == Lo &&
22202 isa<ConstantSDNode>(Val: Hi.getOperand(i: 1)) &&
22203 Hi.getConstantOperandVal(i: 1) == 31;
22204}
22205
22206SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
22207 DAGCombinerInfo &DCI) const {
22208 SelectionDAG &DAG = DCI.DAG;
22209 const MVT XLenVT = Subtarget.getXLenVT();
22210 SDLoc DL(N);
22211
22212 // Helper to call SimplifyDemandedBits on an operand of N where only some low
22213 // bits are demanded. N will be added to the Worklist if it was not deleted.
22214 // Caller should return SDValue(N, 0) if this returns true.
22215 auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
22216 SDValue Op = N->getOperand(Num: OpNo);
22217 APInt Mask = APInt::getLowBitsSet(numBits: Op.getValueSizeInBits(), loBitsSet: LowBits);
22218 if (!SimplifyDemandedBits(Op, DemandedBits: Mask, DCI))
22219 return false;
22220
22221 if (N->getOpcode() != ISD::DELETED_NODE)
22222 DCI.AddToWorklist(N);
22223 return true;
22224 };
22225
22226 switch (N->getOpcode()) {
22227 default:
22228 break;
22229 case RISCVISD::SplitF64: {
22230 SDValue Op0 = N->getOperand(Num: 0);
22231 // If the input to SplitF64 is just BuildPairF64 then the operation is
22232 // redundant. Instead, use BuildPairF64's operands directly.
22233 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
22234 return DCI.CombineTo(N, Res0: Op0.getOperand(i: 0), Res1: Op0.getOperand(i: 1));
22235
22236 if (Op0->isUndef()) {
22237 SDValue Lo = DAG.getUNDEF(VT: MVT::i32);
22238 SDValue Hi = DAG.getUNDEF(VT: MVT::i32);
22239 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
22240 }
22241
22242 // It's cheaper to materialise two 32-bit integers than to load a double
22243 // from the constant pool and transfer it to integer registers through the
22244 // stack.
22245 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
22246 APInt V = C->getValueAPF().bitcastToAPInt();
22247 SDValue Lo = DAG.getConstant(Val: V.trunc(width: 32), DL, VT: MVT::i32);
22248 SDValue Hi = DAG.getConstant(Val: V.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
22249 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
22250 }
22251
22252 // This is a target-specific version of a DAGCombine performed in
22253 // DAGCombiner::visitBITCAST. It performs the equivalent of:
22254 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
22255 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
22256 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
22257 !Op0.getNode()->hasOneUse() || Subtarget.hasStdExtZdinx())
22258 break;
22259 SDValue NewSplitF64 =
22260 DAG.getNode(Opcode: RISCVISD::SplitF64, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22261 N: Op0.getOperand(i: 0));
22262 SDValue Lo = NewSplitF64.getValue(R: 0);
22263 SDValue Hi = NewSplitF64.getValue(R: 1);
22264 APInt SignBit = APInt::getSignMask(BitWidth: 32);
22265 if (Op0.getOpcode() == ISD::FNEG) {
22266 SDValue NewHi = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Hi,
22267 N2: DAG.getConstant(Val: SignBit, DL, VT: MVT::i32));
22268 return DCI.CombineTo(N, Res0: Lo, Res1: NewHi);
22269 }
22270 assert(Op0.getOpcode() == ISD::FABS);
22271 SDValue NewHi = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Hi,
22272 N2: DAG.getConstant(Val: ~SignBit, DL, VT: MVT::i32));
22273 return DCI.CombineTo(N, Res0: Lo, Res1: NewHi);
22274 }
22275 case RISCVISD::SLLW:
22276 case RISCVISD::SRAW:
22277 case RISCVISD::SRLW:
22278 case RISCVISD::RORW:
22279 case RISCVISD::ROLW: {
22280 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
22281 if (SimplifyDemandedLowBitsHelper(0, 32) ||
22282 SimplifyDemandedLowBitsHelper(1, 5))
22283 return SDValue(N, 0);
22284
22285 break;
22286 }
22287 case RISCVISD::ABSW:
22288 case RISCVISD::CLSW:
22289 case RISCVISD::CLZW:
22290 case RISCVISD::CTZW: {
22291 // Only the lower 32 bits of the first operand are read
22292 if (SimplifyDemandedLowBitsHelper(0, 32))
22293 return SDValue(N, 0);
22294 break;
22295 }
22296 case RISCVISD::WMULSU: {
22297 // Convert to MULHSU if only the upper half is used.
22298 if (!N->hasAnyUseOfValue(Value: 0)) {
22299 SDValue Res = DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT: N->getValueType(ResNo: 1),
22300 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
22301 return DCI.CombineTo(N, Res0: Res, Res1: Res);
22302 }
22303 break;
22304 }
22305 case RISCVISD::PSRL:
22306 case RISCVISD::PSRA: {
22307 // Fold (PSRL/PSRA (trunc (PSRL X, C1)), C2) -> (trunc (PSRL/PSRA X, C1+C2))
22308 // Fold (PSRL/PSRA (concat (trunc (PSRL X, C1)), (trunc (PSRL Y, C1))), C2)
22309 // -> (concat (trunc (PSRL/PSRA X, C1+C2)), (trunc (PSRL/PSRA Y, C1+C2)))
22310 // In both cases C1 must equal the number of bits discarded by the truncate.
22311 auto *C2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
22312 if (!C2)
22313 break;
22314 // Match (trunc (PSRL X, C1)) where C1 == bits discarded by the truncate
22315 // and C1+C2 is a valid shift. Returns {NarrowVT, WideVT, NewShAmt, X}
22316 // without creating any DAG nodes.
22317 struct TruncPSRLMatch {
22318 uint64_t NewShAmt;
22319 SDValue Src;
22320 };
22321 auto MatchTruncPSRL =
22322 [](SDValue TruncVal,
22323 ConstantSDNode *C2) -> std::optional<TruncPSRLMatch> {
22324 if (TruncVal.getOpcode() != ISD::TRUNCATE || !TruncVal.hasOneUse())
22325 return std::nullopt;
22326 SDValue PSRLVal = TruncVal.getOperand(i: 0);
22327 if (PSRLVal.getOpcode() != RISCVISD::PSRL || !PSRLVal.hasOneUse())
22328 return std::nullopt;
22329 auto *C1 = dyn_cast<ConstantSDNode>(Val: PSRLVal.getOperand(i: 1));
22330 if (!C1)
22331 return std::nullopt;
22332 MVT NarrowVT = TruncVal.getSimpleValueType();
22333 MVT WideVT = PSRLVal.getSimpleValueType();
22334 unsigned WideEltBits = WideVT.getVectorElementType().getSizeInBits();
22335 unsigned NarrowEltBits = NarrowVT.getVectorElementType().getSizeInBits();
22336 if (C1->getZExtValue() != WideEltBits - NarrowEltBits)
22337 return std::nullopt;
22338 uint64_t NewShAmt = C1->getZExtValue() + C2->getZExtValue();
22339 if (NewShAmt >= WideEltBits)
22340 return std::nullopt;
22341 return TruncPSRLMatch{.NewShAmt: NewShAmt, .Src: PSRLVal.getOperand(i: 0)};
22342 };
22343 auto MakeFoldedShift = [&](const TruncPSRLMatch &M, EVT VT,
22344 unsigned OuterOpc) {
22345 SDValue NewShift = DAG.getNode(Opcode: OuterOpc, DL, VT: M.Src.getValueType(), N1: M.Src,
22346 N2: DAG.getConstant(Val: M.NewShAmt, DL, VT: XLenVT));
22347 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: NewShift);
22348 };
22349
22350 SDValue Src = N->getOperand(Num: 0);
22351 if (auto M = MatchTruncPSRL(Src, C2))
22352 return MakeFoldedShift(*M, Src.getValueType(), N->getOpcode());
22353
22354 if (Src.getOpcode() == ISD::CONCAT_VECTORS && Src.hasOneUse() &&
22355 Src.getNumOperands() == 2) {
22356 SDValue Op0 = Src.getOperand(i: 0);
22357 SDValue Op1 = Src.getOperand(i: 1);
22358 auto M0 = MatchTruncPSRL(Op0, C2);
22359 auto M1 = MatchTruncPSRL(Op1, C2);
22360 if (M0 && M1)
22361 return DAG.getNode(
22362 Opcode: ISD::CONCAT_VECTORS, DL, VT: N->getValueType(ResNo: 0),
22363 N1: MakeFoldedShift(*M0, Op0.getValueType(), N->getOpcode()),
22364 N2: MakeFoldedShift(*M1, Op1.getValueType(), N->getOpcode()));
22365 }
22366
22367 break;
22368 }
22369 case RISCVISD::ADDD: {
22370 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
22371 "ADDD is only for RV32 with P extension");
22372
22373 SDValue Op0Lo = N->getOperand(Num: 0);
22374 SDValue Op0Hi = N->getOperand(Num: 1);
22375 SDValue Op1Lo = N->getOperand(Num: 2);
22376 SDValue Op1Hi = N->getOperand(Num: 3);
22377
22378 // (ADDD lo, hi, x, 0) -> (WADDAU lo, hi, x, 0)
22379 if (isNullConstant(V: Op1Hi)) {
22380 SDValue Result =
22381 DAG.getNode(Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22382 N1: Op0Lo, N2: Op0Hi, N3: Op1Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
22383 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22384 }
22385 // (ADDD x, 0, lo, hi) -> (WADDAU lo, hi, x, 0)
22386 if (isNullConstant(V: Op0Hi)) {
22387 SDValue Result =
22388 DAG.getNode(Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22389 N1: Op1Lo, N2: Op1Hi, N3: Op0Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
22390 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22391 }
22392
22393 // (ADDD lo, hi, x, sra(x, 31)) -> (WADDA lo, hi, x, 0)
22394 if (isI32SignExtended(Lo: Op1Lo, Hi: Op1Hi)) {
22395 SDValue Result =
22396 DAG.getNode(Opcode: RISCVISD::WADDA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22397 N1: Op0Lo, N2: Op0Hi, N3: Op1Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
22398 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22399 }
22400 // (ADDD x, sra(x, 31), lo, hi) -> (WADDA lo, hi, x, 0)
22401 if (isI32SignExtended(Lo: Op0Lo, Hi: Op0Hi)) {
22402 SDValue Result =
22403 DAG.getNode(Opcode: RISCVISD::WADDA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22404 N1: Op1Lo, N2: Op1Hi, N3: Op0Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
22405 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22406 }
22407 break;
22408 }
22409 case RISCVISD::SUBD: {
22410 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
22411 "SUBD is only for RV32 with P extension");
22412
22413 SDValue Op0Lo = N->getOperand(Num: 0);
22414 SDValue Op0Hi = N->getOperand(Num: 1);
22415 SDValue Op1Lo = N->getOperand(Num: 2);
22416 SDValue Op1Hi = N->getOperand(Num: 3);
22417
22418 // (SUBD lo, hi, x, 0) -> (WSUBAU lo, hi, 0, x)
22419 // WSUBAU semantics: rd = rd + zext(rs1) - zext(rs2)
22420 if (isNullConstant(V: Op1Hi)) {
22421 SDValue Result =
22422 DAG.getNode(Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22423 N1: Op0Lo, N2: Op0Hi, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N4: Op1Lo);
22424 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22425 }
22426
22427 // (SUBD lo, hi, x, sra(x, 31)) -> (WSUBA lo, hi, 0, x)
22428 // WSUBA semantics: rd = rd + sext(rs1) - sext(rs2)
22429 if (isI32SignExtended(Lo: Op1Lo, Hi: Op1Hi)) {
22430 SDValue Result =
22431 DAG.getNode(Opcode: RISCVISD::WSUBA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22432 N1: Op0Lo, N2: Op0Hi, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N4: Op1Lo);
22433 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22434 }
22435 break;
22436 }
22437 case RISCVISD::WADDAU: {
22438 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
22439 "WADDAU is only for RV32 with P extension");
22440 SDValue Op0Lo = N->getOperand(Num: 0);
22441 SDValue Op0Hi = N->getOperand(Num: 1);
22442 SDValue Op1 = N->getOperand(Num: 2);
22443 SDValue Op2 = N->getOperand(Num: 3);
22444
22445 // (WADDAU lo, 0, rs1, 0) -> (WADDU lo, rs1)
22446 if (isNullConstant(V: Op0Hi) && isNullConstant(V: Op2)) {
22447 SDValue Result = DAG.getNode(
22448 Opcode: RISCVISD::WADDU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op1);
22449 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22450 }
22451
22452 // (WADDAU -C, -1, rs1, 0) -> (WSUBU rs1, C) where C > 0
22453 if (isNullConstant(V: Op2) && isAllOnesConstant(V: Op0Hi)) {
22454 if (auto *C0 = dyn_cast<ConstantSDNode>(Val&: Op0Lo)) {
22455 int64_t Val = C0->getSExtValue();
22456 if (Val < 0) {
22457 SDValue PosConst = DAG.getConstant(Val: -Val, DL, VT: MVT::i32);
22458 SDValue Result =
22459 DAG.getNode(Opcode: RISCVISD::WSUBU, DL,
22460 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op1, N2: PosConst);
22461 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22462 }
22463 }
22464 }
22465
22466 // FIXME: Canonicalize zero Op1 to Op2.
22467 if (isNullConstant(V: Op2) && Op0Lo.getNode() == Op0Hi.getNode() &&
22468 Op0Lo.getResNo() == 0 && Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() &&
22469 Op0Hi.hasOneUse()) {
22470 // (WADDAU (WADDAU lo, hi, x, 0), y, 0) -> (WADDAU lo, hi, x, y)
22471 if (Op0Lo.getOpcode() == RISCVISD::WADDAU &&
22472 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
22473 SDValue Result = DAG.getNode(
22474 Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22475 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op1);
22476 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22477 }
22478 // (WADDAU (WSUBAU lo, hi, 0, a), b, 0) -> (WSUBAU lo, hi, b, a)
22479 if (Op0Lo.getOpcode() == RISCVISD::WSUBAU &&
22480 isNullConstant(V: Op0Lo.getOperand(i: 2))) {
22481 SDValue Result = DAG.getNode(
22482 Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22483 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op1, N4: Op0Lo.getOperand(i: 3));
22484 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22485 }
22486 }
22487 break;
22488 }
22489 case RISCVISD::WSUBAU: {
22490 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
22491 "WSUBAU is only for RV32 with P extension");
22492 SDValue Op0Lo = N->getOperand(Num: 0);
22493 SDValue Op0Hi = N->getOperand(Num: 1);
22494 SDValue Op1 = N->getOperand(Num: 2);
22495 SDValue Op2 = N->getOperand(Num: 3);
22496
22497 // (WSUBAU lo, 0, 0, rs2) -> (WSUBU lo, rs2)
22498 if (isNullConstant(V: Op0Hi) && isNullConstant(V: Op1)) {
22499 SDValue Result = DAG.getNode(
22500 Opcode: RISCVISD::WSUBU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op2);
22501 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22502 }
22503
22504 // (WSUBAU (WADDAU lo, hi, a, 0), 0, b) -> (WSUBAU lo, hi, a, b)
22505 if (isNullConstant(V: Op1) && Op0Lo.getOpcode() == RISCVISD::WADDAU &&
22506 Op0Lo.getNode() == Op0Hi.getNode() && Op0Lo.getResNo() == 0 &&
22507 Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() && Op0Hi.hasOneUse() &&
22508 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
22509 SDValue Result = DAG.getNode(
22510 Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22511 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op2);
22512 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22513 }
22514 break;
22515 }
22516 case RISCVISD::WADDA: {
22517 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
22518 "WADDA is only for RV32 with P extension");
22519 SDValue Op0Lo = N->getOperand(Num: 0);
22520 SDValue Op0Hi = N->getOperand(Num: 1);
22521 SDValue Op1 = N->getOperand(Num: 2);
22522 SDValue Op2 = N->getOperand(Num: 3);
22523
22524 // Fold a chained accumulate into the free second source slot.
22525 if (isNullConstant(V: Op2) && Op0Lo.getNode() == Op0Hi.getNode() &&
22526 Op0Lo.getResNo() == 0 && Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() &&
22527 Op0Hi.hasOneUse()) {
22528 // (WADDA (WADDA lo, hi, x, 0), y, 0) -> (WADDA lo, hi, x, y)
22529 if (Op0Lo.getOpcode() == RISCVISD::WADDA &&
22530 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
22531 SDValue Result = DAG.getNode(
22532 Opcode: RISCVISD::WADDA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22533 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op1);
22534 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22535 }
22536 // (WADDA (WSUBA lo, hi, 0, a), b, 0) -> (WSUBA lo, hi, b, a)
22537 if (Op0Lo.getOpcode() == RISCVISD::WSUBA &&
22538 isNullConstant(V: Op0Lo.getOperand(i: 2))) {
22539 SDValue Result = DAG.getNode(
22540 Opcode: RISCVISD::WSUBA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22541 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op1, N4: Op0Lo.getOperand(i: 3));
22542 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22543 }
22544 }
22545 break;
22546 }
22547 case RISCVISD::WSUBA: {
22548 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
22549 "WSUBA is only for RV32 with P extension");
22550 SDValue Op0Lo = N->getOperand(Num: 0);
22551 SDValue Op0Hi = N->getOperand(Num: 1);
22552 SDValue Op1 = N->getOperand(Num: 2);
22553 SDValue Op2 = N->getOperand(Num: 3);
22554
22555 // (WSUBA (WADDA lo, hi, a, 0), 0, b) -> (WSUBA lo, hi, a, b)
22556 if (isNullConstant(V: Op1) && Op0Lo.getOpcode() == RISCVISD::WADDA &&
22557 Op0Lo.getNode() == Op0Hi.getNode() && Op0Lo.getResNo() == 0 &&
22558 Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() && Op0Hi.hasOneUse() &&
22559 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
22560 SDValue Result = DAG.getNode(
22561 Opcode: RISCVISD::WSUBA, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
22562 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op2);
22563 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
22564 }
22565 break;
22566 }
22567 case RISCVISD::FMV_W_X_RV64: {
22568 // If the input to FMV_W_X_RV64 is just FMV_X_ANYEXTW_RV64 the the
22569 // conversion is unnecessary and can be replaced with the
22570 // FMV_X_ANYEXTW_RV64 operand.
22571 SDValue Op0 = N->getOperand(Num: 0);
22572 if (Op0.getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64)
22573 return Op0.getOperand(i: 0);
22574 break;
22575 }
22576 case RISCVISD::FMV_X_ANYEXTH:
22577 case RISCVISD::FMV_X_ANYEXTW_RV64: {
22578 SDLoc DL(N);
22579 SDValue Op0 = N->getOperand(Num: 0);
22580 MVT VT = N->getSimpleValueType(ResNo: 0);
22581
22582 // Constant fold.
22583 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
22584 APInt Val = CFP->getValueAPF().bitcastToAPInt().sext(width: VT.getSizeInBits());
22585 return DAG.getConstant(Val, DL, VT);
22586 }
22587
22588 // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
22589 // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
22590 // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
22591 if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
22592 Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
22593 (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
22594 Op0->getOpcode() == RISCVISD::FMV_H_X)) {
22595 assert(Op0.getOperand(0).getValueType() == VT &&
22596 "Unexpected value type!");
22597 return Op0.getOperand(i: 0);
22598 }
22599
22600 if (ISD::isNormalLoad(N: Op0.getNode()) && Op0.hasOneUse() &&
22601 cast<LoadSDNode>(Val&: Op0)->isSimple()) {
22602 MVT IVT = MVT::getIntegerVT(BitWidth: Op0.getValueSizeInBits());
22603 auto *LN0 = cast<LoadSDNode>(Val&: Op0);
22604 SDValue Load =
22605 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: SDLoc(N), VT, Chain: LN0->getChain(),
22606 Ptr: LN0->getBasePtr(), MemVT: IVT, MMO: LN0->getMemOperand());
22607 DAG.ReplaceAllUsesOfValueWith(From: Op0.getValue(R: 1), To: Load.getValue(R: 1));
22608 return Load;
22609 }
22610
22611 // This is a target-specific version of a DAGCombine performed in
22612 // DAGCombiner::visitBITCAST. It performs the equivalent of:
22613 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
22614 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
22615 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
22616 !Op0.getNode()->hasOneUse())
22617 break;
22618 SDValue NewFMV = DAG.getNode(Opcode: N->getOpcode(), DL, VT, Operand: Op0.getOperand(i: 0));
22619 unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
22620 APInt SignBit = APInt::getSignMask(BitWidth: FPBits).sext(width: VT.getSizeInBits());
22621 if (Op0.getOpcode() == ISD::FNEG)
22622 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewFMV,
22623 N2: DAG.getConstant(Val: SignBit, DL, VT));
22624
22625 assert(Op0.getOpcode() == ISD::FABS);
22626 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NewFMV,
22627 N2: DAG.getConstant(Val: ~SignBit, DL, VT));
22628 }
22629 case ISD::ABS:
22630 case ISD::ABS_MIN_POISON: {
22631 EVT VT = N->getValueType(ResNo: 0);
22632 SDValue N0 = N->getOperand(Num: 0);
22633 // abs (sext) -> zext (abs)
22634 // abs (zext) -> zext (handled elsewhere)
22635 if (VT.isVector() && N0.hasOneUse() && N0.getOpcode() == ISD::SIGN_EXTEND) {
22636 SDValue Src = N0.getOperand(i: 0);
22637 SDLoc DL(N);
22638 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT,
22639 Operand: DAG.getNode(Opcode: ISD::ABS, DL, VT: Src.getValueType(), Operand: Src));
22640 }
22641 break;
22642 }
22643 case ISD::ADD: {
22644 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
22645 return V;
22646 if (SDValue V = combineToVWMACC(N, DAG, Subtarget))
22647 return V;
22648 if (SDValue V = combineVdot4aAccum(N, DAG, Subtarget))
22649 return V;
22650 return performADDCombine(N, DCI, Subtarget);
22651 }
22652 case ISD::SUB: {
22653 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
22654 return V;
22655 return performSUBCombine(N, DAG, Subtarget);
22656 }
22657 case ISD::AND:
22658 return performANDCombine(N, DCI, Subtarget);
22659 case ISD::OR: {
22660 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
22661 return V;
22662 return performORCombine(N, DCI, Subtarget);
22663 }
22664 case ISD::XOR:
22665 return performXORCombine(N, DAG, Subtarget);
22666 case ISD::MUL:
22667 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
22668 return V;
22669 return performMULCombine(N, DAG, DCI, Subtarget);
22670 case ISD::SDIV:
22671 case ISD::UDIV:
22672 case ISD::SREM:
22673 case ISD::UREM:
22674 if (SDValue V = combineBinOpOfZExt(N, DAG))
22675 return V;
22676 break;
22677 case ISD::FMUL: {
22678 using namespace SDPatternMatch;
22679 SDLoc DL(N);
22680 EVT VT = N->getValueType(ResNo: 0);
22681 SDValue X, Y;
22682 // InstCombine canonicalizes fneg (fmul x, y) -> fmul x, (fneg y), see
22683 // hoistFNegAboveFMulFDiv.
22684 // Undo this and sink the fneg so we match more fmsub/fnmadd patterns.
22685 if (sd_match(N, P: m_FMul(L: m_Value(N&: X), R: m_OneUse(P: m_FNeg(Op: m_Value(N&: Y))))))
22686 return DAG.getNode(Opcode: ISD::FNEG, DL, VT,
22687 Operand: DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: Y, Flags: N->getFlags()),
22688 Flags: N->getFlags());
22689
22690 // fmul X, (copysign 1.0, Y) -> fsgnjx X, Y
22691 SDValue N0 = N->getOperand(Num: 0);
22692 SDValue N1 = N->getOperand(Num: 1);
22693 if (N0->getOpcode() != ISD::FCOPYSIGN)
22694 std::swap(a&: N0, b&: N1);
22695 if (N0->getOpcode() != ISD::FCOPYSIGN)
22696 return SDValue();
22697 ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: N0->getOperand(Num: 0));
22698 if (!C || !C->getValueAPF().isOne())
22699 return SDValue();
22700 if (VT.isVector() || !isOperationLegal(Op: ISD::FCOPYSIGN, VT))
22701 return SDValue();
22702 SDValue Sign = N0->getOperand(Num: 1);
22703 if (Sign.getValueType() != VT)
22704 return SDValue();
22705 return DAG.getNode(Opcode: RISCVISD::FSGNJX, DL, VT, N1, N2: N0->getOperand(Num: 1));
22706 }
22707 case ISD::UMAX:
22708 case ISD::UMIN:
22709 case ISD::SMAX:
22710 case ISD::SMIN:
22711 if (SDValue V = combineMinMaxToSat(N, DCI, Subtarget))
22712 return V;
22713 [[fallthrough]];
22714 case ISD::FADD:
22715 case ISD::FMAXNUM:
22716 case ISD::FMINNUM: {
22717 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
22718 return V;
22719 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
22720 return V;
22721 return SDValue();
22722 }
22723 case ISD::FMA: {
22724 SDValue N0 = N->getOperand(Num: 0);
22725 SDValue N1 = N->getOperand(Num: 1);
22726 if (N0.getOpcode() != ISD::SPLAT_VECTOR)
22727 std::swap(a&: N0, b&: N1);
22728 if (N0.getOpcode() != ISD::SPLAT_VECTOR)
22729 return SDValue();
22730 SDValue SplatN0 = N0.getOperand(i: 0);
22731 if (SplatN0.getOpcode() != ISD::FNEG || !SplatN0.hasOneUse())
22732 return SDValue();
22733 EVT VT = N->getValueType(ResNo: 0);
22734 SDValue Splat =
22735 DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: SplatN0.getOperand(i: 0));
22736 SDValue Fneg = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Splat);
22737 return DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Fneg, N2: N1, N3: N->getOperand(Num: 2));
22738 }
22739 case ISD::SETCC:
22740 return performSETCCCombine(N, DCI, Subtarget);
22741 case ISD::SIGN_EXTEND_INREG:
22742 return performSIGN_EXTEND_INREGCombine(N, DCI, Subtarget);
22743 case ISD::ZERO_EXTEND:
22744 // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
22745 // type legalization. This is safe because fp_to_uint produces poison if
22746 // it overflows.
22747 if (N->getValueType(ResNo: 0) == MVT::i64 && Subtarget.is64Bit()) {
22748 SDValue Src = N->getOperand(Num: 0);
22749 if (Src.getOpcode() == ISD::FP_TO_UINT &&
22750 isTypeLegal(VT: Src.getOperand(i: 0).getValueType()))
22751 return DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SDLoc(N), VT: MVT::i64,
22752 Operand: Src.getOperand(i: 0));
22753 if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
22754 isTypeLegal(VT: Src.getOperand(i: 1).getValueType())) {
22755 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other);
22756 SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_TO_UINT, DL: SDLoc(N), VTList: VTs,
22757 N1: Src.getOperand(i: 0), N2: Src.getOperand(i: 1));
22758 DCI.CombineTo(N, Res);
22759 DAG.ReplaceAllUsesOfValueWith(From: Src.getValue(R: 1), To: Res.getValue(R: 1));
22760 DCI.recursivelyDeleteUnusedNodes(N: Src.getNode());
22761 return SDValue(N, 0); // Return N so it doesn't get rechecked.
22762 }
22763 }
22764 return SDValue();
22765 case RISCVISD::TRUNCATE_VECTOR_VL:
22766 if (SDValue V = combineTruncOfSraSext(N, DAG))
22767 return V;
22768 return combineTruncToVnclip(N, DAG, Subtarget);
22769 case ISD::TRUNCATE:
22770 return performTRUNCATECombine(N, DAG, Subtarget);
22771 case ISD::SELECT:
22772 return performSELECTCombine(N, DAG, Subtarget);
22773 case ISD::VSELECT:
22774 return performVSELECTCombine(N, DAG, Subtarget);
22775 case RISCVISD::CZERO_EQZ:
22776 case RISCVISD::CZERO_NEZ: {
22777 SDValue Val = N->getOperand(Num: 0);
22778 SDValue Cond = N->getOperand(Num: 1);
22779 MVT VT = N->getSimpleValueType(ResNo: 0);
22780
22781 unsigned Opc = N->getOpcode();
22782
22783 // czero_eqz x, x -> x
22784 if (Opc == RISCVISD::CZERO_EQZ && Val == Cond)
22785 return Val;
22786
22787 unsigned InvOpc =
22788 Opc == RISCVISD::CZERO_EQZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ;
22789
22790 // czero_eqz X, (xor Y, 1) -> czero_nez X, Y if Y is 0 or 1.
22791 // czero_nez X, (xor Y, 1) -> czero_eqz X, Y if Y is 0 or 1.
22792 if (Cond.getOpcode() == ISD::XOR && isOneConstant(V: Cond.getOperand(i: 1))) {
22793 SDValue NewCond = Cond.getOperand(i: 0);
22794 APInt Mask = APInt::getBitsSetFrom(numBits: NewCond.getValueSizeInBits(), loBit: 1);
22795 if (DAG.MaskedValueIsZero(Op: NewCond, Mask))
22796 return DAG.getNode(Opcode: InvOpc, DL: SDLoc(N), VT, N1: Val, N2: NewCond);
22797 }
22798 // czero_eqz x, (setcc y, 0, ne) -> czero_eqz x, y
22799 // czero_nez x, (setcc y, 0, ne) -> czero_nez x, y
22800 // czero_eqz x, (setcc y, 0, eq) -> czero_nez x, y
22801 // czero_nez x, (setcc y, 0, eq) -> czero_eqz x, y
22802 if (Cond.getOpcode() == ISD::SETCC && isNullConstant(V: Cond.getOperand(i: 1))) {
22803 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
22804 if (ISD::isIntEqualitySetCC(Code: CCVal))
22805 return DAG.getNode(Opcode: CCVal == ISD::SETNE ? Opc : InvOpc, DL: SDLoc(N), VT,
22806 N1: Val, N2: Cond.getOperand(i: 0));
22807 }
22808
22809 // Remove SRL from bittest patterns (srl (and X, (1 << C)), C) if the and
22810 // is an ANDI. Because only 1 bit can be set after the AND, it doesn't
22811 // matter if we shift it.
22812 if (Cond.getOpcode() == ISD::SRL &&
22813 isa<ConstantSDNode>(Val: Cond.getOperand(i: 1)) &&
22814 Cond.getOperand(i: 0).getOpcode() == ISD::AND) {
22815 const APInt &ShAmt = Cond.getConstantOperandAPInt(i: 1);
22816 unsigned BitWidth = VT.getSizeInBits();
22817 SDValue And = Cond.getOperand(i: 0);
22818 if (ShAmt.ult(RHS: BitWidth) && isa<ConstantSDNode>(Val: And.getOperand(i: 1))) {
22819 uint64_t AndConst = And.getConstantOperandVal(i: 1);
22820 if (AndConst == (1ULL << ShAmt.getZExtValue()) && isInt<12>(x: AndConst))
22821 return DAG.getNode(Opcode: Opc, DL, VT, N1: Val, N2: And);
22822 }
22823 }
22824
22825 // czero_nez (setcc X, Y, CC), (setcc X, Y, eq) -> (setcc X, Y, CC)
22826 // if CC is a strict inequality (lt, gt, ult, ugt), because when X == Y
22827 // the setcc result is already 0. The eq operands can be in either order.
22828 if (Opc == RISCVISD::CZERO_NEZ && Val.getOpcode() == ISD::SETCC &&
22829 Cond.getOpcode() == ISD::SETCC &&
22830 cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get() == ISD::SETEQ) {
22831 ISD::CondCode ValCC = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
22832 bool SameOperands = (Val.getOperand(i: 0) == Cond.getOperand(i: 0) &&
22833 Val.getOperand(i: 1) == Cond.getOperand(i: 1)) ||
22834 (Val.getOperand(i: 0) == Cond.getOperand(i: 1) &&
22835 Val.getOperand(i: 1) == Cond.getOperand(i: 0));
22836 if (SameOperands && (ValCC == ISD::SETLT || ValCC == ISD::SETGT ||
22837 ValCC == ISD::SETULT || ValCC == ISD::SETUGT))
22838 return Val;
22839 }
22840
22841 return SDValue();
22842 }
22843 case RISCVISD::SELECT_CC: {
22844 // Transform
22845 SDValue LHS = N->getOperand(Num: 0);
22846 SDValue RHS = N->getOperand(Num: 1);
22847 SDValue CC = N->getOperand(Num: 2);
22848 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
22849 SDValue TrueV = N->getOperand(Num: 3);
22850 SDValue FalseV = N->getOperand(Num: 4);
22851 SDLoc DL(N);
22852 EVT VT = N->getValueType(ResNo: 0);
22853
22854 // If the True and False values are the same, we don't need a select_cc.
22855 if (TrueV == FalseV)
22856 return TrueV;
22857
22858 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
22859 // (select (x >= 0), y, z) -> x >> (XLEN - 1) & (z - y) + y
22860 if (!Subtarget.hasShortForwardBranchIALU() && isa<ConstantSDNode>(Val: TrueV) &&
22861 isa<ConstantSDNode>(Val: FalseV) && isNullConstant(V: RHS) &&
22862 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
22863 if (CCVal == ISD::CondCode::SETGE)
22864 std::swap(a&: TrueV, b&: FalseV);
22865
22866 int64_t TrueSImm = cast<ConstantSDNode>(Val&: TrueV)->getSExtValue();
22867 int64_t FalseSImm = cast<ConstantSDNode>(Val&: FalseV)->getSExtValue();
22868 // Only handle simm12, if it is not in this range, it can be considered as
22869 // register.
22870 if (isInt<12>(x: TrueSImm) && isInt<12>(x: FalseSImm) &&
22871 isInt<12>(x: TrueSImm - FalseSImm)) {
22872 SDValue SRA =
22873 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS,
22874 N2: DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT));
22875 SDValue AND =
22876 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
22877 N2: DAG.getSignedConstant(Val: TrueSImm - FalseSImm, DL, VT));
22878 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND, N2: FalseV);
22879 }
22880
22881 if (CCVal == ISD::CondCode::SETGE)
22882 std::swap(a&: TrueV, b&: FalseV);
22883 }
22884
22885 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
22886 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
22887 Ops: {LHS, RHS, CC, TrueV, FalseV});
22888
22889 if (!Subtarget.hasConditionalMoveFusion()) {
22890 // (select c, -1, y) -> -c | y
22891 if (isAllOnesConstant(V: TrueV)) {
22892 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: CCVal);
22893 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22894 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: FalseV);
22895 }
22896 // (select c, y, -1) -> -!c | y
22897 if (isAllOnesConstant(V: FalseV)) {
22898 SDValue C =
22899 DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::getSetCCInverse(Operation: CCVal, Type: VT));
22900 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22901 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: TrueV);
22902 }
22903
22904 // (select c, 0, y) -> -!c & y
22905 if (isNullConstant(V: TrueV)) {
22906 SDValue C =
22907 DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::getSetCCInverse(Operation: CCVal, Type: VT));
22908 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22909 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: FalseV);
22910 }
22911 // (select c, y, 0) -> -c & y
22912 if (isNullConstant(V: FalseV)) {
22913 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: CCVal);
22914 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22915 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: TrueV);
22916 }
22917 // (riscvisd::select_cc x, 0, ne, x, 1) -> (add x, (setcc x, 0, eq))
22918 // (riscvisd::select_cc x, 0, eq, 1, x) -> (add x, (setcc x, 0, eq))
22919 if (((isOneConstant(V: FalseV) && LHS == TrueV &&
22920 CCVal == ISD::CondCode::SETNE) ||
22921 (isOneConstant(V: TrueV) && LHS == FalseV &&
22922 CCVal == ISD::CondCode::SETEQ)) &&
22923 isNullConstant(V: RHS)) {
22924 // freeze it to be safe.
22925 LHS = DAG.getFreeze(V: LHS);
22926 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::CondCode::SETEQ);
22927 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: C);
22928 }
22929 }
22930
22931 // If both true/false are an xor with 1, pull through the select.
22932 // This can occur after op legalization if both operands are setccs that
22933 // require an xor to invert.
22934 // FIXME: Generalize to other binary ops with identical operand?
22935 if (TrueV.getOpcode() == ISD::XOR && FalseV.getOpcode() == ISD::XOR &&
22936 TrueV.getOperand(i: 1) == FalseV.getOperand(i: 1) &&
22937 isOneConstant(V: TrueV.getOperand(i: 1)) &&
22938 TrueV.hasOneUse() && FalseV.hasOneUse()) {
22939 SDValue NewSel = DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, N1: LHS, N2: RHS, N3: CC,
22940 N4: TrueV.getOperand(i: 0), N5: FalseV.getOperand(i: 0));
22941 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewSel, N2: TrueV.getOperand(i: 1));
22942 }
22943
22944 return SDValue();
22945 }
22946 case RISCVISD::BR_CC: {
22947 SDValue LHS = N->getOperand(Num: 1);
22948 SDValue RHS = N->getOperand(Num: 2);
22949 SDValue CC = N->getOperand(Num: 3);
22950 SDLoc DL(N);
22951
22952 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
22953 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
22954 N1: N->getOperand(Num: 0), N2: LHS, N3: RHS, N4: CC, N5: N->getOperand(Num: 4));
22955
22956 return SDValue();
22957 }
22958 case ISD::BITREVERSE:
22959 return performBITREVERSECombine(N, DAG, Subtarget);
22960 case ISD::FP_TO_SINT:
22961 case ISD::FP_TO_UINT:
22962 return performFP_TO_INTCombine(N, DCI, Subtarget);
22963 case ISD::FP_TO_SINT_SAT:
22964 case ISD::FP_TO_UINT_SAT:
22965 return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
22966 case ISD::FCOPYSIGN: {
22967 EVT VT = N->getValueType(ResNo: 0);
22968 if (!VT.isVector())
22969 break;
22970 // There is a form of VFSGNJ which injects the negated sign of its second
22971 // operand. Try and bubble any FNEG up after the extend/round to produce
22972 // this optimized pattern. Avoid modifying cases where FP_ROUND and
22973 // TRUNC=1.
22974 SDValue In2 = N->getOperand(Num: 1);
22975 // Avoid cases where the extend/round has multiple uses, as duplicating
22976 // those is typically more expensive than removing a fneg.
22977 if (!In2.hasOneUse())
22978 break;
22979 if (In2.getOpcode() != ISD::FP_EXTEND &&
22980 (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(i: 1) != 0))
22981 break;
22982 In2 = In2.getOperand(i: 0);
22983 if (In2.getOpcode() != ISD::FNEG)
22984 break;
22985 SDLoc DL(N);
22986 SDValue NewFPExtRound = DAG.getFPExtendOrRound(Op: In2.getOperand(i: 0), DL, VT);
22987 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT, N1: N->getOperand(Num: 0),
22988 N2: DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: NewFPExtRound));
22989 }
22990 case ISD::MGATHER: {
22991 const auto *MGN = cast<MaskedGatherSDNode>(Val: N);
22992 const EVT VT = N->getValueType(ResNo: 0);
22993 SDValue Index = MGN->getIndex();
22994 SDValue ScaleOp = MGN->getScale();
22995 ISD::MemIndexType IndexType = MGN->getIndexType();
22996 assert(!MGN->isIndexScaled() &&
22997 "Scaled gather/scatter should not be formed");
22998
22999 SDLoc DL(N);
23000 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
23001 return DAG.getMaskedGather(
23002 VTs: N->getVTList(), MemVT: MGN->getMemoryVT(), dl: DL,
23003 Ops: {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
23004 MGN->getBasePtr(), Index, ScaleOp},
23005 MMO: MGN->getMemOperand(), IndexType, ExtTy: MGN->getExtensionType());
23006
23007 if (narrowIndex(N&: Index, IndexType, DAG))
23008 return DAG.getMaskedGather(
23009 VTs: N->getVTList(), MemVT: MGN->getMemoryVT(), dl: DL,
23010 Ops: {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
23011 MGN->getBasePtr(), Index, ScaleOp},
23012 MMO: MGN->getMemOperand(), IndexType, ExtTy: MGN->getExtensionType());
23013
23014 if (Index.getOpcode() == ISD::BUILD_VECTOR &&
23015 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) {
23016 // The sequence will be XLenVT, not the type of Index. Tell
23017 // isSimpleVIDSequence this so we avoid overflow.
23018 if (std::optional<VIDSequence> SimpleVID =
23019 isSimpleVIDSequence(Op: Index, EltSizeInBits: Subtarget.getXLen());
23020 SimpleVID && SimpleVID->StepDenominator == 1) {
23021 const int64_t StepNumerator = SimpleVID->StepNumerator;
23022 const int64_t Addend = SimpleVID->Addend;
23023
23024 // Note: We don't need to check alignment here since (by assumption
23025 // from the existence of the gather), our offsets must be sufficiently
23026 // aligned.
23027
23028 const EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
23029 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
23030 assert(IndexType == ISD::UNSIGNED_SCALED);
23031 SDValue BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: MGN->getBasePtr(),
23032 N2: DAG.getSignedConstant(Val: Addend, DL, VT: PtrVT));
23033
23034 SDValue EVL = DAG.getElementCount(DL, VT: Subtarget.getXLenVT(),
23035 EC: VT.getVectorElementCount());
23036 SDValue StridedLoad = DAG.getStridedLoadVP(
23037 VT, DL, Chain: MGN->getChain(), Ptr: BasePtr,
23038 Stride: DAG.getSignedConstant(Val: StepNumerator, DL, VT: XLenVT), Mask: MGN->getMask(),
23039 EVL, MMO: MGN->getMemOperand());
23040 SDValue Select = DAG.getSelect(DL, VT, Cond: MGN->getMask(), LHS: StridedLoad,
23041 RHS: MGN->getPassThru());
23042 return DAG.getMergeValues(Ops: {Select, SDValue(StridedLoad.getNode(), 1)},
23043 dl: DL);
23044 }
23045 }
23046
23047 SmallVector<int> ShuffleMask;
23048 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
23049 matchIndexAsShuffle(VT, Index, Mask: MGN->getMask(), ShuffleMask)) {
23050 SDValue Load = DAG.getMaskedLoad(VT, dl: DL, Chain: MGN->getChain(),
23051 Base: MGN->getBasePtr(), Offset: DAG.getUNDEF(VT: XLenVT),
23052 Mask: MGN->getMask(), Src0: DAG.getUNDEF(VT),
23053 MemVT: MGN->getMemoryVT(), MMO: MGN->getMemOperand(),
23054 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
23055 SDValue Shuffle =
23056 DAG.getVectorShuffle(VT, dl: DL, N1: Load, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
23057 return DAG.getMergeValues(Ops: {Shuffle, Load.getValue(R: 1)}, dl: DL);
23058 }
23059
23060 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
23061 matchIndexAsWiderOp(VT, Index, Mask: MGN->getMask(),
23062 BaseAlign: MGN->getMemOperand()->getBaseAlign(), ST: Subtarget)) {
23063 SmallVector<SDValue> NewIndices;
23064 for (unsigned i = 0; i < Index->getNumOperands(); i += 2)
23065 NewIndices.push_back(Elt: Index.getOperand(i));
23066 EVT IndexVT = Index.getValueType()
23067 .getHalfNumVectorElementsVT(Context&: *DAG.getContext());
23068 Index = DAG.getBuildVector(VT: IndexVT, DL, Ops: NewIndices);
23069
23070 unsigned ElementSize = VT.getScalarStoreSize();
23071 EVT WideScalarVT = MVT::getIntegerVT(BitWidth: ElementSize * 8 * 2);
23072 auto EltCnt = VT.getVectorElementCount();
23073 assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!");
23074 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideScalarVT,
23075 EC: EltCnt.divideCoefficientBy(RHS: 2));
23076 SDValue Passthru = DAG.getBitcast(VT: WideVT, V: MGN->getPassThru());
23077 EVT MaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
23078 EC: EltCnt.divideCoefficientBy(RHS: 2));
23079 SDValue Mask = DAG.getSplat(VT: MaskVT, DL, Op: DAG.getConstant(Val: 1, DL, VT: MVT::i1));
23080
23081 SDValue Gather =
23082 DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), MemVT: WideVT, dl: DL,
23083 Ops: {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
23084 Index, ScaleOp},
23085 MMO: MGN->getMemOperand(), IndexType, ExtTy: ISD::NON_EXTLOAD);
23086 SDValue Result = DAG.getBitcast(VT, V: Gather.getValue(R: 0));
23087 return DAG.getMergeValues(Ops: {Result, Gather.getValue(R: 1)}, dl: DL);
23088 }
23089 break;
23090 }
23091 case ISD::MSCATTER:{
23092 const auto *MSN = cast<MaskedScatterSDNode>(Val: N);
23093 SDValue Index = MSN->getIndex();
23094 SDValue ScaleOp = MSN->getScale();
23095 ISD::MemIndexType IndexType = MSN->getIndexType();
23096 assert(!MSN->isIndexScaled() &&
23097 "Scaled gather/scatter should not be formed");
23098
23099 SDLoc DL(N);
23100 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
23101 return DAG.getMaskedScatter(
23102 VTs: N->getVTList(), MemVT: MSN->getMemoryVT(), dl: DL,
23103 Ops: {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
23104 Index, ScaleOp},
23105 MMO: MSN->getMemOperand(), IndexType, IsTruncating: MSN->isTruncatingStore());
23106
23107 if (narrowIndex(N&: Index, IndexType, DAG))
23108 return DAG.getMaskedScatter(
23109 VTs: N->getVTList(), MemVT: MSN->getMemoryVT(), dl: DL,
23110 Ops: {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
23111 Index, ScaleOp},
23112 MMO: MSN->getMemOperand(), IndexType, IsTruncating: MSN->isTruncatingStore());
23113
23114 EVT VT = MSN->getValue()->getValueType(ResNo: 0);
23115 SmallVector<int> ShuffleMask;
23116 if (!MSN->isTruncatingStore() &&
23117 matchIndexAsShuffle(VT, Index, Mask: MSN->getMask(), ShuffleMask)) {
23118 SDValue Shuffle = DAG.getVectorShuffle(VT, dl: DL, N1: MSN->getValue(),
23119 N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
23120 return DAG.getMaskedStore(Chain: MSN->getChain(), dl: DL, Val: Shuffle, Base: MSN->getBasePtr(),
23121 Offset: DAG.getUNDEF(VT: XLenVT), Mask: MSN->getMask(),
23122 MemVT: MSN->getMemoryVT(), MMO: MSN->getMemOperand(),
23123 AM: ISD::UNINDEXED, IsTruncating: false);
23124 }
23125 break;
23126 }
23127 case ISD::VP_GATHER: {
23128 const auto *VPGN = cast<VPGatherSDNode>(Val: N);
23129 SDValue Index = VPGN->getIndex();
23130 SDValue ScaleOp = VPGN->getScale();
23131 ISD::MemIndexType IndexType = VPGN->getIndexType();
23132 assert(!VPGN->isIndexScaled() &&
23133 "Scaled gather/scatter should not be formed");
23134
23135 SDLoc DL(N);
23136 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
23137 return DAG.getGatherVP(VTs: N->getVTList(), VT: VPGN->getMemoryVT(), dl: DL,
23138 Ops: {VPGN->getChain(), VPGN->getBasePtr(), Index,
23139 ScaleOp, VPGN->getMask(),
23140 VPGN->getVectorLength()},
23141 MMO: VPGN->getMemOperand(), IndexType);
23142
23143 if (narrowIndex(N&: Index, IndexType, DAG))
23144 return DAG.getGatherVP(VTs: N->getVTList(), VT: VPGN->getMemoryVT(), dl: DL,
23145 Ops: {VPGN->getChain(), VPGN->getBasePtr(), Index,
23146 ScaleOp, VPGN->getMask(),
23147 VPGN->getVectorLength()},
23148 MMO: VPGN->getMemOperand(), IndexType);
23149
23150 break;
23151 }
23152 case ISD::VP_SCATTER: {
23153 const auto *VPSN = cast<VPScatterSDNode>(Val: N);
23154 SDValue Index = VPSN->getIndex();
23155 SDValue ScaleOp = VPSN->getScale();
23156 ISD::MemIndexType IndexType = VPSN->getIndexType();
23157 assert(!VPSN->isIndexScaled() &&
23158 "Scaled gather/scatter should not be formed");
23159
23160 SDLoc DL(N);
23161 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
23162 return DAG.getScatterVP(VTs: N->getVTList(), VT: VPSN->getMemoryVT(), dl: DL,
23163 Ops: {VPSN->getChain(), VPSN->getValue(),
23164 VPSN->getBasePtr(), Index, ScaleOp,
23165 VPSN->getMask(), VPSN->getVectorLength()},
23166 MMO: VPSN->getMemOperand(), IndexType);
23167
23168 if (narrowIndex(N&: Index, IndexType, DAG))
23169 return DAG.getScatterVP(VTs: N->getVTList(), VT: VPSN->getMemoryVT(), dl: DL,
23170 Ops: {VPSN->getChain(), VPSN->getValue(),
23171 VPSN->getBasePtr(), Index, ScaleOp,
23172 VPSN->getMask(), VPSN->getVectorLength()},
23173 MMO: VPSN->getMemOperand(), IndexType);
23174 break;
23175 }
23176 case RISCVISD::SHL_VL:
23177 if (SDValue V = performSHLCombine(N, DCI, Subtarget))
23178 return V;
23179 [[fallthrough]];
23180 case RISCVISD::SRA_VL:
23181 case RISCVISD::SRL_VL: {
23182 SDValue ShAmt = N->getOperand(Num: 1);
23183 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
23184 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
23185 SDLoc DL(N);
23186 SDValue VL = N->getOperand(Num: 4);
23187 EVT VT = N->getValueType(ResNo: 0);
23188 ShAmt = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
23189 N2: ShAmt.getOperand(i: 1), N3: VL);
23190 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N->getOperand(Num: 0), N2: ShAmt,
23191 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
23192 }
23193 break;
23194 }
23195 case ISD::SRA:
23196 if (SDValue V = performSRACombine(N, DAG, Subtarget))
23197 return V;
23198 [[fallthrough]];
23199 case ISD::SRL:
23200 case ISD::SHL: {
23201 if (N->getOpcode() == ISD::SHL) {
23202 if (SDValue V = performSHLCombine(N, DCI, Subtarget))
23203 return V;
23204 }
23205 SDValue ShAmt = N->getOperand(Num: 1);
23206 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
23207 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
23208 SDLoc DL(N);
23209 EVT VT = N->getValueType(ResNo: 0);
23210 ShAmt = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
23211 N2: ShAmt.getOperand(i: 1),
23212 N3: DAG.getRegister(Reg: RISCV::X0, VT: Subtarget.getXLenVT()));
23213 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N->getOperand(Num: 0), N2: ShAmt);
23214 }
23215 break;
23216 }
23217 case RISCVISD::ADD_VL:
23218 if (SDValue V = simplifyOp_VL(N))
23219 return V;
23220 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
23221 return V;
23222 if (SDValue V = combineVdot4aAccum(N, DAG, Subtarget))
23223 return V;
23224 return combineToVWMACC(N, DAG, Subtarget);
23225 case RISCVISD::VWADDU_VL:
23226 return performVWABDACombine(N, DAG, Subtarget);
23227 case RISCVISD::VWADDU_W_VL:
23228 if (SDValue V = performVWABDACombineWV(N, DAG, Subtarget))
23229 return V;
23230 [[fallthrough]];
23231 case RISCVISD::VWADD_W_VL:
23232 case RISCVISD::VWSUB_W_VL:
23233 case RISCVISD::VWSUBU_W_VL:
23234 return performVWADDSUBW_VLCombine(N, DCI, Subtarget);
23235 case RISCVISD::OR_VL:
23236 case RISCVISD::SUB_VL:
23237 case RISCVISD::MUL_VL:
23238 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
23239 case RISCVISD::VFMADD_VL:
23240 case RISCVISD::VFNMADD_VL:
23241 case RISCVISD::VFMSUB_VL:
23242 case RISCVISD::VFNMSUB_VL:
23243 case RISCVISD::STRICT_VFMADD_VL:
23244 case RISCVISD::STRICT_VFNMADD_VL:
23245 case RISCVISD::STRICT_VFMSUB_VL:
23246 case RISCVISD::STRICT_VFNMSUB_VL:
23247 return performVFMADD_VLCombine(N, DCI, Subtarget);
23248 case RISCVISD::FADD_VL:
23249 case RISCVISD::FSUB_VL:
23250 case RISCVISD::FMUL_VL:
23251 case RISCVISD::VFWADD_W_VL:
23252 case RISCVISD::VFWSUB_W_VL:
23253 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
23254 case ISD::LOAD:
23255 case ISD::STORE: {
23256 if (DCI.isAfterLegalizeDAG())
23257 if (SDValue V = performMemPairCombine(N, DCI))
23258 return V;
23259
23260 if (N->getOpcode() != ISD::STORE)
23261 break;
23262
23263 auto *Store = cast<StoreSDNode>(Val: N);
23264 SDValue Chain = Store->getChain();
23265 EVT MemVT = Store->getMemoryVT();
23266 SDValue Val = Store->getValue();
23267 SDLoc DL(N);
23268
23269 bool IsScalarizable =
23270 MemVT.isFixedLengthVector() && ISD::isNormalStore(N: Store) &&
23271 Store->isSimple() &&
23272 MemVT.getVectorElementType().bitsLE(VT: Subtarget.getXLenVT()) &&
23273 isPowerOf2_64(Value: MemVT.getSizeInBits()) &&
23274 MemVT.getSizeInBits() <= Subtarget.getXLen();
23275
23276 // If sufficiently aligned we can scalarize stores of constant vectors of
23277 // any power-of-two size up to XLen bits, provided that they aren't too
23278 // expensive to materialize.
23279 // vsetivli zero, 2, e8, m1, ta, ma
23280 // vmv.v.i v8, 4
23281 // vse64.v v8, (a0)
23282 // ->
23283 // li a1, 1028
23284 // sh a1, 0(a0)
23285 if (DCI.isBeforeLegalize() && IsScalarizable &&
23286 ISD::isBuildVectorOfConstantSDNodes(N: Val.getNode())) {
23287 // Get the constant vector bits
23288 APInt NewC(Val.getValueSizeInBits(), 0);
23289 uint64_t EltSize = Val.getScalarValueSizeInBits();
23290 for (unsigned i = 0; i < Val.getNumOperands(); i++) {
23291 if (Val.getOperand(i).isUndef())
23292 continue;
23293 NewC.insertBits(SubBits: Val.getConstantOperandAPInt(i).trunc(width: EltSize),
23294 bitPosition: i * EltSize);
23295 }
23296 MVT NewVT = MVT::getIntegerVT(BitWidth: MemVT.getSizeInBits());
23297
23298 if (RISCVMatInt::getIntMatCost(Val: NewC, Size: Subtarget.getXLen(), STI: Subtarget,
23299 CompressionCost: true) <= 2 &&
23300 allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
23301 VT: NewVT, MMO: *Store->getMemOperand())) {
23302 SDValue NewV = DAG.getConstant(Val: NewC, DL, VT: NewVT);
23303 return DAG.getStore(Chain, dl: DL, Val: NewV, Ptr: Store->getBasePtr(),
23304 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
23305 MMOFlags: Store->getMemOperand()->getFlags());
23306 }
23307 }
23308
23309 // Similarly, if sufficiently aligned we can scalarize vector copies, e.g.
23310 // vsetivli zero, 2, e16, m1, ta, ma
23311 // vle16.v v8, (a0)
23312 // vse16.v v8, (a1)
23313 if (auto *L = dyn_cast<LoadSDNode>(Val);
23314 L && DCI.isBeforeLegalize() && IsScalarizable && L->isSimple() &&
23315 L->hasNUsesOfValue(NUses: 1, Value: 0) && L->hasNUsesOfValue(NUses: 1, Value: 1) &&
23316 Store->getChain() == SDValue(L, 1) && ISD::isNormalLoad(N: L) &&
23317 L->getMemoryVT() == MemVT) {
23318 MVT NewVT = MVT::getIntegerVT(BitWidth: MemVT.getSizeInBits());
23319 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
23320 VT: NewVT, MMO: *Store->getMemOperand()) &&
23321 allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
23322 VT: NewVT, MMO: *L->getMemOperand())) {
23323 SDValue NewL = DAG.getLoad(VT: NewVT, dl: DL, Chain: L->getChain(), Ptr: L->getBasePtr(),
23324 PtrInfo: L->getPointerInfo(), Alignment: L->getBaseAlign(),
23325 MMOFlags: L->getMemOperand()->getFlags());
23326 return DAG.getStore(Chain, dl: DL, Val: NewL, Ptr: Store->getBasePtr(),
23327 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
23328 MMOFlags: Store->getMemOperand()->getFlags());
23329 }
23330 }
23331
23332 // Combine store of vmv.x.s/vfmv.f.s to vse with VL of 1.
23333 // vfmv.f.s is represented as extract element from 0. Match it late to avoid
23334 // any illegal types.
23335 if ((Val.getOpcode() == RISCVISD::VMV_X_S ||
23336 (DCI.isAfterLegalizeDAG() &&
23337 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
23338 isNullConstant(V: Val.getOperand(i: 1)))) &&
23339 Val.hasOneUse()) {
23340 SDValue Src = Val.getOperand(i: 0);
23341 EVT VecVT = Src.getValueType();
23342 // VecVT should be scalable and memory VT should match the element type.
23343 if (!Store->isIndexed() && Store->isSimple() &&
23344 VecVT.isScalableVectorOf(EltVT: MemVT)) {
23345 SDLoc DL(N);
23346 MVT MaskVT = getMaskTypeFor(VecVT: VecVT.getSimpleVT());
23347 // Create a vector memory VT so allowsMisalignedMemoryAccesses will
23348 // work correctly.
23349 MemVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MemVT, NumElements: 1);
23350 return DAG.getStoreVP(
23351 Chain: Store->getChain(), dl: DL, Val: Src, Ptr: Store->getBasePtr(), Offset: Store->getOffset(),
23352 Mask: DAG.getConstant(Val: 1, DL, VT: MaskVT),
23353 EVL: DAG.getConstant(Val: 1, DL, VT: Subtarget.getXLenVT()), MemVT,
23354 MMO: Store->getMemOperand(), AM: Store->getAddressingMode());
23355 }
23356 }
23357
23358 break;
23359 }
23360 case ISD::SPLAT_VECTOR: {
23361 EVT VT = N->getValueType(ResNo: 0);
23362 // Only perform this combine on legal MVT types.
23363 if (!isTypeLegal(VT))
23364 break;
23365 if (auto Gather = matchSplatAsGather(SplatVal: N->getOperand(Num: 0), VT: VT.getSimpleVT(), DL: N,
23366 DAG, Subtarget))
23367 return Gather;
23368 break;
23369 }
23370 case ISD::BUILD_VECTOR:
23371 if (SDValue V = performBUILD_VECTORCombine(N, DAG, Subtarget, TLI: *this))
23372 return V;
23373 break;
23374 case ISD::CONCAT_VECTORS:
23375 if (SDValue V = performCONCAT_VECTORSCombine(N, DAG, Subtarget, TLI: *this))
23376 return V;
23377 break;
23378 case ISD::VECTOR_SHUFFLE:
23379 if (SDValue V = performVECTOR_SHUFFLECombine(N, DAG, Subtarget, TLI: *this))
23380 return V;
23381 break;
23382 case ISD::INSERT_VECTOR_ELT:
23383 if (SDValue V = performINSERT_VECTOR_ELTCombine(N, DAG, Subtarget, TLI: *this))
23384 return V;
23385 break;
23386 case RISCVISD::VFMV_V_F_VL: {
23387 const MVT VT = N->getSimpleValueType(ResNo: 0);
23388 SDValue Passthru = N->getOperand(Num: 0);
23389 SDValue Scalar = N->getOperand(Num: 1);
23390 SDValue VL = N->getOperand(Num: 2);
23391
23392 // If VL is 1, we can use vfmv.s.f.
23393 if (isOneConstant(V: VL))
23394 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
23395 break;
23396 }
23397 case RISCVISD::VMV_V_X_VL: {
23398 const MVT VT = N->getSimpleValueType(ResNo: 0);
23399 SDValue Passthru = N->getOperand(Num: 0);
23400 SDValue Scalar = N->getOperand(Num: 1);
23401 SDValue VL = N->getOperand(Num: 2);
23402
23403 // Tail agnostic VMV.V.X only demands the vector element bitwidth from the
23404 // scalar input.
23405 unsigned ScalarSize = Scalar.getValueSizeInBits();
23406 unsigned EltWidth = VT.getScalarSizeInBits();
23407 if (ScalarSize > EltWidth && Passthru.isUndef())
23408 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
23409 return SDValue(N, 0);
23410
23411 // If VL is 1 and the scalar value won't benefit from immediate, we can
23412 // use vmv.s.x.
23413 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: Scalar);
23414 if (isOneConstant(V: VL) &&
23415 (!Const || Const->isZero() ||
23416 !Const->getAPIntValue().sextOrTrunc(width: EltWidth).isSignedIntN(N: 5)))
23417 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
23418
23419 break;
23420 }
23421 case RISCVISD::VFMV_S_F_VL: {
23422 SDValue Src = N->getOperand(Num: 1);
23423 // Try to remove vector->scalar->vector if the scalar->vector is inserting
23424 // into an undef vector.
23425 // TODO: Could use a vslide or vmv.v.v for non-undef.
23426 if (N->getOperand(Num: 0).isUndef() &&
23427 Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
23428 isNullConstant(V: Src.getOperand(i: 1)) &&
23429 Src.getOperand(i: 0).getValueType().isScalableVector()) {
23430 EVT VT = N->getValueType(ResNo: 0);
23431 SDValue EVSrc = Src.getOperand(i: 0);
23432 EVT EVSrcVT = EVSrc.getValueType();
23433 assert(EVSrcVT.getVectorElementType() == VT.getVectorElementType());
23434 // Widths match, just return the original vector.
23435 if (EVSrcVT == VT)
23436 return EVSrc;
23437 SDLoc DL(N);
23438 // Width is narrower, using insert_subvector.
23439 if (EVSrcVT.getVectorMinNumElements() < VT.getVectorMinNumElements()) {
23440 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: DAG.getUNDEF(VT),
23441 N2: EVSrc,
23442 N3: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()));
23443 }
23444 // Width is wider, using extract_subvector.
23445 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: EVSrc,
23446 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()));
23447 }
23448 [[fallthrough]];
23449 }
23450 case RISCVISD::VMV_S_X_VL: {
23451 const MVT VT = N->getSimpleValueType(ResNo: 0);
23452 SDValue Passthru = N->getOperand(Num: 0);
23453 SDValue Scalar = N->getOperand(Num: 1);
23454 SDValue VL = N->getOperand(Num: 2);
23455
23456 // The vmv.s.x instruction copies the scalar integer register to element 0
23457 // of the destination vector register. If SEW < XLEN, the least-significant
23458 // bits are copied and the upper XLEN-SEW bits are ignored.
23459 unsigned ScalarSize = Scalar.getValueSizeInBits();
23460 unsigned EltWidth = VT.getScalarSizeInBits();
23461 if (ScalarSize > EltWidth && SimplifyDemandedLowBitsHelper(1, EltWidth))
23462 return SDValue(N, 0);
23463
23464 if (Scalar.getOpcode() == RISCVISD::VMV_X_S && Passthru.isUndef() &&
23465 Scalar.getOperand(i: 0).getValueType() == N->getValueType(ResNo: 0))
23466 return Scalar.getOperand(i: 0);
23467
23468 // Use M1 or smaller to avoid over constraining register allocation
23469 const MVT M1VT = RISCVTargetLowering::getM1VT(VT);
23470 if (M1VT.bitsLT(VT)) {
23471 SDValue M1Passthru = DAG.getExtractSubvector(DL, VT: M1VT, Vec: Passthru, Idx: 0);
23472 SDValue Result =
23473 DAG.getNode(Opcode: N->getOpcode(), DL, VT: M1VT, N1: M1Passthru, N2: Scalar, N3: VL);
23474 Result = DAG.getInsertSubvector(DL, Vec: Passthru, SubVec: Result, Idx: 0);
23475 return Result;
23476 }
23477
23478 // We use a vmv.v.i if possible. We limit this to LMUL1. LMUL2 or
23479 // higher would involve overly constraining the register allocator for
23480 // no purpose.
23481 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: Scalar);
23482 Const && !Const->isZero() && isInt<5>(x: Const->getSExtValue()) &&
23483 VT.bitsLE(VT: RISCVTargetLowering::getM1VT(VT)) && Passthru.isUndef())
23484 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
23485
23486 break;
23487 }
23488 case RISCVISD::VMV_X_S: {
23489 SDValue Vec = N->getOperand(Num: 0);
23490 MVT VecVT = N->getOperand(Num: 0).getSimpleValueType();
23491 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: VecVT);
23492 if (M1VT.bitsLT(VT: VecVT)) {
23493 Vec = DAG.getExtractSubvector(DL, VT: M1VT, Vec, Idx: 0);
23494 return DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: N->getValueType(ResNo: 0), Operand: Vec);
23495 }
23496 break;
23497 }
23498 case ISD::INTRINSIC_VOID:
23499 case ISD::INTRINSIC_W_CHAIN:
23500 case ISD::INTRINSIC_WO_CHAIN: {
23501 unsigned IntOpNo = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
23502 unsigned IntNo = N->getConstantOperandVal(Num: IntOpNo);
23503 switch (IntNo) {
23504 // By default we do not combine any intrinsic.
23505 default:
23506 return SDValue();
23507 case Intrinsic::riscv_vcpop:
23508 case Intrinsic::riscv_vcpop_mask:
23509 case Intrinsic::riscv_vfirst:
23510 case Intrinsic::riscv_vfirst_mask: {
23511 SDValue VL = N->getOperand(Num: 2);
23512 if (IntNo == Intrinsic::riscv_vcpop_mask ||
23513 IntNo == Intrinsic::riscv_vfirst_mask)
23514 VL = N->getOperand(Num: 3);
23515 if (!isNullConstant(V: VL))
23516 return SDValue();
23517 // If VL is 0, vcpop -> li 0, vfirst -> li -1.
23518 SDLoc DL(N);
23519 EVT VT = N->getValueType(ResNo: 0);
23520 if (IntNo == Intrinsic::riscv_vfirst ||
23521 IntNo == Intrinsic::riscv_vfirst_mask)
23522 return DAG.getAllOnesConstant(DL, VT);
23523 return DAG.getConstant(Val: 0, DL, VT);
23524 }
23525 case Intrinsic::riscv_vsseg2_mask:
23526 case Intrinsic::riscv_vsseg3_mask:
23527 case Intrinsic::riscv_vsseg4_mask:
23528 case Intrinsic::riscv_vsseg5_mask:
23529 case Intrinsic::riscv_vsseg6_mask:
23530 case Intrinsic::riscv_vsseg7_mask:
23531 case Intrinsic::riscv_vsseg8_mask: {
23532 SDValue Tuple = N->getOperand(Num: 2);
23533 unsigned NF = Tuple.getValueType().getRISCVVectorTupleNumFields();
23534
23535 if (Subtarget.hasOptimizedSegmentLoadStore(NF) || !Tuple.hasOneUse() ||
23536 Tuple.getOpcode() != RISCVISD::TUPLE_INSERT ||
23537 !Tuple.getOperand(i: 0).isUndef())
23538 return SDValue();
23539
23540 SDValue Val = Tuple.getOperand(i: 1);
23541 unsigned Idx = Tuple.getConstantOperandVal(i: 2);
23542
23543 unsigned SEW = Val.getValueType().getScalarSizeInBits();
23544 assert(Log2_64(SEW) == N->getConstantOperandVal(6) &&
23545 "Type mismatch without bitcast?");
23546 unsigned Stride = SEW / 8 * NF;
23547 unsigned Offset = SEW / 8 * Idx;
23548
23549 SDValue Ops[] = {
23550 /*Chain=*/N->getOperand(Num: 0),
23551 /*IntID=*/
23552 DAG.getTargetConstant(Val: Intrinsic::riscv_vsse_mask, DL, VT: XLenVT),
23553 /*StoredVal=*/Val,
23554 /*Ptr=*/
23555 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: N->getOperand(Num: 3),
23556 N2: DAG.getConstant(Val: Offset, DL, VT: XLenVT)),
23557 /*Stride=*/DAG.getConstant(Val: Stride, DL, VT: XLenVT),
23558 /*Mask=*/N->getOperand(Num: 4),
23559 /*VL=*/N->getOperand(Num: 5)};
23560
23561 auto *OldMemSD = cast<MemIntrinsicSDNode>(Val: N);
23562 // Match getTgtMemIntrinsic for non-unit stride case
23563 EVT MemVT = OldMemSD->getMemoryVT().getScalarType();
23564 MachineFunction &MF = DAG.getMachineFunction();
23565 MachineMemOperand *MMO = MF.getMachineMemOperand(
23566 MMO: OldMemSD->getMemOperand(), Offset, Size: MemoryLocation::UnknownSize);
23567
23568 SDVTList VTs = DAG.getVTList(VT: MVT::Other);
23569 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: VTs, Ops, MemVT,
23570 MMO);
23571 }
23572 }
23573 }
23574 case ISD::VECTOR_SPLICE_RIGHT:
23575 case ISD::EXPERIMENTAL_VP_REVERSE:
23576 return performReverseEVLCombine(N, DCI, Subtarget);
23577 case ISD::VP_STORE:
23578 return performVP_STORECombine(N, DAG, Subtarget);
23579 case ISD::BITCAST: {
23580 if (Subtarget.hasStdExtP())
23581 if (SDValue V = performP_BITCASTCombine(N, DAG, Subtarget))
23582 return V;
23583 if (!Subtarget.useRVVForFixedLengthVectors())
23584 return SDValue();
23585 SDValue N0 = N->getOperand(Num: 0);
23586 EVT VT = N->getValueType(ResNo: 0);
23587 EVT SrcVT = N0.getValueType();
23588 if (VT.isRISCVVectorTuple() && N0->getOpcode() == ISD::SPLAT_VECTOR) {
23589 unsigned NF = VT.getRISCVVectorTupleNumFields();
23590 unsigned NumScalElts = VT.getSizeInBits().getKnownMinValue() / (NF * 8);
23591 SDValue EltVal = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
23592 MVT ScalTy = MVT::getScalableVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: NumScalElts);
23593
23594 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: ScalTy, Operand: EltVal);
23595
23596 SDValue Result = DAG.getUNDEF(VT);
23597 for (unsigned i = 0; i < NF; ++i)
23598 Result = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT, N1: Result, N2: Splat,
23599 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
23600 return Result;
23601 }
23602 // If this is a bitcast between a MVT::v4i1/v2i1/v1i1 and an illegal integer
23603 // type, widen both sides to avoid a trip through memory.
23604 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
23605 VT.isScalarInteger()) {
23606 unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
23607 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT: SrcVT));
23608 Ops[0] = N0;
23609 SDLoc DL(N);
23610 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i1, Ops);
23611 N0 = DAG.getBitcast(VT: MVT::i8, V: N0);
23612 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0);
23613 }
23614
23615 return SDValue();
23616 }
23617 case ISD::VECREDUCE_ADD:
23618 if (SDValue V = performVECREDUCECombine(N, DAG, Subtarget, TLI: *this))
23619 return V;
23620 [[fallthrough]];
23621 case ISD::CTPOP:
23622 if (SDValue V = combineToVCPOP(N, DAG, Subtarget))
23623 return V;
23624 break;
23625 case RISCVISD::VRGATHER_VX_VL: {
23626 // Note this assumes that out of bounds indices produce poison
23627 // and can thus be replaced without having to prove them inbounds..
23628 EVT VT = N->getValueType(ResNo: 0);
23629 SDValue Src = N->getOperand(Num: 0);
23630 SDValue Idx = N->getOperand(Num: 1);
23631 SDValue Passthru = N->getOperand(Num: 2);
23632 SDValue VL = N->getOperand(Num: 4);
23633
23634 // Warning: Unlike most cases we strip an insert_subvector, this one
23635 // does not require the first operand to be undef.
23636 if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
23637 isNullConstant(V: Src.getOperand(i: 2)))
23638 Src = Src.getOperand(i: 1);
23639
23640 switch (Src.getOpcode()) {
23641 default:
23642 break;
23643 case RISCVISD::VMV_V_X_VL:
23644 case RISCVISD::VFMV_V_F_VL:
23645 // Drop a redundant vrgather_vx.
23646 // TODO: Remove the type restriction if we find a motivating
23647 // test case?
23648 if (Passthru.isUndef() && VL == Src.getOperand(i: 2) &&
23649 Src.getValueType() == VT)
23650 return Src;
23651 break;
23652 case RISCVISD::VMV_S_X_VL:
23653 case RISCVISD::VFMV_S_F_VL:
23654 // If this use only demands lane zero from the source vmv.s.x, and
23655 // doesn't have a passthru, then this vrgather.vi/vx is equivalent to
23656 // a vmv.v.x. Note that there can be other uses of the original
23657 // vmv.s.x and thus we can't eliminate it. (vfmv.s.f is analogous)
23658 if (isNullConstant(V: Idx) && Passthru.isUndef() &&
23659 VL == Src.getOperand(i: 2)) {
23660 unsigned Opc =
23661 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
23662 return DAG.getNode(Opcode: Opc, DL, VT, N1: DAG.getUNDEF(VT), N2: Src.getOperand(i: 1),
23663 N3: VL);
23664 }
23665 break;
23666 }
23667 break;
23668 }
23669 case RISCVISD::TUPLE_EXTRACT: {
23670 EVT VT = N->getValueType(ResNo: 0);
23671 SDValue Tuple = N->getOperand(Num: 0);
23672 unsigned Idx = N->getConstantOperandVal(Num: 1);
23673 if (!Tuple.hasOneUse() || Tuple.getOpcode() != ISD::INTRINSIC_W_CHAIN)
23674 break;
23675
23676 unsigned NF = 0;
23677 switch (Tuple.getConstantOperandVal(i: 1)) {
23678 default:
23679 break;
23680 case Intrinsic::riscv_vlseg2_mask:
23681 case Intrinsic::riscv_vlseg3_mask:
23682 case Intrinsic::riscv_vlseg4_mask:
23683 case Intrinsic::riscv_vlseg5_mask:
23684 case Intrinsic::riscv_vlseg6_mask:
23685 case Intrinsic::riscv_vlseg7_mask:
23686 case Intrinsic::riscv_vlseg8_mask:
23687 NF = Tuple.getValueType().getRISCVVectorTupleNumFields();
23688 break;
23689 }
23690
23691 if (!NF || Subtarget.hasOptimizedSegmentLoadStore(NF))
23692 break;
23693
23694 unsigned SEW = VT.getScalarSizeInBits();
23695 assert(Log2_64(SEW) == Tuple.getConstantOperandVal(7) &&
23696 "Type mismatch without bitcast?");
23697 unsigned Stride = SEW / 8 * NF;
23698 unsigned Offset = SEW / 8 * Idx;
23699
23700 SDValue Passthru = Tuple.getOperand(i: 2);
23701 if (Passthru.isUndef())
23702 Passthru = DAG.getUNDEF(VT);
23703 else
23704 Passthru = DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT, N1: Passthru,
23705 N2: N->getOperand(Num: 1));
23706
23707 SDValue Ops[] = {
23708 /*Chain=*/Tuple.getOperand(i: 0),
23709 /*IntID=*/DAG.getTargetConstant(Val: Intrinsic::riscv_vlse_mask, DL, VT: XLenVT),
23710 /*Passthru=*/Passthru,
23711 /*Ptr=*/
23712 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: Tuple.getOperand(i: 3),
23713 N2: DAG.getConstant(Val: Offset, DL, VT: XLenVT)),
23714 /*Stride=*/DAG.getConstant(Val: Stride, DL, VT: XLenVT),
23715 /*Mask=*/Tuple.getOperand(i: 4),
23716 /*VL=*/Tuple.getOperand(i: 5),
23717 /*Policy=*/Tuple.getOperand(i: 6)};
23718
23719 auto *TupleMemSD = cast<MemIntrinsicSDNode>(Val&: Tuple);
23720 // Match getTgtMemIntrinsic for non-unit stride case
23721 EVT MemVT = TupleMemSD->getMemoryVT().getScalarType();
23722 MachineFunction &MF = DAG.getMachineFunction();
23723 MachineMemOperand *MMO = MF.getMachineMemOperand(
23724 MMO: TupleMemSD->getMemOperand(), Offset, Size: MemoryLocation::UnknownSize);
23725
23726 SDVTList VTs = DAG.getVTList(VTs: {VT, MVT::Other});
23727 SDValue Result = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs,
23728 Ops, MemVT, MMO);
23729 DAG.ReplaceAllUsesOfValueWith(From: Tuple.getValue(R: 1), To: Result.getValue(R: 1));
23730 return Result.getValue(R: 0);
23731 }
23732 case RISCVISD::TUPLE_INSERT: {
23733 // tuple_insert tuple, undef, idx -> tuple
23734 if (N->getOperand(Num: 1).isUndef())
23735 return N->getOperand(Num: 0);
23736 break;
23737 }
23738 case RISCVISD::VMERGE_VL: {
23739 // vmerge_vl allones, x, y, passthru, vl -> vmv_v_v passthru, x, vl
23740 SDValue Mask = N->getOperand(Num: 0);
23741 SDValue True = N->getOperand(Num: 1);
23742 SDValue Passthru = N->getOperand(Num: 3);
23743 SDValue VL = N->getOperand(Num: 4);
23744
23745 // Fixed vectors are wrapped in scalable containers, unwrap them.
23746 using namespace SDPatternMatch;
23747 SDValue SubVec;
23748 if (sd_match(N: Mask, P: m_InsertSubvector(Base: m_Undef(), Sub: m_Value(N&: SubVec), Idx: m_Zero())))
23749 Mask = SubVec;
23750
23751 if (!isOneOrOneSplat(V: Mask))
23752 break;
23753
23754 return DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
23755 N1: Passthru, N2: True, N3: VL);
23756 }
23757 case RISCVISD::VMV_V_V_VL: {
23758 // vmv_v_v passthru, splat(x), vl -> vmv_v_x passthru, x, vl
23759 SDValue Passthru = N->getOperand(Num: 0);
23760 SDValue Src = N->getOperand(Num: 1);
23761 SDValue VL = N->getOperand(Num: 2);
23762
23763 // Fixed vectors are wrapped in scalable containers, unwrap them.
23764 using namespace SDPatternMatch;
23765 SDValue SubVec;
23766 if (sd_match(N: Src, P: m_InsertSubvector(Base: m_Undef(), Sub: m_Value(N&: SubVec), Idx: m_Zero())))
23767 Src = SubVec;
23768
23769 SDValue SplatVal = DAG.getSplatValue(V: Src, /*LegalTypes=*/true);
23770 if (!SplatVal)
23771 break;
23772 MVT VT = N->getSimpleValueType(ResNo: 0);
23773 return lowerScalarSplat(Passthru, Scalar: SplatVal, VL, VT, DL: SDLoc(N), DAG,
23774 Subtarget);
23775 }
23776 case RISCVISD::VSLIDEDOWN_VL:
23777 case RISCVISD::VSLIDEUP_VL:
23778 if (N->getOperand(Num: 1)->isUndef())
23779 return N->getOperand(Num: 0);
23780 break;
23781 case RISCVISD::VSLIDE1UP_VL:
23782 case RISCVISD::VFSLIDE1UP_VL: {
23783 using namespace SDPatternMatch;
23784 SDValue SrcVec;
23785 SDLoc DL(N);
23786 MVT VT = N->getSimpleValueType(ResNo: 0);
23787 // If the scalar we're sliding in was extracted from the first element of a
23788 // vector, we can use that vector as the passthru in a normal slideup of 1.
23789 // This saves us an extract_element instruction (i.e. vfmv.f.s, vmv.x.s).
23790 if (!N->getOperand(Num: 0).isUndef() ||
23791 !sd_match(N: N->getOperand(Num: 2),
23792 P: m_AnyOf(preds: m_ExtractElt(Vec: m_Value(N&: SrcVec), Idx: m_Zero()),
23793 preds: m_Node(Opcode: RISCVISD::VMV_X_S, preds: m_Value(N&: SrcVec)))))
23794 break;
23795
23796 MVT SrcVecVT = SrcVec.getSimpleValueType();
23797 if (SrcVecVT.getVectorElementType() != VT.getVectorElementType())
23798 break;
23799 // Adapt the value type of source vector.
23800 if (SrcVecVT.isFixedLengthVector()) {
23801 SrcVecVT = getContainerForFixedLengthVector(VT: SrcVecVT);
23802 SrcVec = convertToScalableVector(VT: SrcVecVT, V: SrcVec, DAG, Subtarget);
23803 }
23804 if (SrcVecVT.getVectorMinNumElements() < VT.getVectorMinNumElements())
23805 SrcVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: SrcVec, Idx: 0);
23806 else
23807 SrcVec = DAG.getExtractSubvector(DL, VT, Vec: SrcVec, Idx: 0);
23808
23809 return getVSlideup(DAG, Subtarget, DL, VT, Passthru: SrcVec, Op: N->getOperand(Num: 1),
23810 Offset: DAG.getConstant(Val: 1, DL, VT: XLenVT), Mask: N->getOperand(Num: 3),
23811 VL: N->getOperand(Num: 4));
23812 }
23813 }
23814
23815 return SDValue();
23816}
23817
23818bool RISCVTargetLowering::shouldTransformSignedTruncationCheck(
23819 EVT XVT, unsigned KeptBits) const {
23820 // For vectors, we don't have a preference..
23821 if (XVT.isVector())
23822 return false;
23823
23824 if (XVT != MVT::i32 && XVT != MVT::i64)
23825 return false;
23826
23827 // We can use sext.w for RV64 or an srai 31 on RV32.
23828 if (KeptBits == 32 || KeptBits == 64)
23829 return true;
23830
23831 // With Zbb we can use sext.h/sext.b.
23832 return Subtarget.hasStdExtZbb() &&
23833 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.is64Bit()) ||
23834 KeptBits == 16);
23835}
23836
23837bool RISCVTargetLowering::isDesirableToCommuteWithShift(
23838 const SDNode *N, CombineLevel Level) const {
23839 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
23840 N->getOpcode() == ISD::SRL) &&
23841 "Expected shift op");
23842
23843 // The following folds are only desirable if `(OP _, c1 << c2)` can be
23844 // materialised in fewer instructions than `(OP _, c1)`:
23845 //
23846 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
23847 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
23848 SDValue N0 = N->getOperand(Num: 0);
23849 EVT Ty = N0.getValueType();
23850
23851 // LD/ST will optimize constant Offset extraction, so when AddNode is used by
23852 // LD/ST, it can still complete the folding optimization operation performed
23853 // above.
23854 auto isUsedByLdSt = [](const SDNode *X, const SDNode *User) {
23855 for (SDNode *Use : X->users()) {
23856 // This use is the one we're on right now. Skip it
23857 if (Use == User || Use->getOpcode() == ISD::SELECT)
23858 continue;
23859 if (!isa<StoreSDNode>(Val: Use) && !isa<LoadSDNode>(Val: Use))
23860 return false;
23861 }
23862 return true;
23863 };
23864
23865 if (Ty.isScalarInteger() &&
23866 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
23867 if (N0.getOpcode() == ISD::ADD && !N0->hasOneUse())
23868 return isUsedByLdSt(N0.getNode(), N);
23869
23870 auto *C1 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
23871 auto *C2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
23872
23873 // Bail if we might break a sh{1,2,3}add/qc.shladd pattern.
23874 if (C2 && Subtarget.hasShlAdd(ShAmt: C2->getZExtValue()) && N->hasOneUse() &&
23875 N->user_begin()->getOpcode() == ISD::ADD &&
23876 !isUsedByLdSt(*N->user_begin(), nullptr) &&
23877 !isa<ConstantSDNode>(Val: N->user_begin()->getOperand(Num: 1)))
23878 return false;
23879
23880 if (C1 && C2) {
23881 const APInt &C1Int = C1->getAPIntValue();
23882 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
23883
23884 // We can materialise `c1 << c2` into an add immediate, so it's "free",
23885 // and the combine should happen, to potentially allow further combines
23886 // later.
23887 if (ShiftedC1Int.getSignificantBits() <= 64 &&
23888 isLegalAddImmediate(Imm: ShiftedC1Int.getSExtValue()))
23889 return true;
23890
23891 // We can materialise `c1` in an add immediate, so it's "free", and the
23892 // combine should be prevented.
23893 if (C1Int.getSignificantBits() <= 64 &&
23894 isLegalAddImmediate(Imm: C1Int.getSExtValue()))
23895 return false;
23896
23897 // Neither constant will fit into an immediate, so find materialisation
23898 // costs.
23899 int C1Cost =
23900 RISCVMatInt::getIntMatCost(Val: C1Int, Size: Ty.getSizeInBits(), STI: Subtarget,
23901 /*CompressionCost*/ true);
23902 int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
23903 Val: ShiftedC1Int, Size: Ty.getSizeInBits(), STI: Subtarget,
23904 /*CompressionCost*/ true);
23905
23906 // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
23907 // combine should be prevented.
23908 if (C1Cost < ShiftedC1Cost)
23909 return false;
23910 }
23911 }
23912
23913 if (!N0->hasOneUse())
23914 return false;
23915
23916 if (N0->getOpcode() == ISD::SIGN_EXTEND &&
23917 N0->getOperand(Num: 0)->getOpcode() == ISD::ADD &&
23918 !N0->getOperand(Num: 0)->hasOneUse())
23919 return isUsedByLdSt(N0->getOperand(Num: 0).getNode(), N0.getNode());
23920
23921 return true;
23922}
23923
23924bool RISCVTargetLowering::targetShrinkDemandedConstant(
23925 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
23926 TargetLoweringOpt &TLO) const {
23927 // Delay this optimization as late as possible.
23928 if (!TLO.LegalOps)
23929 return false;
23930
23931 EVT VT = Op.getValueType();
23932 if (VT.isVector())
23933 return false;
23934
23935 unsigned Opcode = Op.getOpcode();
23936 if (Opcode != ISD::AND && Opcode != ISD::OR && Opcode != ISD::XOR)
23937 return false;
23938
23939 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
23940 if (!C)
23941 return false;
23942
23943 const APInt &Mask = C->getAPIntValue();
23944
23945 // Clear all non-demanded bits initially.
23946 APInt ShrunkMask = Mask & DemandedBits;
23947
23948 // Try to make a smaller immediate by setting undemanded bits.
23949
23950 APInt ExpandedMask = Mask | ~DemandedBits;
23951
23952 auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
23953 return ShrunkMask.isSubsetOf(RHS: Mask) && Mask.isSubsetOf(RHS: ExpandedMask);
23954 };
23955 auto UseMask = [Mask, Op, &TLO](const APInt &NewMask) -> bool {
23956 if (NewMask == Mask)
23957 return true;
23958 SDLoc DL(Op);
23959 SDValue NewC = TLO.DAG.getConstant(Val: NewMask, DL, VT: Op.getValueType());
23960 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
23961 N1: Op.getOperand(i: 0), N2: NewC);
23962 return TLO.CombineTo(O: Op, N: NewOp);
23963 };
23964
23965 // If the shrunk mask fits in sign extended 12 bits, let the target
23966 // independent code apply it.
23967 if (ShrunkMask.isSignedIntN(N: 12))
23968 return false;
23969
23970 // And has a few special cases for zext.
23971 if (Opcode == ISD::AND) {
23972 // Preserve (and X, 0xffff), if zext.h exists use zext.h,
23973 // otherwise use SLLI + SRLI.
23974 APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
23975 if (IsLegalMask(NewMask))
23976 return UseMask(NewMask);
23977
23978 // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
23979 if (VT == MVT::i64) {
23980 APInt NewMask = APInt(64, 0xffffffff);
23981 if (IsLegalMask(NewMask))
23982 return UseMask(NewMask);
23983 }
23984 }
23985
23986 // For the remaining optimizations, we need to be able to make a negative
23987 // number through a combination of mask and undemanded bits.
23988 if (ExpandedMask.isNegative()) {
23989 // What is the fewest number of bits we need to represent the negative
23990 // number.
23991 unsigned MinSignedBits = ExpandedMask.getSignificantBits();
23992
23993 // Try to make a 12 bit negative immediate. If that fails try to make a 32
23994 // bit negative immediate unless the shrunk immediate already fits in 32
23995 // bits. If we can't create a simm12, we shouldn't change opaque constants.
23996 if (MinSignedBits <= 12) {
23997 APInt NewMask = ShrunkMask;
23998 NewMask.setBitsFrom(11);
23999 assert(IsLegalMask(NewMask));
24000 return UseMask(NewMask);
24001 }
24002 if (!C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(N: 32)) {
24003 APInt NewMask = ShrunkMask;
24004 NewMask.setBitsFrom(31);
24005 assert(IsLegalMask(NewMask));
24006 return UseMask(NewMask);
24007 }
24008 }
24009
24010 // Try to form a constant that can be materialized with:
24011 // lui a0, hi20
24012 // addi(w) a0, a0, lo12
24013 // slli a1, a0, 32
24014 // add a0, a0, a1
24015 //
24016 // Or:
24017 // lui a0, hi20
24018 // addi(w) a0, a0, lo12
24019 // pack a0, a0, a0
24020 //
24021 if (!ShrunkMask.isSignedIntN(N: 32) && !C->isOpaque() && Opcode == ISD::AND &&
24022 VT == MVT::i64 && Subtarget.is64Bit()) {
24023 uint32_t Lo32Shrunk = Lo_32(Value: ShrunkMask.getZExtValue());
24024 uint32_t Hi32Shrunk = Hi_32(Value: ShrunkMask.getZExtValue());
24025
24026 // Only use this pattern if some bits in the upper and lower half must be
24027 // non-zero.
24028 if (Lo32Shrunk != Hi32Shrunk && Lo32Shrunk != 0 && Hi32Shrunk != 0) {
24029 // Find a 32-bit value that works for both halves.
24030 uint32_t Lo32Required = Lo32Shrunk | Hi32Shrunk;
24031
24032 // Replicate the 32-bit value to both halves.
24033 uint64_t DupConstant = Make_64(High: Lo32Required, Low: Lo32Required);
24034
24035 // Verify the new constant is legal.
24036 APInt CandidateMask(64, DupConstant);
24037 if (IsLegalMask(CandidateMask)) {
24038 unsigned OrigCost =
24039 RISCVMatInt::generateInstSeq(Val: ShrunkMask.getSExtValue(), STI: Subtarget)
24040 .size();
24041 unsigned NewCost =
24042 RISCVMatInt::generateInstSeq(Val: DupConstant, STI: Subtarget).size();
24043 // If the new sequence is shorter than the old sequence and won't
24044 // use a constant pool, make the change.
24045 if (NewCost < OrigCost && (!Subtarget.useConstantPoolForLargeInts() ||
24046 NewCost <= Subtarget.getMaxBuildIntsCost()))
24047 return UseMask(CandidateMask);
24048
24049 // For the 2 register form, if we're optimizing for size, only do
24050 // this if the original constant wasn't going to use a constant pool.
24051 if (!TLO.DAG.shouldOptForSize() ||
24052 !Subtarget.useConstantPoolForLargeInts() ||
24053 OrigCost <= Subtarget.getMaxBuildIntsCost()) {
24054 unsigned ShiftAmt, AddOpc;
24055 RISCVMatInt::InstSeq SeqLo = RISCVMatInt::generateTwoRegInstSeq(
24056 Val: DupConstant, STI: Subtarget, ShiftAmt, AddOpc);
24057 if (!SeqLo.empty()) {
24058 NewCost = SeqLo.size() + 2;
24059 if (NewCost < OrigCost &&
24060 (!Subtarget.useConstantPoolForLargeInts() ||
24061 (NewCost <= Subtarget.getMaxBuildIntsCost())))
24062 return UseMask(CandidateMask);
24063 }
24064 }
24065 }
24066 }
24067 }
24068
24069 return false;
24070}
24071
24072static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC) {
24073 static const uint64_t GREVMasks[] = {
24074 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
24075 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
24076
24077 for (unsigned Stage = 0; Stage != 6; ++Stage) {
24078 unsigned Shift = 1 << Stage;
24079 if (ShAmt & Shift) {
24080 uint64_t Mask = GREVMasks[Stage];
24081 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
24082 if (IsGORC)
24083 Res |= x;
24084 x = Res;
24085 }
24086 }
24087
24088 return x;
24089}
24090
24091void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
24092 KnownBits &Known,
24093 const APInt &DemandedElts,
24094 const SelectionDAG &DAG,
24095 unsigned Depth) const {
24096 unsigned BitWidth = Known.getBitWidth();
24097 unsigned Opc = Op.getOpcode();
24098 assert((Opc >= ISD::BUILTIN_OP_END ||
24099 Opc == ISD::INTRINSIC_WO_CHAIN ||
24100 Opc == ISD::INTRINSIC_W_CHAIN ||
24101 Opc == ISD::INTRINSIC_VOID) &&
24102 "Should use MaskedValueIsZero if you don't know whether Op"
24103 " is a target node!");
24104
24105 Known.resetAll();
24106 switch (Opc) {
24107 default: break;
24108 case RISCVISD::SELECT_CC: {
24109 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 4), Depth: Depth + 1);
24110 // If we don't know any bits, early out.
24111 if (Known.isUnknown())
24112 break;
24113 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 3), Depth: Depth + 1);
24114
24115 // Only known if known in both the LHS and RHS.
24116 Known = Known.intersectWith(RHS: Known2);
24117 break;
24118 }
24119 case RISCVISD::VCPOP_VL: {
24120 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
24121 Known.Zero.setBitsFrom(Known2.countMaxActiveBits());
24122 break;
24123 }
24124 case RISCVISD::CZERO_EQZ:
24125 case RISCVISD::CZERO_NEZ:
24126 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
24127 // Result is either all zero or operand 0. We can propagate zeros, but not
24128 // ones.
24129 Known.One.clearAllBits();
24130 break;
24131 case RISCVISD::REMUW: {
24132 KnownBits Known2;
24133 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24134 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
24135 // We only care about the lower 32 bits.
24136 Known = KnownBits::urem(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 32));
24137 // Restore the original width by sign extending.
24138 Known = Known.sext(BitWidth);
24139 break;
24140 }
24141 case RISCVISD::DIVUW: {
24142 KnownBits Known2;
24143 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24144 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
24145 // We only care about the lower 32 bits.
24146 Known = KnownBits::udiv(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 32));
24147 // Restore the original width by sign extending.
24148 Known = Known.sext(BitWidth);
24149 break;
24150 }
24151 case RISCVISD::SLLW: {
24152 KnownBits Known2;
24153 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24154 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
24155 Known = KnownBits::shl(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
24156 // Restore the original width by sign extending.
24157 Known = Known.sext(BitWidth);
24158 break;
24159 }
24160 case RISCVISD::SRLW: {
24161 KnownBits Known2;
24162 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24163 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
24164 Known = KnownBits::lshr(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
24165 // Restore the original width by sign extending.
24166 Known = Known.sext(BitWidth);
24167 break;
24168 }
24169 case RISCVISD::SRAW: {
24170 KnownBits Known2;
24171 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24172 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
24173 Known = KnownBits::ashr(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
24174 // Restore the original width by sign extending.
24175 Known = Known.sext(BitWidth);
24176 break;
24177 }
24178 case RISCVISD::SHL_ADD: {
24179 KnownBits Known2;
24180 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24181 unsigned ShAmt = Op.getConstantOperandVal(i: 1);
24182 Known <<= ShAmt;
24183 Known.Zero.setLowBits(ShAmt); // the <<= operator left these bits unknown
24184 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
24185 Known = KnownBits::add(LHS: Known, RHS: Known2);
24186 break;
24187 }
24188 case RISCVISD::CTZW: {
24189 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
24190 unsigned PossibleTZ = Known2.trunc(BitWidth: 32).countMaxTrailingZeros();
24191 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
24192 Known.Zero.setBitsFrom(LowBits);
24193 break;
24194 }
24195 case RISCVISD::CLZW: {
24196 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
24197 unsigned PossibleLZ = Known2.trunc(BitWidth: 32).countMaxLeadingZeros();
24198 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
24199 Known.Zero.setBitsFrom(LowBits);
24200 break;
24201 }
24202 case RISCVISD::CLSW: {
24203 // The upper 32 bits are ignored by the instruction, but ComputeNumSignBits
24204 // doesn't give us a way to ignore them. If there are fewer than 33 sign
24205 // bits in the input consider it as having no redundant sign bits. Otherwise
24206 // the lower bound of the result is NumSignBits-33. The maximum value of the
24207 // the result is 31.
24208 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
24209 unsigned MinRedundantSignBits = NumSignBits < 33 ? 0 : NumSignBits - 33;
24210 // Create a ConstantRange [MinRedundantSignBits, 32) and convert it to
24211 // KnownBits.
24212 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
24213 APInt(BitWidth, 32));
24214 Known = Range.toKnownBits();
24215 break;
24216 }
24217 case RISCVISD::BREV8:
24218 case RISCVISD::ORC_B: {
24219 // FIXME: This is based on the non-ratified Zbp GREV and GORC where a
24220 // control value of 7 is equivalent to brev8 and orc.b.
24221 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
24222 bool IsGORC = Op.getOpcode() == RISCVISD::ORC_B;
24223 // To compute zeros for ORC_B, we need to invert the value and invert it
24224 // back after. This inverting is harmless for BREV8.
24225 Known.Zero =
24226 ~computeGREVOrGORC(x: ~Known.Zero.getZExtValue(), ShAmt: 7, IsGORC);
24227 Known.One = computeGREVOrGORC(x: Known.One.getZExtValue(), ShAmt: 7, IsGORC);
24228 break;
24229 }
24230 case RISCVISD::USATI: {
24231 unsigned Width = Op.getConstantOperandVal(i: 1);
24232 Known.Zero.setBitsFrom(Width);
24233 break;
24234 }
24235 case RISCVISD::READ_VLENB: {
24236 // We can use the minimum and maximum VLEN values to bound VLENB. We
24237 // know VLEN must be a power of two.
24238 const unsigned MinVLenB = Subtarget.getRealMinVLen() / 8;
24239 const unsigned MaxVLenB = Subtarget.getRealMaxVLen() / 8;
24240 assert(MinVLenB > 0 && "READ_VLENB without vector extension enabled?");
24241 Known.Zero.setLowBits(Log2_32(Value: MinVLenB));
24242 Known.Zero.setBitsFrom(Log2_32(Value: MaxVLenB)+1);
24243 if (MaxVLenB == MinVLenB)
24244 Known.One.setBit(Log2_32(Value: MinVLenB));
24245 break;
24246 }
24247 case RISCVISD::FCLASS: {
24248 // fclass will only set one of the low 10 bits.
24249 Known.Zero.setBitsFrom(10);
24250 break;
24251 }
24252 case ISD::INTRINSIC_W_CHAIN:
24253 case ISD::INTRINSIC_WO_CHAIN: {
24254 unsigned IntNo =
24255 Op.getConstantOperandVal(i: Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
24256 switch (IntNo) {
24257 default:
24258 // We can't do anything for most intrinsics.
24259 break;
24260 case Intrinsic::riscv_vsetvli:
24261 case Intrinsic::riscv_vsetvlimax: {
24262 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
24263 unsigned VSEW = Op.getConstantOperandVal(i: HasAVL + 1);
24264 RISCVVType::VLMUL VLMUL =
24265 static_cast<RISCVVType::VLMUL>(Op.getConstantOperandVal(i: HasAVL + 2));
24266 unsigned SEW = RISCVVType::decodeVSEW(VSEW);
24267 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul: VLMUL);
24268 uint64_t MaxVL = Subtarget.getRealMaxVLen() / SEW;
24269 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
24270
24271 // Result of vsetvli must be not larger than AVL.
24272 if (HasAVL && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
24273 MaxVL = std::min(a: MaxVL, b: Op.getConstantOperandVal(i: 1));
24274
24275 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
24276 if (BitWidth > KnownZeroFirstBit)
24277 Known.Zero.setBitsFrom(KnownZeroFirstBit);
24278 break;
24279 }
24280 }
24281 break;
24282 }
24283 }
24284}
24285
24286void RISCVTargetLowering::computeKnownBitsForTargetInstr(
24287 GISelValueTracking &Analysis, Register R, KnownBits &Known,
24288 const APInt &DemandedElts, const MachineRegisterInfo &MRI,
24289 unsigned Depth) const {
24290 Known.resetAll();
24291
24292 const MachineInstr *MI = MRI.getVRegDef(Reg: R);
24293 switch (MI->getOpcode()) {
24294 default:
24295 return;
24296 case RISCV::G_BREV8: {
24297 Analysis.computeKnownBitsImpl(R: MI->getOperand(i: 1).getReg(), Known,
24298 DemandedElts, Depth: Depth + 1);
24299
24300 Known.Zero =
24301 ~computeGREVOrGORC(x: ~Known.Zero.getZExtValue(), ShAmt: 7, /*IsGORC=*/false);
24302 Known.One =
24303 computeGREVOrGORC(x: Known.One.getZExtValue(), ShAmt: 7, /*IsGORC=*/false);
24304 return;
24305 }
24306 }
24307}
24308
24309unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
24310 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
24311 unsigned Depth) const {
24312 switch (Op.getOpcode()) {
24313 default:
24314 break;
24315 case RISCVISD::SELECT_CC: {
24316 unsigned Tmp =
24317 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth + 1);
24318 if (Tmp == 1) return 1; // Early out.
24319 unsigned Tmp2 =
24320 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 4), DemandedElts, Depth: Depth + 1);
24321 return std::min(a: Tmp, b: Tmp2);
24322 }
24323 case RISCVISD::CZERO_EQZ:
24324 case RISCVISD::CZERO_NEZ:
24325 // Output is either all zero or operand 0. We can propagate sign bit count
24326 // from operand 0.
24327 return DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24328 case RISCVISD::NEGW_MAX: {
24329 // We expand this at isel to negw+max. The result will have 33 sign bits
24330 // if the input has at least 33 sign bits.
24331 unsigned Tmp =
24332 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24333 if (Tmp < 33) return 1;
24334 return 33;
24335 }
24336 case RISCVISD::SRAW: {
24337 unsigned Tmp =
24338 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
24339 // sraw produces at least 33 sign bits. If the input already has more than
24340 // 33 sign bits sraw, will preserve them.
24341 // TODO: A more precise answer could be calculated depending on known bits
24342 // in the shift amount.
24343 return std::max(a: Tmp, b: 33U);
24344 }
24345 case RISCVISD::SLLW:
24346 case RISCVISD::SRLW:
24347 case RISCVISD::DIVW:
24348 case RISCVISD::DIVUW:
24349 case RISCVISD::REMUW:
24350 case RISCVISD::ROLW:
24351 case RISCVISD::RORW:
24352 case RISCVISD::ABSW:
24353 case RISCVISD::FCVT_W_RV64:
24354 case RISCVISD::FCVT_WU_RV64:
24355 case RISCVISD::STRICT_FCVT_W_RV64:
24356 case RISCVISD::STRICT_FCVT_WU_RV64:
24357 // TODO: As the result is sign-extended, this is conservatively correct.
24358 return 33;
24359 case RISCVISD::SATI: {
24360 unsigned Width = Op.getConstantOperandVal(i: 1);
24361 return Op.getScalarValueSizeInBits() - Width;
24362 }
24363 case RISCVISD::VMV_X_S: {
24364 // The number of sign bits of the scalar result is computed by obtaining the
24365 // element type of the input vector operand, subtracting its width from the
24366 // XLEN, and then adding one (sign bit within the element type). If the
24367 // element type is wider than XLen, the least-significant XLEN bits are
24368 // taken.
24369 unsigned XLen = Subtarget.getXLen();
24370 unsigned EltBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
24371 if (EltBits <= XLen)
24372 return XLen - EltBits + 1;
24373 break;
24374 }
24375 case ISD::INTRINSIC_W_CHAIN: {
24376 unsigned IntNo = Op.getConstantOperandVal(i: 1);
24377 switch (IntNo) {
24378 default:
24379 break;
24380 case Intrinsic::riscv_masked_atomicrmw_xchg:
24381 case Intrinsic::riscv_masked_atomicrmw_add:
24382 case Intrinsic::riscv_masked_atomicrmw_sub:
24383 case Intrinsic::riscv_masked_atomicrmw_nand:
24384 case Intrinsic::riscv_masked_atomicrmw_max:
24385 case Intrinsic::riscv_masked_atomicrmw_min:
24386 case Intrinsic::riscv_masked_atomicrmw_umax:
24387 case Intrinsic::riscv_masked_atomicrmw_umin:
24388 case Intrinsic::riscv_masked_cmpxchg:
24389 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
24390 // narrow atomic operation. These are implemented using atomic
24391 // operations at the minimum supported atomicrmw/cmpxchg width whose
24392 // result is then sign extended to XLEN. With +A, the minimum width is
24393 // 32 for both 64 and 32.
24394 assert(getMinCmpXchgSizeInBits() == 32);
24395 assert(Subtarget.hasStdExtZalrsc());
24396 return Op.getValueSizeInBits() - 31;
24397 }
24398 break;
24399 }
24400 }
24401
24402 return 1;
24403}
24404
24405bool RISCVTargetLowering::SimplifyDemandedBitsForTargetNode(
24406 SDValue Op, const APInt &OriginalDemandedBits,
24407 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
24408 unsigned Depth) const {
24409 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
24410
24411 switch (Op.getOpcode()) {
24412 case RISCVISD::BREV8:
24413 case RISCVISD::ORC_B: {
24414 KnownBits Known2;
24415 bool IsGORC = Op.getOpcode() == RISCVISD::ORC_B;
24416 // For BREV8, we need to do BREV8 on the demanded bits.
24417 // For ORC_B, any bit in the output demandeds all bits from the same byte.
24418 // So we need to do ORC_B on the demanded bits.
24419 APInt DemandedBits =
24420 APInt(BitWidth, computeGREVOrGORC(x: OriginalDemandedBits.getZExtValue(),
24421 ShAmt: 7, IsGORC));
24422 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits,
24423 DemandedElts: OriginalDemandedElts, Known&: Known2, TLO, Depth: Depth + 1))
24424 return true;
24425
24426 // To compute zeros for ORC_B, we need to invert the value and invert it
24427 // back after. This inverting is harmless for BREV8.
24428 Known.Zero = ~computeGREVOrGORC(x: ~Known2.Zero.getZExtValue(), ShAmt: 7, IsGORC);
24429 Known.One = computeGREVOrGORC(x: Known2.One.getZExtValue(), ShAmt: 7, IsGORC);
24430 return false;
24431 }
24432 }
24433
24434 return TargetLowering::SimplifyDemandedBitsForTargetNode(
24435 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
24436}
24437
24438bool RISCVTargetLowering::canCreateUndefOrPoisonForTargetNode(
24439 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
24440 UndefPoisonKind Kind, bool ConsiderFlags, unsigned Depth) const {
24441
24442 // TODO: Add more target nodes.
24443 switch (Op.getOpcode()) {
24444 case RISCVISD::READ_VLENB:
24445 return false;
24446 case RISCVISD::SLLW:
24447 case RISCVISD::SRAW:
24448 case RISCVISD::SRLW:
24449 case RISCVISD::RORW:
24450 case RISCVISD::ROLW:
24451 // Only the lower 5 bits of RHS are read, guaranteeing the rotate/shift
24452 // amount is bounds.
24453 return false;
24454 case RISCVISD::SELECT_CC:
24455 // Integer comparisons cannot create poison.
24456 assert(Op.getOperand(0).getValueType().isInteger() &&
24457 "RISCVISD::SELECT_CC only compares integers");
24458 return false;
24459 }
24460 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
24461 Op, DemandedElts, DAG, Kind, ConsiderFlags, Depth);
24462}
24463
24464const Constant *
24465RISCVTargetLowering::getTargetConstantFromLoad(LoadSDNode *Ld) const {
24466 assert(Ld && "Unexpected null LoadSDNode");
24467 if (!ISD::isNormalLoad(N: Ld))
24468 return nullptr;
24469
24470 SDValue Ptr = Ld->getBasePtr();
24471
24472 // Only constant pools with no offset are supported.
24473 auto GetSupportedConstantPool = [](SDValue Ptr) -> ConstantPoolSDNode * {
24474 auto *CNode = dyn_cast<ConstantPoolSDNode>(Val&: Ptr);
24475 if (!CNode || CNode->isMachineConstantPoolEntry() ||
24476 CNode->getOffset() != 0)
24477 return nullptr;
24478
24479 return CNode;
24480 };
24481
24482 // Simple case, LLA.
24483 if (Ptr.getOpcode() == RISCVISD::LLA) {
24484 auto *CNode = GetSupportedConstantPool(Ptr.getOperand(i: 0));
24485 if (!CNode || CNode->getTargetFlags() != 0)
24486 return nullptr;
24487
24488 return CNode->getConstVal();
24489 }
24490
24491 // Look for a HI and ADD_LO pair.
24492 if (Ptr.getOpcode() != RISCVISD::ADD_LO ||
24493 Ptr.getOperand(i: 0).getOpcode() != RISCVISD::HI)
24494 return nullptr;
24495
24496 auto *CNodeLo = GetSupportedConstantPool(Ptr.getOperand(i: 1));
24497 auto *CNodeHi = GetSupportedConstantPool(Ptr.getOperand(i: 0).getOperand(i: 0));
24498
24499 if (!CNodeLo || CNodeLo->getTargetFlags() != RISCVII::MO_LO ||
24500 !CNodeHi || CNodeHi->getTargetFlags() != RISCVII::MO_HI)
24501 return nullptr;
24502
24503 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
24504 return nullptr;
24505
24506 return CNodeLo->getConstVal();
24507}
24508
24509static MachineBasicBlock *emitReadCounterWidePseudo(MachineInstr &MI,
24510 MachineBasicBlock *BB) {
24511 assert(MI.getOpcode() == RISCV::ReadCounterWide && "Unexpected instruction");
24512
24513 // To read a 64-bit counter CSR on a 32-bit target, we read the two halves.
24514 // Should the count have wrapped while it was being read, we need to try
24515 // again.
24516 // For example:
24517 // ```
24518 // read:
24519 // csrrs x3, counterh # load high word of counter
24520 // csrrs x2, counter # load low word of counter
24521 // csrrs x4, counterh # load high word of counter
24522 // bne x3, x4, read # check if high word reads match, otherwise try again
24523 // ```
24524
24525 MachineFunction &MF = *BB->getParent();
24526 const BasicBlock *LLVMBB = BB->getBasicBlock();
24527 MachineFunction::iterator It = ++BB->getIterator();
24528
24529 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
24530 MF.insert(MBBI: It, MBB: LoopMBB);
24531
24532 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
24533 MF.insert(MBBI: It, MBB: DoneMBB);
24534
24535 // Transfer the remainder of BB and its successor edges to DoneMBB.
24536 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
24537 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
24538 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
24539
24540 BB->addSuccessor(Succ: LoopMBB);
24541
24542 MachineRegisterInfo &RegInfo = MF.getRegInfo();
24543 Register ReadAgainReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24544 Register LoReg = MI.getOperand(i: 0).getReg();
24545 Register HiReg = MI.getOperand(i: 1).getReg();
24546 int64_t LoCounter = MI.getOperand(i: 2).getImm();
24547 int64_t HiCounter = MI.getOperand(i: 3).getImm();
24548 DebugLoc DL = MI.getDebugLoc();
24549
24550 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
24551 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
24552 .addImm(Val: HiCounter)
24553 .addReg(RegNo: RISCV::X0);
24554 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
24555 .addImm(Val: LoCounter)
24556 .addReg(RegNo: RISCV::X0);
24557 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
24558 .addImm(Val: HiCounter)
24559 .addReg(RegNo: RISCV::X0);
24560
24561 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
24562 .addReg(RegNo: HiReg)
24563 .addReg(RegNo: ReadAgainReg)
24564 .addMBB(MBB: LoopMBB);
24565
24566 LoopMBB->addSuccessor(Succ: LoopMBB);
24567 LoopMBB->addSuccessor(Succ: DoneMBB);
24568
24569 MI.eraseFromParent();
24570
24571 return DoneMBB;
24572}
24573
24574static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
24575 MachineBasicBlock *BB,
24576 const RISCVSubtarget &Subtarget) {
24577 assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
24578
24579 MachineFunction &MF = *BB->getParent();
24580 DebugLoc DL = MI.getDebugLoc();
24581 const RISCVInstrInfo &TII = *MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
24582 Register LoReg = MI.getOperand(i: 0).getReg();
24583 Register HiReg = MI.getOperand(i: 1).getReg();
24584 Register SrcReg = MI.getOperand(i: 2).getReg();
24585
24586 const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
24587 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
24588
24589 TII.storeRegToStackSlot(MBB&: *BB, MBBI: MI, SrcReg, IsKill: MI.getOperand(i: 2).isKill(), FrameIndex: FI, RC: SrcRC,
24590 VReg: Register());
24591 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
24592 MachineMemOperand *MMOLo =
24593 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(8));
24594 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
24595 PtrInfo: MPI.getWithOffset(O: 4), F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(8));
24596
24597 // For big-endian, the high part is at offset 0 and the low part at offset 4.
24598 if (!Subtarget.isLittleEndian())
24599 std::swap(a&: LoReg, b&: HiReg);
24600
24601 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::LW), DestReg: LoReg)
24602 .addFrameIndex(Idx: FI)
24603 .addImm(Val: 0)
24604 .addMemOperand(MMO: MMOLo);
24605 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::LW), DestReg: HiReg)
24606 .addFrameIndex(Idx: FI)
24607 .addImm(Val: 4)
24608 .addMemOperand(MMO: MMOHi);
24609 MI.eraseFromParent(); // The pseudo instruction is gone now.
24610 return BB;
24611}
24612
24613static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
24614 MachineBasicBlock *BB,
24615 const RISCVSubtarget &Subtarget) {
24616 assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
24617 "Unexpected instruction");
24618
24619 MachineFunction &MF = *BB->getParent();
24620 DebugLoc DL = MI.getDebugLoc();
24621 const RISCVInstrInfo &TII = *MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
24622 Register DstReg = MI.getOperand(i: 0).getReg();
24623 Register LoReg = MI.getOperand(i: 1).getReg();
24624 Register HiReg = MI.getOperand(i: 2).getReg();
24625 bool KillLo = MI.getOperand(i: 1).isKill();
24626 bool KillHi = MI.getOperand(i: 2).isKill();
24627
24628 const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
24629 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
24630
24631 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
24632 MachineMemOperand *MMOLo =
24633 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(8));
24634 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
24635 PtrInfo: MPI.getWithOffset(O: 4), F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(8));
24636
24637 // For big-endian, store the high part at offset 0 and the low part at
24638 // offset 4.
24639 if (!Subtarget.isLittleEndian()) {
24640 std::swap(a&: LoReg, b&: HiReg);
24641 std::swap(a&: KillLo, b&: KillHi);
24642 }
24643
24644 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::SW))
24645 .addReg(RegNo: LoReg, Flags: getKillRegState(B: KillLo))
24646 .addFrameIndex(Idx: FI)
24647 .addImm(Val: 0)
24648 .addMemOperand(MMO: MMOLo);
24649 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::SW))
24650 .addReg(RegNo: HiReg, Flags: getKillRegState(B: KillHi))
24651 .addFrameIndex(Idx: FI)
24652 .addImm(Val: 4)
24653 .addMemOperand(MMO: MMOHi);
24654 TII.loadRegFromStackSlot(MBB&: *BB, MBBI: MI, DstReg, FrameIndex: FI, RC: DstRC, VReg: Register());
24655 MI.eraseFromParent(); // The pseudo instruction is gone now.
24656 return BB;
24657}
24658
24659static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
24660 unsigned RelOpcode, unsigned EqOpcode,
24661 const RISCVSubtarget &Subtarget) {
24662 DebugLoc DL = MI.getDebugLoc();
24663 Register DstReg = MI.getOperand(i: 0).getReg();
24664 Register Src1Reg = MI.getOperand(i: 1).getReg();
24665 Register Src2Reg = MI.getOperand(i: 2).getReg();
24666 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
24667 Register SavedFFlags = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24668 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
24669
24670 // Save the current FFLAGS.
24671 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::ReadFFLAGS), DestReg: SavedFFlags);
24672
24673 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RelOpcode), DestReg: DstReg)
24674 .addReg(RegNo: Src1Reg)
24675 .addReg(RegNo: Src2Reg);
24676 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
24677 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
24678
24679 // Restore the FFLAGS.
24680 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::WriteFFLAGS))
24681 .addReg(RegNo: SavedFFlags, Flags: RegState::Kill);
24682
24683 // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
24684 auto MIB2 = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: EqOpcode), DestReg: RISCV::X0)
24685 .addReg(RegNo: Src1Reg, Flags: getKillRegState(B: MI.getOperand(i: 1).isKill()))
24686 .addReg(RegNo: Src2Reg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
24687 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
24688 MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
24689
24690 // Erase the pseudoinstruction.
24691 MI.eraseFromParent();
24692 return BB;
24693}
24694
24695static MachineBasicBlock *
24696EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second,
24697 MachineBasicBlock *ThisMBB,
24698 const RISCVSubtarget &Subtarget) {
24699 // Select_FPRX_ (rs1, rs2, imm, rs4, (Select_FPRX_ rs1, rs2, imm, rs4, rs5)
24700 // Without this, custom-inserter would have generated:
24701 //
24702 // A
24703 // | \
24704 // | B
24705 // | /
24706 // C
24707 // | \
24708 // | D
24709 // | /
24710 // E
24711 //
24712 // A: X = ...; Y = ...
24713 // B: empty
24714 // C: Z = PHI [X, A], [Y, B]
24715 // D: empty
24716 // E: PHI [X, C], [Z, D]
24717 //
24718 // If we lower both Select_FPRX_ in a single step, we can instead generate:
24719 //
24720 // A
24721 // | \
24722 // | C
24723 // | /|
24724 // |/ |
24725 // | |
24726 // | D
24727 // | /
24728 // E
24729 //
24730 // A: X = ...; Y = ...
24731 // D: empty
24732 // E: PHI [X, A], [X, C], [Y, D]
24733
24734 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
24735 const DebugLoc &DL = First.getDebugLoc();
24736 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
24737 MachineFunction *F = ThisMBB->getParent();
24738 MachineBasicBlock *FirstMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
24739 MachineBasicBlock *SecondMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
24740 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
24741 MachineFunction::iterator It = ++ThisMBB->getIterator();
24742 F->insert(MBBI: It, MBB: FirstMBB);
24743 F->insert(MBBI: It, MBB: SecondMBB);
24744 F->insert(MBBI: It, MBB: SinkMBB);
24745
24746 // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
24747 SinkMBB->splice(Where: SinkMBB->begin(), Other: ThisMBB,
24748 From: std::next(x: MachineBasicBlock::iterator(First)),
24749 To: ThisMBB->end());
24750 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: ThisMBB);
24751
24752 // Fallthrough block for ThisMBB.
24753 ThisMBB->addSuccessor(Succ: FirstMBB);
24754 // Fallthrough block for FirstMBB.
24755 FirstMBB->addSuccessor(Succ: SecondMBB);
24756 ThisMBB->addSuccessor(Succ: SinkMBB);
24757 FirstMBB->addSuccessor(Succ: SinkMBB);
24758 // This is fallthrough.
24759 SecondMBB->addSuccessor(Succ: SinkMBB);
24760
24761 auto FirstCC = static_cast<RISCVCC::CondCode>(First.getOperand(i: 3).getImm());
24762 Register FLHS = First.getOperand(i: 1).getReg();
24763 Register FRHS = First.getOperand(i: 2).getReg();
24764 // Insert appropriate branch.
24765 BuildMI(BB: FirstMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC: FirstCC, SelectOpc: First.getOpcode())))
24766 .addReg(RegNo: FLHS)
24767 .addReg(RegNo: FRHS)
24768 .addMBB(MBB: SinkMBB);
24769
24770 Register SLHS = Second.getOperand(i: 1).getReg();
24771 Register SRHS = Second.getOperand(i: 2).getReg();
24772 Register Op1Reg4 = First.getOperand(i: 4).getReg();
24773 Register Op1Reg5 = First.getOperand(i: 5).getReg();
24774
24775 auto SecondCC = static_cast<RISCVCC::CondCode>(Second.getOperand(i: 3).getImm());
24776 // Insert appropriate branch.
24777 BuildMI(BB: ThisMBB, MIMD: DL,
24778 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC: SecondCC, SelectOpc: Second.getOpcode())))
24779 .addReg(RegNo: SLHS)
24780 .addReg(RegNo: SRHS)
24781 .addMBB(MBB: SinkMBB);
24782
24783 Register DestReg = Second.getOperand(i: 0).getReg();
24784 Register Op2Reg4 = Second.getOperand(i: 4).getReg();
24785 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: RISCV::PHI), DestReg)
24786 .addReg(RegNo: Op2Reg4)
24787 .addMBB(MBB: ThisMBB)
24788 .addReg(RegNo: Op1Reg4)
24789 .addMBB(MBB: FirstMBB)
24790 .addReg(RegNo: Op1Reg5)
24791 .addMBB(MBB: SecondMBB);
24792
24793 // Now remove the Select_FPRX_s.
24794 First.eraseFromParent();
24795 Second.eraseFromParent();
24796 return SinkMBB;
24797}
24798
24799static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
24800 MachineBasicBlock *BB,
24801 const RISCVSubtarget &Subtarget) {
24802 // To "insert" Select_* instructions, we actually have to insert the triangle
24803 // control-flow pattern. The incoming instructions know the destination vreg
24804 // to set, the condition code register to branch on, the true/false values to
24805 // select between, and the condcode to use to select the appropriate branch.
24806 //
24807 // We produce the following control flow:
24808 // HeadMBB
24809 // | \
24810 // | IfFalseMBB
24811 // | /
24812 // TailMBB
24813 //
24814 // When we find a sequence of selects we attempt to optimize their emission
24815 // by sharing the control flow. Currently we only handle cases where we have
24816 // multiple selects with the exact same condition (same LHS, RHS and CC).
24817 // The selects may be interleaved with other instructions if the other
24818 // instructions meet some requirements we deem safe:
24819 // - They are not pseudo instructions.
24820 // - They are debug instructions. Otherwise,
24821 // - They do not have side-effects, do not access memory and their inputs do
24822 // not depend on the results of the select pseudo-instructions.
24823 // - They don't adjust stack.
24824 // The TrueV/FalseV operands of the selects cannot depend on the result of
24825 // previous selects in the sequence.
24826 // These conditions could be further relaxed. See the X86 target for a
24827 // related approach and more information.
24828 //
24829 // Select_FPRX_ (rs1, rs2, imm, rs4, (Select_FPRX_ rs1, rs2, imm, rs4, rs5))
24830 // is checked here and handled by a separate function -
24831 // EmitLoweredCascadedSelect.
24832
24833 auto Next = next_nodbg(It: MI.getIterator(), End: BB->instr_end());
24834 if (MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
24835 MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
24836 Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
24837 Next->getOperand(i: 5).getReg() == MI.getOperand(i: 0).getReg() &&
24838 Next->getOperand(i: 5).isKill())
24839 return EmitLoweredCascadedSelect(First&: MI, Second&: *Next, ThisMBB: BB, Subtarget);
24840
24841 Register LHS = MI.getOperand(i: 1).getReg();
24842 Register RHS;
24843 if (MI.getOperand(i: 2).isReg())
24844 RHS = MI.getOperand(i: 2).getReg();
24845 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
24846
24847 SmallVector<MachineInstr *, 4> SelectDebugValues;
24848 SmallSet<Register, 4> SelectDests;
24849 SelectDests.insert(V: MI.getOperand(i: 0).getReg());
24850
24851 MachineInstr *LastSelectPseudo = &MI;
24852 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
24853
24854 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
24855 SequenceMBBI != E; ++SequenceMBBI) {
24856 if (SequenceMBBI->isDebugInstr())
24857 continue;
24858 if (RISCVInstrInfo::isSelectPseudo(MI: *SequenceMBBI)) {
24859 if (SequenceMBBI->getOperand(i: 1).getReg() != LHS ||
24860 !SequenceMBBI->getOperand(i: 2).isReg() ||
24861 SequenceMBBI->getOperand(i: 2).getReg() != RHS ||
24862 SequenceMBBI->getOperand(i: 3).getImm() != CC ||
24863 SelectDests.count(V: SequenceMBBI->getOperand(i: 4).getReg()) ||
24864 SelectDests.count(V: SequenceMBBI->getOperand(i: 5).getReg()))
24865 break;
24866 LastSelectPseudo = &*SequenceMBBI;
24867 SequenceMBBI->collectDebugValues(DbgValues&: SelectDebugValues);
24868 SelectDests.insert(V: SequenceMBBI->getOperand(i: 0).getReg());
24869 continue;
24870 }
24871 if (SequenceMBBI->hasUnmodeledSideEffects() ||
24872 SequenceMBBI->mayLoadOrStore() ||
24873 SequenceMBBI->usesCustomInsertionHook() ||
24874 TII.isFrameInstr(I: *SequenceMBBI) ||
24875 SequenceMBBI->isStackAligningInlineAsm())
24876 break;
24877 if (llvm::any_of(Range: SequenceMBBI->operands(), P: [&](MachineOperand &MO) {
24878 return MO.isReg() && MO.isUse() && SelectDests.count(V: MO.getReg());
24879 }))
24880 break;
24881 }
24882
24883 const BasicBlock *LLVM_BB = BB->getBasicBlock();
24884 DebugLoc DL = MI.getDebugLoc();
24885 MachineFunction::iterator I = ++BB->getIterator();
24886
24887 MachineBasicBlock *HeadMBB = BB;
24888 MachineFunction *F = BB->getParent();
24889 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
24890 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
24891
24892 F->insert(MBBI: I, MBB: IfFalseMBB);
24893 F->insert(MBBI: I, MBB: TailMBB);
24894
24895 // Set the call frame size on entry to the new basic blocks.
24896 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI&: *LastSelectPseudo);
24897 IfFalseMBB->setCallFrameSize(CallFrameSize);
24898 TailMBB->setCallFrameSize(CallFrameSize);
24899
24900 // Transfer debug instructions associated with the selects to TailMBB.
24901 for (MachineInstr *DebugInstr : SelectDebugValues) {
24902 TailMBB->push_back(MI: DebugInstr->removeFromParent());
24903 }
24904
24905 // Move all instructions after the sequence to TailMBB.
24906 TailMBB->splice(Where: TailMBB->end(), Other: HeadMBB,
24907 From: std::next(x: LastSelectPseudo->getIterator()), To: HeadMBB->end());
24908 // Update machine-CFG edges by transferring all successors of the current
24909 // block to the new block which will contain the Phi nodes for the selects.
24910 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: HeadMBB);
24911 // Set the successors for HeadMBB.
24912 HeadMBB->addSuccessor(Succ: IfFalseMBB);
24913 HeadMBB->addSuccessor(Succ: TailMBB);
24914
24915 // Insert appropriate branch.
24916 if (MI.getOperand(i: 2).isImm())
24917 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC, SelectOpc: MI.getOpcode())))
24918 .addReg(RegNo: LHS)
24919 .addImm(Val: MI.getOperand(i: 2).getImm())
24920 .addMBB(MBB: TailMBB);
24921 else
24922 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC, SelectOpc: MI.getOpcode())))
24923 .addReg(RegNo: LHS)
24924 .addReg(RegNo: RHS)
24925 .addMBB(MBB: TailMBB);
24926
24927 // IfFalseMBB just falls through to TailMBB.
24928 IfFalseMBB->addSuccessor(Succ: TailMBB);
24929
24930 // Create PHIs for all of the select pseudo-instructions.
24931 auto SelectMBBI = MI.getIterator();
24932 auto SelectEnd = std::next(x: LastSelectPseudo->getIterator());
24933 auto InsertionPoint = TailMBB->begin();
24934 while (SelectMBBI != SelectEnd) {
24935 auto Next = std::next(x: SelectMBBI);
24936 if (RISCVInstrInfo::isSelectPseudo(MI: *SelectMBBI)) {
24937 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
24938 BuildMI(BB&: *TailMBB, I: InsertionPoint, MIMD: SelectMBBI->getDebugLoc(),
24939 MCID: TII.get(Opcode: RISCV::PHI), DestReg: SelectMBBI->getOperand(i: 0).getReg())
24940 .addReg(RegNo: SelectMBBI->getOperand(i: 4).getReg())
24941 .addMBB(MBB: HeadMBB)
24942 .addReg(RegNo: SelectMBBI->getOperand(i: 5).getReg())
24943 .addMBB(MBB: IfFalseMBB);
24944 SelectMBBI->eraseFromParent();
24945 }
24946 SelectMBBI = Next;
24947 }
24948
24949 F->getProperties().resetNoPHIs();
24950 return TailMBB;
24951}
24952
24953// Helper to find Masked Pseudo instruction from MC instruction, LMUL and SEW.
24954static const RISCV::RISCVMaskedPseudoInfo *
24955lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVVType::VLMUL LMul, unsigned SEW) {
24956 const RISCVVInversePseudosTable::PseudoInfo *Inverse =
24957 RISCVVInversePseudosTable::getBaseInfo(BaseInstr: MCOpcode, VLMul: LMul, SEW);
24958 assert(Inverse && "Unexpected LMUL and SEW pair for instruction");
24959 const RISCV::RISCVMaskedPseudoInfo *Masked =
24960 RISCV::lookupMaskedIntrinsicByUnmasked(UnmaskedPseudo: Inverse->Pseudo);
24961 assert(Masked && "Could not find masked instruction for LMUL and SEW pair");
24962 return Masked;
24963}
24964
24965static MachineBasicBlock *emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI,
24966 MachineBasicBlock *BB,
24967 unsigned CVTXOpc) {
24968 DebugLoc DL = MI.getDebugLoc();
24969
24970 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
24971
24972 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
24973 Register SavedFFLAGS = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24974
24975 // Save the old value of FFLAGS.
24976 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::ReadFFLAGS), DestReg: SavedFFLAGS);
24977
24978 assert(MI.getNumOperands() == 7);
24979
24980 // Emit a VFCVT_X_F
24981 const TargetRegisterInfo *TRI =
24982 BB->getParent()->getSubtarget().getRegisterInfo();
24983 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: 0, TII: &TII, TRI);
24984 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
24985 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: CVTXOpc), DestReg: Tmp)
24986 .add(MO: MI.getOperand(i: 1))
24987 .add(MO: MI.getOperand(i: 2))
24988 .add(MO: MI.getOperand(i: 3))
24989 .add(MO: MachineOperand::CreateImm(Val: 7)) // frm = DYN
24990 .add(MO: MI.getOperand(i: 4))
24991 .add(MO: MI.getOperand(i: 5))
24992 .add(MO: MI.getOperand(i: 6))
24993 .add(MO: MachineOperand::CreateReg(Reg: RISCV::FRM,
24994 /*IsDef*/ isDef: false,
24995 /*IsImp*/ isImp: true));
24996
24997 // Emit a VFCVT_F_X
24998 RISCVVType::VLMUL LMul = RISCVII::getLMul(TSFlags: MI.getDesc().TSFlags);
24999 unsigned Log2SEW = MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
25000 // There is no E8 variant for VFCVT_F_X.
25001 assert(Log2SEW >= 4);
25002 unsigned CVTFOpc =
25003 lookupMaskedIntrinsic(MCOpcode: RISCV::VFCVT_F_X_V, LMul, SEW: 1 << Log2SEW)
25004 ->MaskedPseudo;
25005
25006 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: CVTFOpc))
25007 .add(MO: MI.getOperand(i: 0))
25008 .add(MO: MI.getOperand(i: 1))
25009 .addReg(RegNo: Tmp)
25010 .add(MO: MI.getOperand(i: 3))
25011 .add(MO: MachineOperand::CreateImm(Val: 7)) // frm = DYN
25012 .add(MO: MI.getOperand(i: 4))
25013 .add(MO: MI.getOperand(i: 5))
25014 .add(MO: MI.getOperand(i: 6))
25015 .add(MO: MachineOperand::CreateReg(Reg: RISCV::FRM,
25016 /*IsDef*/ isDef: false,
25017 /*IsImp*/ isImp: true));
25018
25019 // Restore FFLAGS.
25020 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::WriteFFLAGS))
25021 .addReg(RegNo: SavedFFLAGS, Flags: RegState::Kill);
25022
25023 // Erase the pseudoinstruction.
25024 MI.eraseFromParent();
25025 return BB;
25026}
25027
25028static MachineBasicBlock *emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB,
25029 const RISCVSubtarget &Subtarget) {
25030 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
25031 const TargetRegisterClass *RC;
25032 switch (MI.getOpcode()) {
25033 default:
25034 llvm_unreachable("Unexpected opcode");
25035 case RISCV::PseudoFROUND_H:
25036 CmpOpc = RISCV::FLT_H;
25037 F2IOpc = RISCV::FCVT_W_H;
25038 I2FOpc = RISCV::FCVT_H_W;
25039 FSGNJOpc = RISCV::FSGNJ_H;
25040 FSGNJXOpc = RISCV::FSGNJX_H;
25041 RC = &RISCV::FPR16RegClass;
25042 break;
25043 case RISCV::PseudoFROUND_H_INX:
25044 CmpOpc = RISCV::FLT_H_INX;
25045 F2IOpc = RISCV::FCVT_W_H_INX;
25046 I2FOpc = RISCV::FCVT_H_W_INX;
25047 FSGNJOpc = RISCV::FSGNJ_H_INX;
25048 FSGNJXOpc = RISCV::FSGNJX_H_INX;
25049 RC = &RISCV::GPRF16RegClass;
25050 break;
25051 case RISCV::PseudoFROUND_S:
25052 CmpOpc = RISCV::FLT_S;
25053 F2IOpc = RISCV::FCVT_W_S;
25054 I2FOpc = RISCV::FCVT_S_W;
25055 FSGNJOpc = RISCV::FSGNJ_S;
25056 FSGNJXOpc = RISCV::FSGNJX_S;
25057 RC = &RISCV::FPR32RegClass;
25058 break;
25059 case RISCV::PseudoFROUND_S_INX:
25060 CmpOpc = RISCV::FLT_S_INX;
25061 F2IOpc = RISCV::FCVT_W_S_INX;
25062 I2FOpc = RISCV::FCVT_S_W_INX;
25063 FSGNJOpc = RISCV::FSGNJ_S_INX;
25064 FSGNJXOpc = RISCV::FSGNJX_S_INX;
25065 RC = &RISCV::GPRF32RegClass;
25066 break;
25067 case RISCV::PseudoFROUND_D:
25068 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
25069 CmpOpc = RISCV::FLT_D;
25070 F2IOpc = RISCV::FCVT_L_D;
25071 I2FOpc = RISCV::FCVT_D_L;
25072 FSGNJOpc = RISCV::FSGNJ_D;
25073 FSGNJXOpc = RISCV::FSGNJX_D;
25074 RC = &RISCV::FPR64RegClass;
25075 break;
25076 case RISCV::PseudoFROUND_D_INX:
25077 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
25078 CmpOpc = RISCV::FLT_D_INX;
25079 F2IOpc = RISCV::FCVT_L_D_INX;
25080 I2FOpc = RISCV::FCVT_D_L_INX;
25081 FSGNJOpc = RISCV::FSGNJ_D_INX;
25082 FSGNJXOpc = RISCV::FSGNJX_D_INX;
25083 RC = &RISCV::GPRRegClass;
25084 break;
25085 }
25086
25087 const BasicBlock *BB = MBB->getBasicBlock();
25088 DebugLoc DL = MI.getDebugLoc();
25089 MachineFunction::iterator I = ++MBB->getIterator();
25090
25091 MachineFunction *F = MBB->getParent();
25092 MachineBasicBlock *CvtMBB = F->CreateMachineBasicBlock(BB);
25093 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB);
25094
25095 F->insert(MBBI: I, MBB: CvtMBB);
25096 F->insert(MBBI: I, MBB: DoneMBB);
25097 // Move all instructions after the sequence to DoneMBB.
25098 DoneMBB->splice(Where: DoneMBB->end(), Other: MBB, From: MachineBasicBlock::iterator(MI),
25099 To: MBB->end());
25100 // Update machine-CFG edges by transferring all successors of the current
25101 // block to the new block which will contain the Phi nodes for the selects.
25102 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
25103 // Set the successors for MBB.
25104 MBB->addSuccessor(Succ: CvtMBB);
25105 MBB->addSuccessor(Succ: DoneMBB);
25106
25107 Register DstReg = MI.getOperand(i: 0).getReg();
25108 Register SrcReg = MI.getOperand(i: 1).getReg();
25109 Register MaxReg = MI.getOperand(i: 2).getReg();
25110 int64_t FRM = MI.getOperand(i: 3).getImm();
25111
25112 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
25113 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
25114
25115 Register FabsReg = MRI.createVirtualRegister(RegClass: RC);
25116 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: FSGNJXOpc), DestReg: FabsReg).addReg(RegNo: SrcReg).addReg(RegNo: SrcReg);
25117
25118 // Compare the FP value to the max value.
25119 Register CmpReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
25120 auto MIB =
25121 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: CmpOpc), DestReg: CmpReg).addReg(RegNo: FabsReg).addReg(RegNo: MaxReg);
25122 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
25123 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
25124
25125 // Insert branch.
25126 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: RISCV::BEQ))
25127 .addReg(RegNo: CmpReg)
25128 .addReg(RegNo: RISCV::X0)
25129 .addMBB(MBB: DoneMBB);
25130
25131 CvtMBB->addSuccessor(Succ: DoneMBB);
25132
25133 // Convert to integer.
25134 Register F2IReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
25135 MIB = BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: F2IOpc), DestReg: F2IReg).addReg(RegNo: SrcReg).addImm(Val: FRM);
25136 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
25137 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
25138
25139 // Convert back to FP.
25140 Register I2FReg = MRI.createVirtualRegister(RegClass: RC);
25141 MIB = BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: I2FOpc), DestReg: I2FReg).addReg(RegNo: F2IReg).addImm(Val: FRM);
25142 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
25143 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
25144
25145 // Restore the sign bit.
25146 Register CvtReg = MRI.createVirtualRegister(RegClass: RC);
25147 BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: FSGNJOpc), DestReg: CvtReg).addReg(RegNo: I2FReg).addReg(RegNo: SrcReg);
25148
25149 // Merge the results.
25150 BuildMI(BB&: *DoneMBB, I: DoneMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: RISCV::PHI), DestReg: DstReg)
25151 .addReg(RegNo: SrcReg)
25152 .addMBB(MBB)
25153 .addReg(RegNo: CvtReg)
25154 .addMBB(MBB: CvtMBB);
25155
25156 MI.eraseFromParent();
25157 return DoneMBB;
25158}
25159
25160MachineBasicBlock *
25161RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
25162 MachineBasicBlock *BB) const {
25163 switch (MI.getOpcode()) {
25164 default:
25165 llvm_unreachable("Unexpected instr type to insert");
25166 case RISCV::ReadCounterWide:
25167 assert(!Subtarget.is64Bit() &&
25168 "ReadCounterWide is only to be used on riscv32");
25169 return emitReadCounterWidePseudo(MI, BB);
25170 case RISCV::Select_GPR_Using_CC_GPR:
25171 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
25172 case RISCV::Select_GPR_Using_CC_SImm5_CV:
25173 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
25174 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
25175 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
25176 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
25177 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
25178 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
25179 case RISCV::Select_FPR16_Using_CC_GPR:
25180 case RISCV::Select_FPR16INX_Using_CC_GPR:
25181 case RISCV::Select_FPR32_Using_CC_GPR:
25182 case RISCV::Select_FPR32INX_Using_CC_GPR:
25183 case RISCV::Select_FPR64_Using_CC_GPR:
25184 case RISCV::Select_FPR64INX_Using_CC_GPR:
25185 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
25186 return emitSelectPseudo(MI, BB, Subtarget);
25187 case RISCV::BuildPairF64Pseudo:
25188 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
25189 case RISCV::SplitF64Pseudo:
25190 return emitSplitF64Pseudo(MI, BB, Subtarget);
25191 case RISCV::PseudoQuietFLE_H:
25192 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_H, EqOpcode: RISCV::FEQ_H, Subtarget);
25193 case RISCV::PseudoQuietFLE_H_INX:
25194 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_H_INX, EqOpcode: RISCV::FEQ_H_INX, Subtarget);
25195 case RISCV::PseudoQuietFLT_H:
25196 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_H, EqOpcode: RISCV::FEQ_H, Subtarget);
25197 case RISCV::PseudoQuietFLT_H_INX:
25198 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_H_INX, EqOpcode: RISCV::FEQ_H_INX, Subtarget);
25199 case RISCV::PseudoQuietFLE_S:
25200 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_S, EqOpcode: RISCV::FEQ_S, Subtarget);
25201 case RISCV::PseudoQuietFLE_S_INX:
25202 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_S_INX, EqOpcode: RISCV::FEQ_S_INX, Subtarget);
25203 case RISCV::PseudoQuietFLT_S:
25204 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_S, EqOpcode: RISCV::FEQ_S, Subtarget);
25205 case RISCV::PseudoQuietFLT_S_INX:
25206 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_S_INX, EqOpcode: RISCV::FEQ_S_INX, Subtarget);
25207 case RISCV::PseudoQuietFLE_D:
25208 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D, EqOpcode: RISCV::FEQ_D, Subtarget);
25209 case RISCV::PseudoQuietFLE_D_INX:
25210 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D_INX, EqOpcode: RISCV::FEQ_D_INX, Subtarget);
25211 case RISCV::PseudoQuietFLE_D_IN32X:
25212 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D_IN32X, EqOpcode: RISCV::FEQ_D_IN32X,
25213 Subtarget);
25214 case RISCV::PseudoQuietFLT_D:
25215 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D, EqOpcode: RISCV::FEQ_D, Subtarget);
25216 case RISCV::PseudoQuietFLT_D_INX:
25217 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D_INX, EqOpcode: RISCV::FEQ_D_INX, Subtarget);
25218 case RISCV::PseudoQuietFLT_D_IN32X:
25219 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D_IN32X, EqOpcode: RISCV::FEQ_D_IN32X,
25220 Subtarget);
25221
25222 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
25223 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M1_MASK);
25224 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
25225 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M2_MASK);
25226 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
25227 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M4_MASK);
25228 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
25229 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M8_MASK);
25230 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
25231 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_MF2_MASK);
25232 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
25233 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_MF4_MASK);
25234 case RISCV::PseudoFROUND_H:
25235 case RISCV::PseudoFROUND_H_INX:
25236 case RISCV::PseudoFROUND_S:
25237 case RISCV::PseudoFROUND_S_INX:
25238 case RISCV::PseudoFROUND_D:
25239 case RISCV::PseudoFROUND_D_INX:
25240 case RISCV::PseudoFROUND_D_IN32X:
25241 return emitFROUND(MI, MBB: BB, Subtarget);
25242 case RISCV::PROBED_STACKALLOC_DYN:
25243 return emitDynamicProbedAlloc(MI, MBB: BB);
25244 case TargetOpcode::STATEPOINT:
25245 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
25246 // while jal call instruction (where statepoint will be lowered at the end)
25247 // has implicit def. This def is early-clobber as it will be set at
25248 // the moment of the call and earlier than any use is read.
25249 // Add this implicit dead def here as a workaround.
25250 MI.addOperand(MF&: *MI.getMF(),
25251 Op: MachineOperand::CreateReg(
25252 Reg: RISCV::X1, /*isDef*/ true,
25253 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
25254 /*isUndef*/ false, /*isEarlyClobber*/ true));
25255 [[fallthrough]];
25256 case TargetOpcode::STACKMAP:
25257 case TargetOpcode::PATCHPOINT:
25258 if (!Subtarget.is64Bit())
25259 reportFatalUsageError(reason: "STACKMAP, PATCHPOINT and STATEPOINT are only "
25260 "supported on 64-bit targets");
25261 return emitPatchPoint(MI, MBB: BB);
25262 }
25263}
25264
25265void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
25266 SDNode *Node) const {
25267 // If instruction defines FRM operand, conservatively set it as non-dead to
25268 // express data dependency with FRM users and prevent incorrect instruction
25269 // reordering.
25270 if (auto *FRMDef = MI.findRegisterDefOperand(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
25271 FRMDef->setIsDead(false);
25272 return;
25273 }
25274 // Add FRM dependency to any instructions with dynamic rounding mode.
25275 int Idx = RISCV::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: RISCV::OpName::frm);
25276 if (Idx < 0) {
25277 // Vector pseudos have FRM index indicated by TSFlags.
25278 Idx = RISCVII::getFRMOpNum(Desc: MI.getDesc());
25279 if (Idx < 0)
25280 return;
25281 }
25282 if (MI.getOperand(i: Idx).getImm() != RISCVFPRndMode::DYN)
25283 return;
25284 // If the instruction already reads FRM, don't add another read.
25285 if (MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr))
25286 return;
25287 MI.addOperand(
25288 Op: MachineOperand::CreateReg(Reg: RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
25289}
25290
25291// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
25292// values.
25293static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
25294 const CCValAssign &VA, const SDLoc &DL,
25295 const RISCVSubtarget &Subtarget) {
25296 if (VA.needsCustom()) {
25297 if (VA.getLocVT().isInteger() &&
25298 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
25299 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT: VA.getValVT(), Operand: Val);
25300 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
25301 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Val);
25302 if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
25303 return convertFromScalableVector(VT: VA.getValVT(), V: Val, DAG, Subtarget);
25304 llvm_unreachable("Unexpected Custom handling.");
25305 }
25306
25307 switch (VA.getLocInfo()) {
25308 default:
25309 llvm_unreachable("Unexpected CCValAssign::LocInfo");
25310 case CCValAssign::Full:
25311 break;
25312 case CCValAssign::BCvt:
25313 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
25314 break;
25315 }
25316 return Val;
25317}
25318
25319// The caller is responsible for loading the full value if the argument is
25320// passed with CCValAssign::Indirect.
25321static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
25322 const CCValAssign &VA, const SDLoc &DL,
25323 const ISD::InputArg &In,
25324 const RISCVTargetLowering &TLI) {
25325 MachineFunction &MF = DAG.getMachineFunction();
25326 MachineRegisterInfo &RegInfo = MF.getRegInfo();
25327 EVT LocVT = VA.getLocVT();
25328 SDValue Val;
25329 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: LocVT.getSimpleVT());
25330 Register VReg = RegInfo.createVirtualRegister(RegClass: RC);
25331 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
25332 Val = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
25333
25334 // If input is sign extended from 32 bits, note it for the SExtWRemoval pass.
25335 if (In.isOrigArg()) {
25336 Argument *OrigArg = MF.getFunction().getArg(i: In.getOrigArgIndex());
25337 if (OrigArg->getType()->isIntegerTy()) {
25338 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
25339 // An input zero extended from i31 can also be considered sign extended.
25340 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
25341 (BitWidth < 32 && In.Flags.isZExt())) {
25342 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
25343 RVFI->addSExt32Register(Reg: VReg);
25344 }
25345 }
25346 }
25347
25348 if (VA.getLocInfo() == CCValAssign::Indirect)
25349 return Val;
25350
25351 return convertLocVTToValVT(DAG, Val, VA, DL, Subtarget: TLI.getSubtarget());
25352}
25353
25354static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
25355 const CCValAssign &VA, const SDLoc &DL,
25356 const RISCVSubtarget &Subtarget) {
25357 EVT LocVT = VA.getLocVT();
25358
25359 if (VA.needsCustom()) {
25360 if (LocVT.isInteger() &&
25361 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
25362 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: LocVT, Operand: Val);
25363 if (LocVT == MVT::i64 && VA.getValVT() == MVT::f32)
25364 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Val);
25365 if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
25366 return convertToScalableVector(VT: LocVT, V: Val, DAG, Subtarget);
25367 llvm_unreachable("Unexpected Custom handling.");
25368 }
25369
25370 switch (VA.getLocInfo()) {
25371 default:
25372 llvm_unreachable("Unexpected CCValAssign::LocInfo");
25373 case CCValAssign::Full:
25374 break;
25375 case CCValAssign::BCvt:
25376 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LocVT, Operand: Val);
25377 break;
25378 }
25379 return Val;
25380}
25381
25382// The caller is responsible for loading the full value if the argument is
25383// passed with CCValAssign::Indirect.
25384static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
25385 const CCValAssign &VA, const SDLoc &DL,
25386 const RISCVTargetLowering &TLI) {
25387 MachineFunction &MF = DAG.getMachineFunction();
25388 MachineFrameInfo &MFI = MF.getFrameInfo();
25389 EVT LocVT = VA.getLocVT();
25390 EVT PtrVT = MVT::getIntegerVT(BitWidth: DAG.getDataLayout().getPointerSizeInBits(AS: 0));
25391 int FI = MFI.CreateFixedObject(Size: LocVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
25392 /*IsImmutable=*/true);
25393 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
25394 SDValue Val = DAG.getLoad(
25395 VT: LocVT, dl: DL, Chain, Ptr: FIN,
25396 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
25397
25398 if (VA.getLocInfo() == CCValAssign::Indirect)
25399 return Val;
25400
25401 return convertLocVTToValVT(DAG, Val, VA, DL, Subtarget: TLI.getSubtarget());
25402}
25403
25404static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
25405 const CCValAssign &VA,
25406 const CCValAssign &HiVA,
25407 const SDLoc &DL) {
25408 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
25409 "Unexpected VA");
25410 MachineFunction &MF = DAG.getMachineFunction();
25411 MachineFrameInfo &MFI = MF.getFrameInfo();
25412 MachineRegisterInfo &RegInfo = MF.getRegInfo();
25413
25414 assert(VA.isRegLoc() && "Expected register VA assignment");
25415
25416 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
25417 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
25418 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
25419 SDValue Hi;
25420 if (HiVA.isMemLoc()) {
25421 // Second half of f64 is passed on the stack.
25422 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
25423 /*IsImmutable=*/true);
25424 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
25425 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
25426 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
25427 } else {
25428 // Second half of f64 is passed in another GPR.
25429 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
25430 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
25431 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
25432 }
25433
25434 // For big-endian, swap the order of Lo and Hi when building the pair.
25435 const RISCVSubtarget &Subtarget = DAG.getSubtarget<RISCVSubtarget>();
25436 if (!Subtarget.isLittleEndian())
25437 std::swap(a&: Lo, b&: Hi);
25438
25439 return DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
25440}
25441
25442static SDValue unpackGPRVecOnRV32(SelectionDAG &DAG, SDValue Chain,
25443 const CCValAssign &VA,
25444 const CCValAssign &HiVA, const SDLoc &DL) {
25445 MachineFunction &MF = DAG.getMachineFunction();
25446 MachineFrameInfo &MFI = MF.getFrameInfo();
25447 MachineRegisterInfo &RegInfo = MF.getRegInfo();
25448
25449 assert(VA.isRegLoc() && "Expected register VA assignment");
25450
25451 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
25452 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
25453 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
25454 SDValue Hi;
25455 if (HiVA.isMemLoc()) {
25456 // Second half of f64 is passed on the stack.
25457 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
25458 /*IsImmutable=*/true);
25459 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
25460 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
25461 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
25462 } else {
25463 // Second half of f64 is passed in another GPR.
25464 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
25465 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
25466 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
25467 }
25468
25469 return DAG.getNode(Opcode: RISCVISD::BuildPairGPRVec, DL, VT: VA.getValVT(), N1: Lo, N2: Hi);
25470}
25471
25472// Transform physical registers into virtual registers.
25473SDValue RISCVTargetLowering::LowerFormalArguments(
25474 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
25475 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
25476 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
25477
25478 MachineFunction &MF = DAG.getMachineFunction();
25479
25480 switch (CallConv) {
25481 default:
25482 reportFatalUsageError(reason: "Unsupported calling convention");
25483 case CallingConv::C:
25484 case CallingConv::Fast:
25485 case CallingConv::PreserveMost:
25486 case CallingConv::GRAAL:
25487 case CallingConv::RISCV_VectorCall:
25488#define CC_VLS_CASE(ABI_VLEN) case CallingConv::RISCV_VLSCall_##ABI_VLEN:
25489 CC_VLS_CASE(32)
25490 CC_VLS_CASE(64)
25491 CC_VLS_CASE(128)
25492 CC_VLS_CASE(256)
25493 CC_VLS_CASE(512)
25494 CC_VLS_CASE(1024)
25495 CC_VLS_CASE(2048)
25496 CC_VLS_CASE(4096)
25497 CC_VLS_CASE(8192)
25498 CC_VLS_CASE(16384)
25499 CC_VLS_CASE(32768)
25500 CC_VLS_CASE(65536)
25501#undef CC_VLS_CASE
25502 break;
25503 case CallingConv::GHC:
25504 if (Subtarget.hasStdExtE())
25505 reportFatalUsageError(reason: "GHC calling convention is not supported on RVE!");
25506 if (!Subtarget.hasStdExtFOrZfinx() || !Subtarget.hasStdExtDOrZdinx())
25507 reportFatalUsageError(reason: "GHC calling convention requires the (Zfinx/F) and "
25508 "(Zdinx/D) instruction set extensions");
25509 }
25510
25511 const Function &Func = MF.getFunction();
25512 if (Func.hasFnAttribute(Kind: "interrupt")) {
25513 if (!Func.arg_empty())
25514 reportFatalUsageError(
25515 reason: "Functions with the interrupt attribute cannot have arguments!");
25516
25517 StringRef Kind =
25518 MF.getFunction().getFnAttribute(Kind: "interrupt").getValueAsString();
25519
25520 constexpr StringLiteral SupportedInterruptKinds[] = {
25521 "machine",
25522 "supervisor",
25523 "rnmi",
25524 "qci-nest",
25525 "qci-nonest",
25526 "SiFive-CLIC-preemptible",
25527 "SiFive-CLIC-stack-swap",
25528 "SiFive-CLIC-preemptible-stack-swap",
25529 };
25530 if (!llvm::is_contained(Range: SupportedInterruptKinds, Element: Kind))
25531 reportFatalUsageError(
25532 reason: "Function interrupt attribute argument not supported!");
25533
25534 if (Kind.starts_with(Prefix: "qci-") && !Subtarget.hasVendorXqciint())
25535 reportFatalUsageError(
25536 reason: "'qci-*' interrupt kinds require Xqciint extension");
25537
25538 if (Kind.starts_with(Prefix: "SiFive-CLIC-") && !Subtarget.hasVendorXSfmclic())
25539 reportFatalUsageError(
25540 reason: "'SiFive-CLIC-*' interrupt kinds require XSfmclic extension");
25541
25542 if (Kind == "rnmi" && !Subtarget.hasStdExtSmrnmi())
25543 reportFatalUsageError(reason: "'rnmi' interrupt kind requires Srnmi extension");
25544 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
25545 if (Kind.starts_with(Prefix: "SiFive-CLIC-preemptible") && TFI->hasFP(MF))
25546 reportFatalUsageError(reason: "'SiFive-CLIC-preemptible' interrupt kinds cannot "
25547 "have a frame pointer");
25548 }
25549
25550 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
25551 MVT XLenVT = Subtarget.getXLenVT();
25552 unsigned XLenInBytes = Subtarget.getXLen() / 8;
25553
25554 // Check if this function has any musttail calls. If so, incoming indirect
25555 // arg pointers must be saved in virtual registers so they survive across
25556 // basic blocks (the SelectionDAG is cleared between BBs). Only do this
25557 // when needed to avoid adding register pressure to non-musttail functions.
25558 bool HasMusttail = llvm::any_of(Range: Func, P: [](const BasicBlock &BB) {
25559 return llvm::any_of(Range: BB, P: [](const Instruction &I) {
25560 if (const auto *CI = dyn_cast<CallInst>(Val: &I))
25561 return CI->isMustTailCall();
25562 return false;
25563 });
25564 });
25565 // Used with vargs to accumulate store chains.
25566 std::vector<SDValue> OutChains;
25567
25568 // Assign locations to all of the incoming arguments.
25569 SmallVector<CCValAssign, 16> ArgLocs;
25570 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
25571
25572 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_RISCV);
25573
25574 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
25575 CCValAssign &VA = ArgLocs[i];
25576 SDValue ArgValue;
25577 // Passing f64 on RV32D with a soft float ABI must be handled as a special
25578 // case.
25579 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
25580 assert(VA.needsCustom());
25581 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
25582 } else if (VA.getLocVT() == MVT::i32 &&
25583 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT()) &&
25584 VA.getLocInfo() != CCValAssign::Indirect) {
25585 assert(VA.needsCustom());
25586 ArgValue = unpackGPRVecOnRV32(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
25587 } else if (VA.isRegLoc())
25588 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, In: Ins[InsIdx], TLI: *this);
25589 else
25590 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL, TLI: *this);
25591
25592 if (VA.getLocInfo() == CCValAssign::Indirect) {
25593 // If the original argument was split and passed by reference (e.g. i128
25594 // on RV32), we need to load all parts of it here (using the same
25595 // address). Vectors may be partly split to registers and partly to the
25596 // stack, in which case the base address is partly offset and subsequent
25597 // stores are relative to that.
25598 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
25599 PtrInfo: MachinePointerInfo()));
25600 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
25601 if (HasMusttail) {
25602 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
25603 Register VReg =
25604 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
25605 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VReg, N: ArgValue);
25606 RVFI->setIncomingIndirectArg(ArgIndex, Reg: VReg);
25607 }
25608 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
25609 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
25610 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
25611 CCValAssign &PartVA = ArgLocs[i + 1];
25612 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
25613 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
25614 if (PartVA.getValVT().isScalableVector())
25615 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
25616 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue, N2: Offset);
25617 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
25618 PtrInfo: MachinePointerInfo()));
25619 ++i;
25620 ++InsIdx;
25621 }
25622 continue;
25623 }
25624 InVals.push_back(Elt: ArgValue);
25625 }
25626
25627 if (any_of(Range&: ArgLocs,
25628 P: [](CCValAssign &VA) { return VA.getLocVT().isScalableVector(); }))
25629 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
25630
25631 if (IsVarArg) {
25632 ArrayRef<MCPhysReg> ArgRegs = RISCV::getArgGPRs(STI: Subtarget);
25633 unsigned Idx = CCInfo.getFirstUnallocated(Regs: ArgRegs);
25634 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
25635 MachineFrameInfo &MFI = MF.getFrameInfo();
25636 MachineRegisterInfo &RegInfo = MF.getRegInfo();
25637 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
25638
25639 // Size of the vararg save area. For now, the varargs save area is either
25640 // zero or large enough to hold a0-a7.
25641 int VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
25642 int FI;
25643
25644 // If all registers are allocated, then all varargs must be passed on the
25645 // stack and we don't need to save any argregs.
25646 if (VarArgsSaveSize == 0) {
25647 int VaArgOffset = CCInfo.getStackSize();
25648 FI = MFI.CreateFixedObject(Size: XLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
25649 } else {
25650 int VaArgOffset = -VarArgsSaveSize;
25651 FI = MFI.CreateFixedObject(Size: VarArgsSaveSize, SPOffset: VaArgOffset, IsImmutable: true);
25652
25653 // If saving an odd number of registers then create an extra stack slot to
25654 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
25655 // offsets to even-numbered registers remain 2*XLEN-aligned.
25656 if (Idx % 2) {
25657 MFI.CreateFixedObject(
25658 Size: XLenInBytes, SPOffset: VaArgOffset - static_cast<int>(XLenInBytes), IsImmutable: true);
25659 VarArgsSaveSize += XLenInBytes;
25660 }
25661
25662 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
25663
25664 // Copy the integer registers that may have been used for passing varargs
25665 // to the vararg save area.
25666 for (unsigned I = Idx; I < ArgRegs.size(); ++I) {
25667 const Register Reg = RegInfo.createVirtualRegister(RegClass: RC);
25668 RegInfo.addLiveIn(Reg: ArgRegs[I], vreg: Reg);
25669 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: XLenVT);
25670 SDValue Store = DAG.getStore(
25671 Chain, dl: DL, Val: ArgValue, Ptr: FIN,
25672 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: (I - Idx) * XLenInBytes));
25673 OutChains.push_back(x: Store);
25674 FIN =
25675 DAG.getMemBasePlusOffset(Base: FIN, Offset: TypeSize::getFixed(ExactSize: XLenInBytes), DL);
25676 }
25677 }
25678
25679 // Record the frame index of the first variable argument
25680 // which is a value necessary to VASTART.
25681 RVFI->setVarArgsFrameIndex(FI);
25682 RVFI->setVarArgsSaveSize(VarArgsSaveSize);
25683 }
25684
25685 // All stores are grouped in one node to allow the matching between
25686 // the size of Ins and InVals. This only happens for vararg functions.
25687 if (!OutChains.empty()) {
25688 assert(IsVarArg && "Only variadic functions should have OutChains");
25689 OutChains.push_back(x: Chain);
25690 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
25691 }
25692
25693 return Chain;
25694}
25695
25696/// isEligibleForTailCallOptimization - Check whether the call is eligible
25697/// for tail call optimization.
25698/// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
25699bool RISCVTargetLowering::isEligibleForTailCallOptimization(
25700 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
25701 const SmallVector<CCValAssign, 16> &ArgLocs) const {
25702
25703 auto CalleeCC = CLI.CallConv;
25704 auto &Outs = CLI.Outs;
25705 auto &Caller = MF.getFunction();
25706 auto CallerCC = Caller.getCallingConv();
25707
25708 // Exception-handling functions need a special set of instructions to
25709 // indicate a return to the hardware. Tail-calling another function would
25710 // probably break this.
25711 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
25712 // should be expanded as new function attributes are introduced.
25713 if (Caller.hasFnAttribute(Kind: "interrupt"))
25714 return false;
25715
25716 bool IsMustTail = CLI.CB && CLI.CB->isMustTailCall();
25717
25718 // Byval parameters hand the function a pointer directly into the stack area
25719 // we want to reuse during a tail call. Working around this *is* possible
25720 // but less efficient and uglier in LowerCall. For musttail, there is no
25721 // workaround today: a byval arg requires a local copy that becomes invalid
25722 // after the tail call deallocates the caller's frame, so rejecting here
25723 // (and triggering reportFatalInternalError in LowerCall) is safer than
25724 // miscompiling.
25725 for (auto &Arg : Outs)
25726 if (Arg.Flags.isByVal())
25727 return false;
25728
25729 // musttail bypasses the remaining checks: the checks either reject cases
25730 // we handle specially (indirect args are forwarded via incoming pointers,
25731 // stack-passed args reuse the matching incoming layout, sret is forwarded
25732 // like any other pointer arg) or are optimizations not applicable to
25733 // mandatory tail calls.
25734 if (IsMustTail)
25735 return true;
25736
25737 // Do not tail call opt if the stack is used to pass parameters.
25738 if (CCInfo.getStackSize() != 0)
25739 return false;
25740
25741 // Do not tail call opt if any parameters need to be passed indirectly.
25742 // Since long doubles (fp128) and i128 are larger than 2*XLEN, they are
25743 // passed indirectly. The caller allocates stack space for the value and
25744 // passes a pointer. On a tail call the caller's frame is deallocated
25745 // before the callee executes, leaving the pointer dangling.
25746 for (auto &VA : ArgLocs)
25747 if (VA.getLocInfo() == CCValAssign::Indirect)
25748 return false;
25749
25750 // Do not tail call opt if either caller or callee uses struct return
25751 // semantics.
25752 auto IsCallerStructRet = Caller.hasStructRetAttr();
25753 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
25754 if (IsCallerStructRet || IsCalleeStructRet)
25755 return false;
25756
25757 // The callee has to preserve all registers the caller needs to preserve.
25758 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
25759 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
25760 if (CalleeCC != CallerCC) {
25761 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
25762 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
25763 return false;
25764 }
25765
25766 return true;
25767}
25768
25769static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
25770 return DAG.getDataLayout().getPrefTypeAlign(
25771 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
25772}
25773
25774// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
25775// and output parameter nodes.
25776SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
25777 SmallVectorImpl<SDValue> &InVals) const {
25778 SelectionDAG &DAG = CLI.DAG;
25779 SDLoc &DL = CLI.DL;
25780 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
25781 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
25782 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
25783 SDValue Chain = CLI.Chain;
25784 SDValue Callee = CLI.Callee;
25785 bool &IsTailCall = CLI.IsTailCall;
25786 CallingConv::ID CallConv = CLI.CallConv;
25787 bool IsVarArg = CLI.IsVarArg;
25788 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
25789 MVT XLenVT = Subtarget.getXLenVT();
25790 const CallBase *CB = CLI.CB;
25791
25792 MachineFunction &MF = DAG.getMachineFunction();
25793 MachineFunction::CallSiteInfo CSInfo;
25794
25795 // Set type id for call site info.
25796 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
25797
25798 // Analyze the operands of the call, assigning locations to each operand.
25799 SmallVector<CCValAssign, 16> ArgLocs;
25800 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
25801
25802 switch (CallConv) {
25803 case CallingConv::GHC:
25804 if (Subtarget.hasStdExtE())
25805 reportFatalUsageError(reason: "GHC calling convention is not supported on RVE!");
25806 break;
25807 }
25808
25809 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_RISCV);
25810
25811 // Check if it's really possible to do a tail call.
25812 if (IsTailCall)
25813 IsTailCall = isEligibleForTailCallOptimization(CCInfo&: ArgCCInfo, CLI, MF, ArgLocs);
25814
25815 if (IsTailCall)
25816 ++NumTailCalls;
25817 else if (CLI.CB && CLI.CB->isMustTailCall())
25818 reportFatalInternalError(reason: "failed to perform tail call elimination on a "
25819 "call site marked musttail");
25820
25821 // Get a count of how many bytes are to be pushed on the stack.
25822 unsigned NumBytes = ArgCCInfo.getStackSize();
25823
25824 // Create local copies for byval args
25825 SmallVector<SDValue, 8> ByValArgs;
25826 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
25827 ISD::ArgFlagsTy Flags = Outs[i].Flags;
25828 if (!Flags.isByVal())
25829 continue;
25830
25831 SDValue Arg = OutVals[i];
25832 unsigned Size = Flags.getByValSize();
25833 Align Alignment = Flags.getNonZeroByValAlign();
25834
25835 int FI =
25836 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/isSpillSlot: false);
25837 SDValue FIPtr = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
25838 SDValue SizeNode = DAG.getConstant(Val: Size, DL, VT: XLenVT);
25839
25840 Chain = DAG.getMemcpy(Chain, dl: DL, Dst: FIPtr, Src: Arg, Size: SizeNode, DstAlign: Alignment, SrcAlign: Alignment,
25841 /*IsVolatile=*/isVol: false,
25842 /*AlwaysInline=*/false, /*CI*/ nullptr, OverrideTailCall: IsTailCall,
25843 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
25844 ByValArgs.push_back(Elt: FIPtr);
25845 }
25846
25847 if (!IsTailCall)
25848 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
25849
25850 // Copy argument values to their designated locations.
25851 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
25852 SmallVector<SDValue, 8> MemOpChains;
25853 SDValue StackPtr;
25854 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
25855 ++i, ++OutIdx) {
25856 CCValAssign &VA = ArgLocs[i];
25857 SDValue ArgValue = OutVals[OutIdx];
25858 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
25859
25860 // Handle passing f64 on RV32D with a soft float ABI as a special case.
25861 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
25862 assert(VA.isRegLoc() && "Expected register VA assignment");
25863 assert(VA.needsCustom());
25864 SDValue SplitF64 = DAG.getNode(
25865 Opcode: RISCVISD::SplitF64, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
25866 SDValue Lo = SplitF64.getValue(R: 0);
25867 SDValue Hi = SplitF64.getValue(R: 1);
25868
25869 // For big-endian, swap the order of Lo and Hi when passing.
25870 if (!Subtarget.isLittleEndian())
25871 std::swap(a&: Lo, b&: Hi);
25872
25873 Register RegLo = VA.getLocReg();
25874 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
25875
25876 // Get the CCValAssign for the Hi part.
25877 CCValAssign &HiVA = ArgLocs[++i];
25878
25879 if (HiVA.isMemLoc()) {
25880 // Second half of f64 is passed on the stack.
25881 if (!StackPtr.getNode())
25882 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
25883 SDValue Address =
25884 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
25885 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
25886 // Emit the store.
25887 MemOpChains.push_back(Elt: DAG.getStore(
25888 Chain, dl: DL, Val: Hi, Ptr: Address,
25889 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
25890 } else {
25891 // Second half of f64 is passed in another GPR.
25892 Register RegHigh = HiVA.getLocReg();
25893 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
25894 }
25895 continue;
25896 }
25897
25898 // Handle passing 64-bit vector on RV32 as a special case.
25899 if (VA.getLocVT() == MVT::i32 &&
25900 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT()) &&
25901 VA.getLocInfo() != CCValAssign::Indirect) {
25902 assert(VA.isRegLoc() && "Expected register VA assignment");
25903 assert(VA.needsCustom());
25904 SDValue SplitGPRVec =
25905 DAG.getNode(Opcode: RISCVISD::SplitGPRVec, DL,
25906 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
25907 SDValue Lo = SplitGPRVec.getValue(R: 0);
25908 SDValue Hi = SplitGPRVec.getValue(R: 1);
25909
25910 Register RegLo = VA.getLocReg();
25911 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
25912
25913 // Get the CCValAssign for the Hi part.
25914 CCValAssign &HiVA = ArgLocs[++i];
25915
25916 if (HiVA.isMemLoc()) {
25917 // Second half of vector is passed on the stack.
25918 if (!StackPtr.getNode())
25919 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
25920 SDValue Address =
25921 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
25922 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
25923 // Emit the store.
25924 MemOpChains.push_back(Elt: DAG.getStore(
25925 Chain, dl: DL, Val: Hi, Ptr: Address,
25926 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
25927 } else {
25928 // Second half of vector is passed in another GPR.
25929 Register RegHigh = HiVA.getLocReg();
25930 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
25931 }
25932 continue;
25933 }
25934
25935 // Promote the value if needed.
25936 // For now, only handle fully promoted and indirect arguments.
25937 if (VA.getLocInfo() == CCValAssign::Indirect) {
25938 // For musttail calls, reuse incoming indirect pointers instead of
25939 // creating new stack temporaries. The incoming pointers point to the
25940 // caller's caller's frame, which remains valid after a tail call.
25941 if (IsTailCall && CLI.CB && CLI.CB->isMustTailCall()) {
25942 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
25943 unsigned CallArgIdx = Outs[OutIdx].OrigArgIndex;
25944
25945 // Resolve which formal parameter is being passed at this call
25946 // position.
25947 //
25948 // FIXME: Ins[].OrigArgIndex is Argument::getArgNo() (unfiltered),
25949 // but Outs[].OrigArgIndex is an index into a filtered arg list
25950 // (empty types removed, via CallLoweringInfo in the target-
25951 // independent layer). IncomingIndirectArgs is keyed by the
25952 // caller's unfiltered Argument::getArgNo(), so we have to walk
25953 // the caller's formals (same filter) to translate the index.
25954 // This target-independent asymmetry should be normalized so
25955 // backends do not need to re-derive the mapping.
25956 //
25957 // Steps:
25958 // 1. Find the call operand at filtered position CallArgIdx.
25959 // 2. If it is an Argument, use getArgNo() directly (same filter
25960 // for caller formals and call operands).
25961 // 3. Otherwise (computed value), walk the caller's formals and
25962 // skip empty types to map the filtered index to getArgNo().
25963 const Argument *FormalArg = nullptr;
25964 unsigned FilteredIdx = 0;
25965 for (const auto &CallArg : CLI.CB->args()) {
25966 if (CallArg->getType()->isEmptyTy())
25967 continue;
25968 if (FilteredIdx == CallArgIdx) {
25969 FormalArg = dyn_cast<Argument>(Val: CallArg);
25970 break;
25971 }
25972 ++FilteredIdx;
25973 }
25974
25975 // For forwarded args, getArgNo() gives the unfiltered index directly.
25976 // For computed args, walk the caller's formals to resolve it.
25977 unsigned FormalArgIdx = CallArgIdx;
25978 if (FormalArg) {
25979 FormalArgIdx = FormalArg->getArgNo();
25980 } else {
25981 FilteredIdx = 0;
25982 for (const auto &Arg : MF.getFunction().args()) {
25983 if (Arg.getType()->isEmptyTy())
25984 continue;
25985 if (FilteredIdx == CallArgIdx) {
25986 FormalArgIdx = Arg.getArgNo();
25987 break;
25988 }
25989 ++FilteredIdx;
25990 }
25991 }
25992
25993 Register VReg = RVFI->getIncomingIndirectArg(ArgIndex: FormalArgIdx);
25994 SDValue CopyOp = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: PtrVT);
25995 // Thread the CopyFromReg output chain through MemOpChains so the
25996 // TokenFactor below sequences the copy with any stores we emit
25997 // for this argument.
25998 MemOpChains.push_back(Elt: CopyOp.getValue(R: 1));
25999 SDValue IncomingPtr = CopyOp;
26000
26001 if (!FormalArg) {
26002 // Computed value: store into the incoming indirect pointer for the
26003 // same-position formal parameter (musttail guarantees matching
26004 // prototypes, so types match). The pointer survives the tail call
26005 // since it points to the caller's caller's frame.
26006 //
26007 // The data-flow edge through IncomingPtr already prevents the
26008 // store from being scheduled before the CopyFromReg. Threading
26009 // CopyOp.getValue(1) (the copy's output chain) into the store
26010 // makes that ordering explicit on the chain edge as well, which
26011 // is the convention for memory ops chaining off their producers.
26012 MemOpChains.push_back(
26013 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: ArgValue, Ptr: IncomingPtr,
26014 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
26015 // Store any split parts at their respective offsets. Scalable
26016 // vectors need their part offsets multiplied by VSCALE, matching
26017 // the non-musttail spill path below.
26018 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
26019 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
26020 SDValue PartValue = OutVals[OutIdx + 1];
26021 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
26022 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
26023 EVT PartVT = PartValue.getValueType();
26024 if (PartVT.isScalableVector())
26025 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
26026 SDValue Addr =
26027 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: IncomingPtr, N2: Offset);
26028 MemOpChains.push_back(
26029 Elt: DAG.getStore(Chain: CopyOp.getValue(R: 1), dl: DL, Val: PartValue, Ptr: Addr,
26030 PtrInfo: MachinePointerInfo::getUnknownStack(MF)));
26031 ++i;
26032 ++OutIdx;
26033 }
26034 }
26035 ArgValue = IncomingPtr;
26036
26037 // Skip any remaining split parts (for forwarded args, they are
26038 // covered by the forwarded pointer).
26039 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == CallArgIdx) {
26040 ++i;
26041 ++OutIdx;
26042 }
26043 } else {
26044 // Store the argument in a stack slot and pass its address.
26045 Align StackAlign =
26046 std::max(a: getPrefTypeAlign(VT: Outs[OutIdx].ArgVT, DAG),
26047 b: getPrefTypeAlign(VT: ArgValue.getValueType(), DAG));
26048 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
26049 // If the original argument was split (e.g. i128), we need
26050 // to store the required parts of it here (and pass just one address).
26051 // Vectors may be partly split to registers and partly to the stack, in
26052 // which case the base address is partly offset and subsequent stores
26053 // are relative to that.
26054 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
26055 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
26056 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
26057 // Calculate the total size to store. We don't have access to what
26058 // we're actually storing other than performing the loop and collecting
26059 // the info.
26060 SmallVector<std::pair<SDValue, SDValue>> Parts;
26061 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
26062 SDValue PartValue = OutVals[OutIdx + 1];
26063 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
26064 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
26065 EVT PartVT = PartValue.getValueType();
26066 if (PartVT.isScalableVector())
26067 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
26068 StoredSize += PartVT.getStoreSize();
26069 StackAlign = std::max(a: StackAlign, b: getPrefTypeAlign(VT: PartVT, DAG));
26070 Parts.push_back(Elt: std::make_pair(x&: PartValue, y&: Offset));
26071 ++i;
26072 ++OutIdx;
26073 }
26074 SDValue SpillSlot = DAG.CreateStackTemporary(Bytes: StoredSize, Alignment: StackAlign);
26075 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
26076 MemOpChains.push_back(
26077 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
26078 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
26079 for (const auto &Part : Parts) {
26080 SDValue PartValue = Part.first;
26081 SDValue PartOffset = Part.second;
26082 SDValue Address =
26083 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot, N2: PartOffset);
26084 MemOpChains.push_back(
26085 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
26086 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
26087 }
26088 ArgValue = SpillSlot;
26089 }
26090 } else {
26091 ArgValue = convertValVTToLocVT(DAG, Val: ArgValue, VA, DL, Subtarget);
26092 }
26093
26094 // Use local copy if it is a byval arg.
26095 if (Flags.isByVal())
26096 ArgValue = ByValArgs[j++];
26097
26098 if (VA.isRegLoc()) {
26099 // Queue up the argument copies and emit them at the end.
26100 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
26101
26102 const TargetOptions &Options = DAG.getTarget().Options;
26103 if (Options.EmitCallSiteInfo)
26104 CSInfo.ArgRegPairs.emplace_back(Args: VA.getLocReg(), Args&: i);
26105 } else {
26106 assert(VA.isMemLoc() && "Argument not register or memory");
26107 assert((!IsTailCall || (CLI.CB && CLI.CB->isMustTailCall())) &&
26108 "Tail call not allowed if stack is used for passing parameters");
26109
26110 // Work out the address of the stack slot.
26111 if (!StackPtr.getNode())
26112 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
26113 SDValue Address =
26114 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
26115 N2: DAG.getIntPtrConstant(Val: VA.getLocMemOffset(), DL));
26116
26117 // Emit the store.
26118 MemOpChains.push_back(
26119 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: Address,
26120 PtrInfo: MachinePointerInfo::getStack(MF, Offset: VA.getLocMemOffset())));
26121 }
26122 }
26123
26124 // Join the stores, which are independent of one another.
26125 if (!MemOpChains.empty())
26126 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
26127
26128 SDValue Glue;
26129
26130 // Build a sequence of copy-to-reg nodes, chained and glued together.
26131 for (auto &Reg : RegsToPass) {
26132 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Reg.first, N: Reg.second, Glue);
26133 Glue = Chain.getValue(R: 1);
26134 }
26135
26136 // Validate that none of the argument registers have been marked as
26137 // reserved, if so report an error. Do the same for the return address if this
26138 // is not a tailcall.
26139 validateCCReservedRegs(Regs: RegsToPass, MF);
26140 if (!IsTailCall && MF.getSubtarget().isRegisterReservedByUser(R: RISCV::X1))
26141 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
26142 MF.getFunction(),
26143 "Return address register required, but has been reserved."});
26144
26145 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
26146 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
26147 // split it and then direct call can be matched by PseudoCALL.
26148 bool CalleeIsLargeExternalSymbol = false;
26149 if (getTargetMachine().getCodeModel() == CodeModel::Large) {
26150 if (auto *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
26151 Callee = getLargeGlobalAddress(N: S, DL, Ty: PtrVT, DAG);
26152 else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
26153 Callee = getLargeExternalSymbol(N: S, DL, Ty: PtrVT, DAG);
26154 CalleeIsLargeExternalSymbol = true;
26155 }
26156 } else if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
26157 const GlobalValue *GV = S->getGlobal();
26158 Callee = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: RISCVII::MO_CALL);
26159 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
26160 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: RISCVII::MO_CALL);
26161 }
26162
26163 // The first call operand is the chain and the second is the target address.
26164 SmallVector<SDValue, 8> Ops;
26165 Ops.push_back(Elt: Chain);
26166 Ops.push_back(Elt: Callee);
26167
26168 // Add argument registers to the end of the list so that they are
26169 // known live into the call.
26170 for (auto &Reg : RegsToPass)
26171 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
26172
26173 // Add a register mask operand representing the call-preserved registers.
26174 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
26175 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
26176 assert(Mask && "Missing call preserved mask for calling convention");
26177 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
26178
26179 // Glue the call to the argument copies, if any.
26180 if (Glue.getNode())
26181 Ops.push_back(Elt: Glue);
26182
26183 assert((!CLI.CFIType || CLI.CB->isIndirectCall()) &&
26184 "Unexpected CFI type for a direct call");
26185
26186 // Emit the call.
26187 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
26188
26189 // Use software guarded branch for large code model non-indirect calls
26190 // Tail call to external symbol will have a null CLI.CB and we need another
26191 // way to determine the callsite type
26192 bool NeedSWGuarded = false;
26193 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
26194 MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch() &&
26195 ((CLI.CB && !CLI.CB->isIndirectCall()) || CalleeIsLargeExternalSymbol))
26196 NeedSWGuarded = true;
26197
26198 // Use special pseudo for returns_twice calls (e.g., setjmp) when
26199 // cf-protection-branch is enabled, to ensure LPAD is inserted after the call.
26200 bool NeedLpadCall =
26201 CLI.CB && CLI.CB->hasFnAttr(Kind: Attribute::ReturnsTwice) &&
26202 MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch();
26203
26204 if (IsTailCall) {
26205 MF.getFrameInfo().setHasTailCall();
26206 unsigned CallOpc =
26207 NeedSWGuarded ? RISCVISD::SW_GUARDED_TAIL : RISCVISD::TAIL;
26208 SDValue Ret = DAG.getNode(Opcode: CallOpc, DL, VTList: NodeTys, Ops);
26209 if (CLI.CFIType)
26210 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
26211 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
26212 DAG.addCallSiteInfo(Node: Ret.getNode(), CallInfo: std::move(CSInfo));
26213 return Ret;
26214 }
26215
26216 unsigned CallOpc;
26217 // FIXME: Large Code Model + Zicfilp: SW_GUARDED_CALL takes priority over
26218 // LPAD_CALL for returns_twice calls, breaking LPAD alignment.
26219 if (NeedSWGuarded)
26220 CallOpc = RISCVISD::SW_GUARDED_CALL;
26221 else if (NeedLpadCall && CLI.CB->isIndirectCall())
26222 CallOpc = RISCVISD::LPAD_CALL_INDIRECT;
26223 else if (NeedLpadCall)
26224 CallOpc = RISCVISD::LPAD_CALL;
26225 else
26226 CallOpc = RISCVISD::CALL;
26227 Chain = DAG.getNode(Opcode: CallOpc, DL, VTList: NodeTys, Ops);
26228 if (CLI.CFIType)
26229 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
26230
26231 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
26232 DAG.addCallSiteInfo(Node: Chain.getNode(), CallInfo: std::move(CSInfo));
26233 Glue = Chain.getValue(R: 1);
26234
26235 // Mark the end of the call, which is glued to the call itself.
26236 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
26237 Glue = Chain.getValue(R: 1);
26238
26239 // Assign locations to each value returned by this call.
26240 SmallVector<CCValAssign, 16> RVLocs;
26241 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
26242 RetCCInfo.AnalyzeFormalArguments(Ins, Fn: RetCC_RISCV);
26243
26244 // Copy all of the result registers out of their specified physreg.
26245 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
26246 auto &VA = RVLocs[i];
26247 // Copy the value out
26248 SDValue RetValue =
26249 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue);
26250 // Glue the RetValue to the end of the call sequence
26251 Chain = RetValue.getValue(R: 1);
26252 Glue = RetValue.getValue(R: 2);
26253
26254 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
26255 assert(VA.needsCustom());
26256 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
26257 VT: MVT::i32, Glue);
26258 Chain = RetValue2.getValue(R: 1);
26259 Glue = RetValue2.getValue(R: 2);
26260
26261 // For big-endian, swap the order when building the pair.
26262 SDValue Lo = RetValue;
26263 SDValue Hi = RetValue2;
26264 if (!Subtarget.isLittleEndian())
26265 std::swap(a&: Lo, b&: Hi);
26266
26267 RetValue = DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
26268 } else if (VA.getLocVT() == MVT::i32 &&
26269 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT())) {
26270 assert(VA.needsCustom());
26271 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
26272 VT: MVT::i32, Glue);
26273 Chain = RetValue2.getValue(R: 1);
26274 Glue = RetValue2.getValue(R: 2);
26275
26276 RetValue = DAG.getNode(Opcode: RISCVISD::BuildPairGPRVec, DL, VT: VA.getValVT(),
26277 N1: RetValue, N2: RetValue2);
26278 } else
26279 RetValue = convertLocVTToValVT(DAG, Val: RetValue, VA, DL, Subtarget);
26280
26281 InVals.push_back(Elt: RetValue);
26282 }
26283
26284 return Chain;
26285}
26286
26287bool RISCVTargetLowering::CanLowerReturn(
26288 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
26289 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
26290 const Type *RetTy) const {
26291 SmallVector<CCValAssign, 16> RVLocs;
26292 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
26293 return CCInfo.CheckReturn(Outs, Fn: RetCC_RISCV);
26294}
26295
26296SDValue
26297RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
26298 bool IsVarArg,
26299 const SmallVectorImpl<ISD::OutputArg> &Outs,
26300 const SmallVectorImpl<SDValue> &OutVals,
26301 const SDLoc &DL, SelectionDAG &DAG) const {
26302 MachineFunction &MF = DAG.getMachineFunction();
26303
26304 // Stores the assignment of the return value to a location.
26305 SmallVector<CCValAssign, 16> RVLocs;
26306
26307 // Info about the registers and stack slot.
26308 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
26309 *DAG.getContext());
26310
26311 CCInfo.AnalyzeCallOperands(Outs, Fn: RetCC_RISCV);
26312
26313 if (CallConv == CallingConv::GHC && !RVLocs.empty())
26314 reportFatalUsageError(reason: "GHC functions return void only");
26315
26316 SDValue Glue;
26317 SmallVector<SDValue, 4> RetOps(1, Chain);
26318
26319 // Copy the result values into the output registers.
26320 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
26321 SDValue Val = OutVals[OutIdx];
26322 CCValAssign &VA = RVLocs[i];
26323 assert(VA.isRegLoc() && "Can only return in registers!");
26324
26325 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
26326 // Handle returning f64 on RV32D with a soft float ABI.
26327 assert(VA.isRegLoc() && "Expected return via registers");
26328 assert(VA.needsCustom());
26329 SDValue SplitF64 = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
26330 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
26331 SDValue Lo = SplitF64.getValue(R: 0);
26332 SDValue Hi = SplitF64.getValue(R: 1);
26333
26334 // For big-endian, swap the order of Lo and Hi when returning.
26335 if (!Subtarget.isLittleEndian())
26336 std::swap(a&: Lo, b&: Hi);
26337
26338 Register RegLo = VA.getLocReg();
26339 Register RegHi = RVLocs[++i].getLocReg();
26340
26341 if (Subtarget.isRegisterReservedByUser(i: RegLo) ||
26342 Subtarget.isRegisterReservedByUser(i: RegHi))
26343 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
26344 MF.getFunction(),
26345 "Return value register required, but has been reserved."});
26346
26347 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
26348 Glue = Chain.getValue(R: 1);
26349 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
26350 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
26351 Glue = Chain.getValue(R: 1);
26352 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
26353 } else if (VA.getLocVT() == MVT::i32 &&
26354 Subtarget.isPExtPackedDoubleType(VT: VA.getValVT())) {
26355 // Handle returning 64-bit vector on RV32.
26356 assert(VA.isRegLoc() && "Expected return via registers");
26357 assert(VA.needsCustom());
26358 SDValue SplitGPRVec = DAG.getNode(Opcode: RISCVISD::SplitGPRVec, DL,
26359 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
26360 SDValue Lo = SplitGPRVec.getValue(R: 0);
26361 SDValue Hi = SplitGPRVec.getValue(R: 1);
26362
26363 Register RegLo = VA.getLocReg();
26364 Register RegHi = RVLocs[++i].getLocReg();
26365
26366 if (Subtarget.isRegisterReservedByUser(i: RegLo) ||
26367 Subtarget.isRegisterReservedByUser(i: RegHi))
26368 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
26369 MF.getFunction(),
26370 "Return value register required, but has been reserved."});
26371
26372 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
26373 Glue = Chain.getValue(R: 1);
26374 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
26375 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
26376 Glue = Chain.getValue(R: 1);
26377 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
26378 } else {
26379 // Handle a 'normal' return.
26380 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
26381 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Val, Glue);
26382
26383 if (Subtarget.isRegisterReservedByUser(i: VA.getLocReg()))
26384 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
26385 MF.getFunction(),
26386 "Return value register required, but has been reserved."});
26387
26388 // Guarantee that all emitted copies are stuck together.
26389 Glue = Chain.getValue(R: 1);
26390 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
26391 }
26392 }
26393
26394 RetOps[0] = Chain; // Update chain.
26395
26396 // Add the glue node if we have it.
26397 if (Glue.getNode()) {
26398 RetOps.push_back(Elt: Glue);
26399 }
26400
26401 if (any_of(Range&: RVLocs,
26402 P: [](CCValAssign &VA) { return VA.getLocVT().isScalableVector(); }))
26403 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
26404
26405 unsigned RetOpc = RISCVISD::RET_GLUE;
26406 // Interrupt service routines use different return instructions.
26407 const Function &Func = DAG.getMachineFunction().getFunction();
26408 if (Func.hasFnAttribute(Kind: "interrupt")) {
26409 if (!Func.getReturnType()->isVoidTy())
26410 reportFatalUsageError(
26411 reason: "Functions with the interrupt attribute must have void return type!");
26412
26413 MachineFunction &MF = DAG.getMachineFunction();
26414 StringRef Kind =
26415 MF.getFunction().getFnAttribute(Kind: "interrupt").getValueAsString();
26416
26417 if (Kind == "supervisor")
26418 RetOpc = RISCVISD::SRET_GLUE;
26419 else if (Kind == "rnmi") {
26420 assert(Subtarget.hasFeature(RISCV::FeatureStdExtSmrnmi) &&
26421 "Need Smrnmi extension for rnmi");
26422 RetOpc = RISCVISD::MNRET_GLUE;
26423 } else if (Kind == "qci-nest" || Kind == "qci-nonest") {
26424 assert(Subtarget.hasFeature(RISCV::FeatureVendorXqciint) &&
26425 "Need Xqciint for qci-(no)nest");
26426 RetOpc = RISCVISD::QC_C_MILEAVERET_GLUE;
26427 } else
26428 RetOpc = RISCVISD::MRET_GLUE;
26429 }
26430
26431 return DAG.getNode(Opcode: RetOpc, DL, VT: MVT::Other, Ops: RetOps);
26432}
26433
26434void RISCVTargetLowering::validateCCReservedRegs(
26435 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
26436 MachineFunction &MF) const {
26437 const Function &F = MF.getFunction();
26438
26439 if (llvm::any_of(Range: Regs, P: [this](auto Reg) {
26440 return Subtarget.isRegisterReservedByUser(i: Reg.first);
26441 }))
26442 F.getContext().diagnose(DI: DiagnosticInfoUnsupported{
26443 F, "Argument register required, but has been reserved."});
26444}
26445
26446// Check if the result of the node is only used as a return value, as
26447// otherwise we can't perform a tail-call.
26448bool RISCVTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
26449 if (N->getNumValues() != 1)
26450 return false;
26451 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
26452 return false;
26453
26454 SDNode *Copy = *N->user_begin();
26455
26456 if (Copy->getOpcode() == ISD::BITCAST) {
26457 return isUsedByReturnOnly(N: Copy, Chain);
26458 }
26459
26460 // TODO: Handle additional opcodes in order to support tail-calling libcalls
26461 // with soft float ABIs.
26462 if (Copy->getOpcode() != ISD::CopyToReg) {
26463 return false;
26464 }
26465
26466 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
26467 // isn't safe to perform a tail call.
26468 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
26469 return false;
26470
26471 // The copy must be used by a RISCVISD::RET_GLUE, and nothing else.
26472 bool HasRet = false;
26473 for (SDNode *Node : Copy->users()) {
26474 if (Node->getOpcode() != RISCVISD::RET_GLUE)
26475 return false;
26476 HasRet = true;
26477 }
26478 if (!HasRet)
26479 return false;
26480
26481 Chain = Copy->getOperand(Num: 0);
26482 return true;
26483}
26484
26485bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
26486 return CI->isTailCall();
26487}
26488
26489/// getConstraintType - Given a constraint letter, return the type of
26490/// constraint it is for this target.
26491RISCVTargetLowering::ConstraintType
26492RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
26493 if (Constraint.size() == 1) {
26494 switch (Constraint[0]) {
26495 default:
26496 break;
26497 case 'f':
26498 case 'R':
26499 return C_RegisterClass;
26500 case 'I':
26501 case 'J':
26502 case 'K':
26503 return C_Immediate;
26504 case 'A':
26505 return C_Memory;
26506 case 's':
26507 case 'S': // A symbolic address
26508 return C_Other;
26509 }
26510 } else {
26511 if (Constraint == "vr" || Constraint == "vd" || Constraint == "vm")
26512 return C_RegisterClass;
26513 if (Constraint == "cr" || Constraint == "cR" || Constraint == "cf")
26514 return C_RegisterClass;
26515 }
26516 return TargetLowering::getConstraintType(Constraint);
26517}
26518
26519std::pair<unsigned, const TargetRegisterClass *>
26520RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
26521 StringRef Constraint,
26522 MVT VT) const {
26523 // First, see if this is a constraint that directly corresponds to a RISC-V
26524 // register class.
26525 if (Constraint.size() == 1) {
26526 switch (Constraint[0]) {
26527 case 'r':
26528 // TODO: Support fixed vectors up to XLen for P extension?
26529 if (VT.isVector())
26530 break;
26531 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
26532 return std::make_pair(x: 0U, y: &RISCV::GPRF16NoX0RegClass);
26533 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
26534 return std::make_pair(x: 0U, y: &RISCV::GPRF32NoX0RegClass);
26535 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
26536 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
26537 return std::make_pair(x: 0U, y: &RISCV::GPRNoX0RegClass);
26538 case 'f':
26539 if (VT == MVT::f16) {
26540 if (Subtarget.hasStdExtZfhmin())
26541 return std::make_pair(x: 0U, y: &RISCV::FPR16RegClass);
26542 if (Subtarget.hasStdExtZhinxmin())
26543 return std::make_pair(x: 0U, y: &RISCV::GPRF16NoX0RegClass);
26544 } else if (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
26545 return std::make_pair(x: 0U, y: &RISCV::FPR16RegClass);
26546 } else if (VT == MVT::f32) {
26547 if (Subtarget.hasStdExtF())
26548 return std::make_pair(x: 0U, y: &RISCV::FPR32RegClass);
26549 if (Subtarget.hasStdExtZfinx())
26550 return std::make_pair(x: 0U, y: &RISCV::GPRF32NoX0RegClass);
26551 } else if (VT == MVT::f64) {
26552 if (Subtarget.hasStdExtD())
26553 return std::make_pair(x: 0U, y: &RISCV::FPR64RegClass);
26554 if (Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
26555 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
26556 if (Subtarget.hasStdExtZdinx() && Subtarget.is64Bit())
26557 return std::make_pair(x: 0U, y: &RISCV::GPRNoX0RegClass);
26558 }
26559 break;
26560 case 'R':
26561 if (((VT == MVT::i64 || VT == MVT::f64) && !Subtarget.is64Bit()) ||
26562 (VT == MVT::i128 && Subtarget.is64Bit()))
26563 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
26564 break;
26565 default:
26566 break;
26567 }
26568 } else if (Constraint == "vr") {
26569 // Check VM and fractional LMUL first so that those types will use that
26570 // class instead of VR.
26571 for (const auto *RC :
26572 {&RISCV::ZZZ_VMRegClass, &RISCV::ZZZ_VRMF8RegClass,
26573 &RISCV::ZZZ_VRMF4RegClass, &RISCV::ZZZ_VRMF2RegClass,
26574 &RISCV::VRRegClass, &RISCV::VRM2RegClass, &RISCV::VRM4RegClass,
26575 &RISCV::VRM8RegClass, &RISCV::VRN2M1RegClass, &RISCV::VRN3M1RegClass,
26576 &RISCV::VRN4M1RegClass, &RISCV::VRN5M1RegClass,
26577 &RISCV::VRN6M1RegClass, &RISCV::VRN7M1RegClass,
26578 &RISCV::VRN8M1RegClass, &RISCV::VRN2M2RegClass,
26579 &RISCV::VRN3M2RegClass, &RISCV::VRN4M2RegClass,
26580 &RISCV::VRN2M4RegClass}) {
26581 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy))
26582 return std::make_pair(x: 0U, y&: RC);
26583
26584 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
26585 MVT ContainerVT = getContainerForFixedLengthVector(VT);
26586 if (TRI->isTypeLegalForClass(RC: *RC, T: ContainerVT))
26587 return std::make_pair(x: 0U, y&: RC);
26588 }
26589 }
26590 } else if (Constraint == "vd") {
26591 // Check VMNoV0 and fractional LMUL first so that those types will use that
26592 // class instead of VRNoV0.
26593 for (const auto *RC :
26594 {&RISCV::ZZZ_VMNoV0RegClass, &RISCV::ZZZ_VRMF8NoV0RegClass,
26595 &RISCV::ZZZ_VRMF4NoV0RegClass, &RISCV::ZZZ_VRMF2NoV0RegClass,
26596 &RISCV::VRNoV0RegClass, &RISCV::VRM2NoV0RegClass,
26597 &RISCV::VRM4NoV0RegClass, &RISCV::VRM8NoV0RegClass,
26598 &RISCV::VRN2M1NoV0RegClass, &RISCV::VRN3M1NoV0RegClass,
26599 &RISCV::VRN4M1NoV0RegClass, &RISCV::VRN5M1NoV0RegClass,
26600 &RISCV::VRN6M1NoV0RegClass, &RISCV::VRN7M1NoV0RegClass,
26601 &RISCV::VRN8M1NoV0RegClass, &RISCV::VRN2M2NoV0RegClass,
26602 &RISCV::VRN3M2NoV0RegClass, &RISCV::VRN4M2NoV0RegClass,
26603 &RISCV::VRN2M4NoV0RegClass}) {
26604 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy))
26605 return std::make_pair(x: 0U, y&: RC);
26606
26607 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
26608 MVT ContainerVT = getContainerForFixedLengthVector(VT);
26609 if (TRI->isTypeLegalForClass(RC: *RC, T: ContainerVT))
26610 return std::make_pair(x: 0U, y&: RC);
26611 }
26612 }
26613 } else if (Constraint == "vm") {
26614 if (TRI->isTypeLegalForClass(RC: RISCV::VMV0RegClass, T: VT.SimpleTy))
26615 return std::make_pair(x: 0U, y: &RISCV::VMV0RegClass);
26616
26617 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
26618 MVT ContainerVT = getContainerForFixedLengthVector(VT);
26619 // VT here might be coerced to vector with i8 elements, so we need to
26620 // check if this is a M1 register here instead of checking VMV0RegClass.
26621 if (TRI->isTypeLegalForClass(RC: RISCV::VRRegClass, T: ContainerVT))
26622 return std::make_pair(x: 0U, y: &RISCV::VMV0RegClass);
26623 }
26624 } else if (Constraint == "cr") {
26625 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
26626 return std::make_pair(x: 0U, y: &RISCV::GPRF16CRegClass);
26627 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
26628 return std::make_pair(x: 0U, y: &RISCV::GPRF32CRegClass);
26629 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
26630 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
26631 if (!VT.isVector())
26632 return std::make_pair(x: 0U, y: &RISCV::GPRCRegClass);
26633 } else if (Constraint == "cR") {
26634 if (((VT == MVT::i64 || VT == MVT::f64) && !Subtarget.is64Bit()) ||
26635 (VT == MVT::i128 && Subtarget.is64Bit()))
26636 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
26637 } else if (Constraint == "cf") {
26638 if (VT == MVT::f16) {
26639 if (Subtarget.hasStdExtZfhmin())
26640 return std::make_pair(x: 0U, y: &RISCV::FPR16CRegClass);
26641 if (Subtarget.hasStdExtZhinxmin())
26642 return std::make_pair(x: 0U, y: &RISCV::GPRF16CRegClass);
26643 } else if (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
26644 return std::make_pair(x: 0U, y: &RISCV::FPR16CRegClass);
26645 } else if (VT == MVT::f32) {
26646 if (Subtarget.hasStdExtF())
26647 return std::make_pair(x: 0U, y: &RISCV::FPR32CRegClass);
26648 if (Subtarget.hasStdExtZfinx())
26649 return std::make_pair(x: 0U, y: &RISCV::GPRF32CRegClass);
26650 } else if (VT == MVT::f64) {
26651 if (Subtarget.hasStdExtD())
26652 return std::make_pair(x: 0U, y: &RISCV::FPR64CRegClass);
26653 if (Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
26654 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
26655 if (Subtarget.hasStdExtZdinx() && Subtarget.is64Bit())
26656 return std::make_pair(x: 0U, y: &RISCV::GPRCRegClass);
26657 }
26658 }
26659
26660 // Clang will correctly decode the usage of register name aliases into their
26661 // official names. However, other frontends like `rustc` do not. This allows
26662 // users of these frontends to use the ABI names for registers in LLVM-style
26663 // register constraints.
26664 unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
26665 .Case(S: "{zero}", Value: RISCV::X0)
26666 .Case(S: "{ra}", Value: RISCV::X1)
26667 .Case(S: "{sp}", Value: RISCV::X2)
26668 .Case(S: "{gp}", Value: RISCV::X3)
26669 .Case(S: "{tp}", Value: RISCV::X4)
26670 .Case(S: "{t0}", Value: RISCV::X5)
26671 .Case(S: "{t1}", Value: RISCV::X6)
26672 .Case(S: "{t2}", Value: RISCV::X7)
26673 .Cases(CaseStrings: {"{s0}", "{fp}"}, Value: RISCV::X8)
26674 .Case(S: "{s1}", Value: RISCV::X9)
26675 .Case(S: "{a0}", Value: RISCV::X10)
26676 .Case(S: "{a1}", Value: RISCV::X11)
26677 .Case(S: "{a2}", Value: RISCV::X12)
26678 .Case(S: "{a3}", Value: RISCV::X13)
26679 .Case(S: "{a4}", Value: RISCV::X14)
26680 .Case(S: "{a5}", Value: RISCV::X15)
26681 .Case(S: "{a6}", Value: RISCV::X16)
26682 .Case(S: "{a7}", Value: RISCV::X17)
26683 .Case(S: "{s2}", Value: RISCV::X18)
26684 .Case(S: "{s3}", Value: RISCV::X19)
26685 .Case(S: "{s4}", Value: RISCV::X20)
26686 .Case(S: "{s5}", Value: RISCV::X21)
26687 .Case(S: "{s6}", Value: RISCV::X22)
26688 .Case(S: "{s7}", Value: RISCV::X23)
26689 .Case(S: "{s8}", Value: RISCV::X24)
26690 .Case(S: "{s9}", Value: RISCV::X25)
26691 .Case(S: "{s10}", Value: RISCV::X26)
26692 .Case(S: "{s11}", Value: RISCV::X27)
26693 .Case(S: "{t3}", Value: RISCV::X28)
26694 .Case(S: "{t4}", Value: RISCV::X29)
26695 .Case(S: "{t5}", Value: RISCV::X30)
26696 .Case(S: "{t6}", Value: RISCV::X31)
26697 .Default(Value: RISCV::NoRegister);
26698 if (XRegFromAlias != RISCV::NoRegister)
26699 return std::make_pair(x&: XRegFromAlias, y: &RISCV::GPRRegClass);
26700
26701 // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
26702 // TableGen record rather than the AsmName to choose registers for InlineAsm
26703 // constraints, plus we want to match those names to the widest floating point
26704 // register type available, manually select floating point registers here.
26705 //
26706 // The second case is the ABI name of the register, so that frontends can also
26707 // use the ABI names in register constraint lists.
26708 if (Subtarget.hasStdExtF()) {
26709 unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
26710 .Cases(CaseStrings: {"{f0}", "{ft0}"}, Value: RISCV::F0_F)
26711 .Cases(CaseStrings: {"{f1}", "{ft1}"}, Value: RISCV::F1_F)
26712 .Cases(CaseStrings: {"{f2}", "{ft2}"}, Value: RISCV::F2_F)
26713 .Cases(CaseStrings: {"{f3}", "{ft3}"}, Value: RISCV::F3_F)
26714 .Cases(CaseStrings: {"{f4}", "{ft4}"}, Value: RISCV::F4_F)
26715 .Cases(CaseStrings: {"{f5}", "{ft5}"}, Value: RISCV::F5_F)
26716 .Cases(CaseStrings: {"{f6}", "{ft6}"}, Value: RISCV::F6_F)
26717 .Cases(CaseStrings: {"{f7}", "{ft7}"}, Value: RISCV::F7_F)
26718 .Cases(CaseStrings: {"{f8}", "{fs0}"}, Value: RISCV::F8_F)
26719 .Cases(CaseStrings: {"{f9}", "{fs1}"}, Value: RISCV::F9_F)
26720 .Cases(CaseStrings: {"{f10}", "{fa0}"}, Value: RISCV::F10_F)
26721 .Cases(CaseStrings: {"{f11}", "{fa1}"}, Value: RISCV::F11_F)
26722 .Cases(CaseStrings: {"{f12}", "{fa2}"}, Value: RISCV::F12_F)
26723 .Cases(CaseStrings: {"{f13}", "{fa3}"}, Value: RISCV::F13_F)
26724 .Cases(CaseStrings: {"{f14}", "{fa4}"}, Value: RISCV::F14_F)
26725 .Cases(CaseStrings: {"{f15}", "{fa5}"}, Value: RISCV::F15_F)
26726 .Cases(CaseStrings: {"{f16}", "{fa6}"}, Value: RISCV::F16_F)
26727 .Cases(CaseStrings: {"{f17}", "{fa7}"}, Value: RISCV::F17_F)
26728 .Cases(CaseStrings: {"{f18}", "{fs2}"}, Value: RISCV::F18_F)
26729 .Cases(CaseStrings: {"{f19}", "{fs3}"}, Value: RISCV::F19_F)
26730 .Cases(CaseStrings: {"{f20}", "{fs4}"}, Value: RISCV::F20_F)
26731 .Cases(CaseStrings: {"{f21}", "{fs5}"}, Value: RISCV::F21_F)
26732 .Cases(CaseStrings: {"{f22}", "{fs6}"}, Value: RISCV::F22_F)
26733 .Cases(CaseStrings: {"{f23}", "{fs7}"}, Value: RISCV::F23_F)
26734 .Cases(CaseStrings: {"{f24}", "{fs8}"}, Value: RISCV::F24_F)
26735 .Cases(CaseStrings: {"{f25}", "{fs9}"}, Value: RISCV::F25_F)
26736 .Cases(CaseStrings: {"{f26}", "{fs10}"}, Value: RISCV::F26_F)
26737 .Cases(CaseStrings: {"{f27}", "{fs11}"}, Value: RISCV::F27_F)
26738 .Cases(CaseStrings: {"{f28}", "{ft8}"}, Value: RISCV::F28_F)
26739 .Cases(CaseStrings: {"{f29}", "{ft9}"}, Value: RISCV::F29_F)
26740 .Cases(CaseStrings: {"{f30}", "{ft10}"}, Value: RISCV::F30_F)
26741 .Cases(CaseStrings: {"{f31}", "{ft11}"}, Value: RISCV::F31_F)
26742 .Default(Value: RISCV::NoRegister);
26743 if (FReg != RISCV::NoRegister) {
26744 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
26745 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
26746 unsigned RegNo = FReg - RISCV::F0_F;
26747 unsigned DReg = RISCV::F0_D + RegNo;
26748 return std::make_pair(x&: DReg, y: &RISCV::FPR64RegClass);
26749 }
26750 if (VT == MVT::f32 || VT == MVT::Other)
26751 return std::make_pair(x&: FReg, y: &RISCV::FPR32RegClass);
26752 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
26753 unsigned RegNo = FReg - RISCV::F0_F;
26754 unsigned HReg = RISCV::F0_H + RegNo;
26755 return std::make_pair(x&: HReg, y: &RISCV::FPR16RegClass);
26756 }
26757 }
26758 }
26759
26760 if (Subtarget.hasVInstructions()) {
26761 Register VReg = StringSwitch<Register>(Constraint.lower())
26762 .Case(S: "{v0}", Value: RISCV::V0)
26763 .Case(S: "{v1}", Value: RISCV::V1)
26764 .Case(S: "{v2}", Value: RISCV::V2)
26765 .Case(S: "{v3}", Value: RISCV::V3)
26766 .Case(S: "{v4}", Value: RISCV::V4)
26767 .Case(S: "{v5}", Value: RISCV::V5)
26768 .Case(S: "{v6}", Value: RISCV::V6)
26769 .Case(S: "{v7}", Value: RISCV::V7)
26770 .Case(S: "{v8}", Value: RISCV::V8)
26771 .Case(S: "{v9}", Value: RISCV::V9)
26772 .Case(S: "{v10}", Value: RISCV::V10)
26773 .Case(S: "{v11}", Value: RISCV::V11)
26774 .Case(S: "{v12}", Value: RISCV::V12)
26775 .Case(S: "{v13}", Value: RISCV::V13)
26776 .Case(S: "{v14}", Value: RISCV::V14)
26777 .Case(S: "{v15}", Value: RISCV::V15)
26778 .Case(S: "{v16}", Value: RISCV::V16)
26779 .Case(S: "{v17}", Value: RISCV::V17)
26780 .Case(S: "{v18}", Value: RISCV::V18)
26781 .Case(S: "{v19}", Value: RISCV::V19)
26782 .Case(S: "{v20}", Value: RISCV::V20)
26783 .Case(S: "{v21}", Value: RISCV::V21)
26784 .Case(S: "{v22}", Value: RISCV::V22)
26785 .Case(S: "{v23}", Value: RISCV::V23)
26786 .Case(S: "{v24}", Value: RISCV::V24)
26787 .Case(S: "{v25}", Value: RISCV::V25)
26788 .Case(S: "{v26}", Value: RISCV::V26)
26789 .Case(S: "{v27}", Value: RISCV::V27)
26790 .Case(S: "{v28}", Value: RISCV::V28)
26791 .Case(S: "{v29}", Value: RISCV::V29)
26792 .Case(S: "{v30}", Value: RISCV::V30)
26793 .Case(S: "{v31}", Value: RISCV::V31)
26794 .Default(Value: RISCV::NoRegister);
26795 if (VReg != RISCV::NoRegister) {
26796 if (TRI->isTypeLegalForClass(RC: RISCV::ZZZ_VMRegClass, T: VT.SimpleTy))
26797 return std::make_pair(x&: VReg, y: &RISCV::ZZZ_VMRegClass);
26798 if (TRI->isTypeLegalForClass(RC: RISCV::VRRegClass, T: VT.SimpleTy))
26799 return std::make_pair(x&: VReg, y: &RISCV::VRRegClass);
26800 for (const auto *RC :
26801 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
26802 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy)) {
26803 VReg = TRI->getMatchingSuperReg(Reg: VReg, SubIdx: RISCV::sub_vrm1_0, RC);
26804 return std::make_pair(x&: VReg, y&: RC);
26805 }
26806 }
26807 }
26808 }
26809
26810 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
26811}
26812
26813InlineAsm::ConstraintCode
26814RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
26815 // Currently only support length 1 constraints.
26816 if (ConstraintCode.size() == 1) {
26817 switch (ConstraintCode[0]) {
26818 case 'A':
26819 return InlineAsm::ConstraintCode::A;
26820 default:
26821 break;
26822 }
26823 }
26824
26825 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
26826}
26827
26828void RISCVTargetLowering::LowerAsmOperandForConstraint(
26829 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
26830 SelectionDAG &DAG) const {
26831 // Currently only support length 1 constraints.
26832 if (Constraint.size() == 1) {
26833 switch (Constraint[0]) {
26834 case 'I':
26835 // Validate & create a 12-bit signed immediate operand.
26836 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
26837 uint64_t CVal = C->getSExtValue();
26838 if (isInt<12>(x: CVal))
26839 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
26840 VT: Subtarget.getXLenVT()));
26841 }
26842 return;
26843 case 'J':
26844 // Validate & create an integer zero operand.
26845 if (isNullConstant(V: Op))
26846 Ops.push_back(
26847 x: DAG.getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget.getXLenVT()));
26848 return;
26849 case 'K':
26850 // Validate & create a 5-bit unsigned immediate operand.
26851 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
26852 uint64_t CVal = C->getZExtValue();
26853 if (isUInt<5>(x: CVal))
26854 Ops.push_back(
26855 x: DAG.getTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Subtarget.getXLenVT()));
26856 }
26857 return;
26858 case 'S':
26859 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint: "s", Ops, DAG);
26860 return;
26861 default:
26862 break;
26863 }
26864 }
26865 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
26866}
26867
26868Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
26869 Instruction *Inst,
26870 AtomicOrdering Ord) const {
26871 if (Subtarget.hasStdExtZtso()) {
26872 if (isa<LoadInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
26873 return Builder.CreateFence(Ordering: Ord);
26874 return nullptr;
26875 }
26876
26877 if (isa<LoadInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
26878 return Builder.CreateFence(Ordering: Ord);
26879 if (isa<StoreInst>(Val: Inst) && isReleaseOrStronger(AO: Ord))
26880 return Builder.CreateFence(Ordering: AtomicOrdering::Release);
26881 return nullptr;
26882}
26883
26884Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
26885 Instruction *Inst,
26886 AtomicOrdering Ord) const {
26887 if (Subtarget.hasStdExtZtso()) {
26888 if (isa<StoreInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
26889 return Builder.CreateFence(Ordering: Ord);
26890 return nullptr;
26891 }
26892
26893 if (isa<LoadInst>(Val: Inst) && isAcquireOrStronger(AO: Ord))
26894 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire);
26895 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Val: Inst) &&
26896 Ord == AtomicOrdering::SequentiallyConsistent)
26897 return Builder.CreateFence(Ordering: AtomicOrdering::SequentiallyConsistent);
26898 return nullptr;
26899}
26900
26901TargetLowering::AtomicExpansionKind
26902RISCVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
26903 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
26904 // point operations can't be used in an lr/sc sequence without breaking the
26905 // forward-progress guarantee.
26906 if (AI->isFloatingPointOperation() ||
26907 AI->getOperation() == AtomicRMWInst::UIncWrap ||
26908 AI->getOperation() == AtomicRMWInst::UDecWrap ||
26909 AI->getOperation() == AtomicRMWInst::USubCond ||
26910 AI->getOperation() == AtomicRMWInst::USubSat)
26911 return AtomicExpansionKind::CmpXChg;
26912
26913 // Don't expand forced atomics, we want to have __sync libcalls instead.
26914 if (Subtarget.hasForcedAtomics())
26915 return AtomicExpansionKind::None;
26916
26917 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
26918 if (AI->getOperation() == AtomicRMWInst::Nand) {
26919 if (Subtarget.hasStdExtZacas() &&
26920 (Size >= 32 || Subtarget.hasStdExtZabha()))
26921 return AtomicExpansionKind::CmpXChg;
26922 if (Size < 32)
26923 return AtomicExpansionKind::MaskedIntrinsic;
26924 }
26925
26926 if (Size < 32 && !Subtarget.hasStdExtZabha())
26927 return AtomicExpansionKind::MaskedIntrinsic;
26928
26929 return AtomicExpansionKind::None;
26930}
26931
26932static Intrinsic::ID
26933getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
26934 switch (BinOp) {
26935 default:
26936 llvm_unreachable("Unexpected AtomicRMW BinOp");
26937 case AtomicRMWInst::Xchg:
26938 return Intrinsic::riscv_masked_atomicrmw_xchg;
26939 case AtomicRMWInst::Add:
26940 return Intrinsic::riscv_masked_atomicrmw_add;
26941 case AtomicRMWInst::Sub:
26942 return Intrinsic::riscv_masked_atomicrmw_sub;
26943 case AtomicRMWInst::Nand:
26944 return Intrinsic::riscv_masked_atomicrmw_nand;
26945 case AtomicRMWInst::Max:
26946 return Intrinsic::riscv_masked_atomicrmw_max;
26947 case AtomicRMWInst::Min:
26948 return Intrinsic::riscv_masked_atomicrmw_min;
26949 case AtomicRMWInst::UMax:
26950 return Intrinsic::riscv_masked_atomicrmw_umax;
26951 case AtomicRMWInst::UMin:
26952 return Intrinsic::riscv_masked_atomicrmw_umin;
26953 }
26954}
26955
26956Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
26957 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
26958 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
26959 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
26960 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
26961 // mask, as this produces better code than the LR/SC loop emitted by
26962 // int_riscv_masked_atomicrmw_xchg.
26963 if (AI->getOperation() == AtomicRMWInst::Xchg &&
26964 isa<ConstantInt>(Val: AI->getValOperand())) {
26965 ConstantInt *CVal = cast<ConstantInt>(Val: AI->getValOperand());
26966 if (CVal->isZero())
26967 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::And, Ptr: AlignedAddr,
26968 Val: Builder.CreateNot(V: Mask, Name: "Inv_Mask"),
26969 Align: AI->getAlign(), Ordering: Ord);
26970 if (CVal->isMinusOne())
26971 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: AlignedAddr, Val: Mask,
26972 Align: AI->getAlign(), Ordering: Ord);
26973 }
26974
26975 unsigned XLen = Subtarget.getXLen();
26976 Value *Ordering =
26977 Builder.getIntN(N: XLen, C: static_cast<uint64_t>(AI->getOrdering()));
26978 Type *Tys[] = {Builder.getIntNTy(N: XLen), AlignedAddr->getType()};
26979 Function *LrwOpScwLoop = Intrinsic::getOrInsertDeclaration(
26980 M: AI->getModule(),
26981 id: getIntrinsicForMaskedAtomicRMWBinOp(XLen, BinOp: AI->getOperation()), OverloadTys: Tys);
26982
26983 if (XLen == 64) {
26984 Incr = Builder.CreateSExt(V: Incr, DestTy: Builder.getInt64Ty());
26985 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
26986 ShiftAmt = Builder.CreateSExt(V: ShiftAmt, DestTy: Builder.getInt64Ty());
26987 }
26988
26989 Value *Result;
26990
26991 // Must pass the shift amount needed to sign extend the loaded value prior
26992 // to performing a signed comparison for min/max. ShiftAmt is the number of
26993 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
26994 // is the number of bits to left+right shift the value in order to
26995 // sign-extend.
26996 if (AI->getOperation() == AtomicRMWInst::Min ||
26997 AI->getOperation() == AtomicRMWInst::Max) {
26998 const DataLayout &DL = AI->getDataLayout();
26999 unsigned ValWidth =
27000 DL.getTypeStoreSizeInBits(Ty: AI->getValOperand()->getType());
27001 Value *SextShamt =
27002 Builder.CreateSub(LHS: Builder.getIntN(N: XLen, C: XLen - ValWidth), RHS: ShiftAmt);
27003 Result = Builder.CreateCall(Callee: LrwOpScwLoop,
27004 Args: {AlignedAddr, Incr, Mask, SextShamt, Ordering});
27005 } else {
27006 Result =
27007 Builder.CreateCall(Callee: LrwOpScwLoop, Args: {AlignedAddr, Incr, Mask, Ordering});
27008 }
27009
27010 if (XLen == 64)
27011 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
27012 return Result;
27013}
27014
27015TargetLowering::AtomicExpansionKind
27016RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
27017 const AtomicCmpXchgInst *CI) const {
27018 // Don't expand forced atomics, we want to have __sync libcalls instead.
27019 if (Subtarget.hasForcedAtomics())
27020 return AtomicExpansionKind::None;
27021
27022 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
27023 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
27024 (Size == 8 || Size == 16))
27025 return AtomicExpansionKind::MaskedIntrinsic;
27026 return AtomicExpansionKind::None;
27027}
27028
27029Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
27030 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
27031 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
27032 unsigned XLen = Subtarget.getXLen();
27033 Value *Ordering = Builder.getIntN(N: XLen, C: static_cast<uint64_t>(Ord));
27034 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg;
27035 if (XLen == 64) {
27036 CmpVal = Builder.CreateSExt(V: CmpVal, DestTy: Builder.getInt64Ty());
27037 NewVal = Builder.CreateSExt(V: NewVal, DestTy: Builder.getInt64Ty());
27038 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
27039 }
27040 Type *Tys[] = {Builder.getIntNTy(N: XLen), AlignedAddr->getType()};
27041 Value *Result = Builder.CreateIntrinsic(
27042 ID: CmpXchgIntrID, OverloadTypes: Tys, Args: {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
27043 if (XLen == 64)
27044 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
27045 return Result;
27046}
27047
27048bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(SDValue Extend,
27049 EVT DataVT) const {
27050 // We have indexed loads for all supported EEW types. Indices are always
27051 // zero extended.
27052 return Extend.getOpcode() == ISD::ZERO_EXTEND &&
27053 isTypeLegal(VT: Extend.getValueType()) &&
27054 isTypeLegal(VT: Extend.getOperand(i: 0).getValueType()) &&
27055 Extend.getOperand(i: 0).getValueType().getVectorElementType() != MVT::i1;
27056}
27057
27058bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
27059 EVT VT) const {
27060 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
27061 return false;
27062
27063 switch (FPVT.getSimpleVT().SimpleTy) {
27064 case MVT::f16:
27065 return Subtarget.hasStdExtZfhmin();
27066 case MVT::f32:
27067 return Subtarget.hasStdExtF();
27068 case MVT::f64:
27069 return Subtarget.hasStdExtD();
27070 default:
27071 return false;
27072 }
27073}
27074
27075unsigned RISCVTargetLowering::getJumpTableEncoding() const {
27076 // If we are using the small code model, we can reduce size of jump table
27077 // entry to 4 bytes.
27078 if (Subtarget.is64Bit() && !isPositionIndependent() &&
27079 getTargetMachine().getCodeModel() == CodeModel::Small) {
27080 return MachineJumpTableInfo::EK_Custom32;
27081 }
27082 return TargetLowering::getJumpTableEncoding();
27083}
27084
27085const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
27086 const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
27087 unsigned uid, MCContext &Ctx) const {
27088 assert(Subtarget.is64Bit() && !isPositionIndependent() &&
27089 getTargetMachine().getCodeModel() == CodeModel::Small);
27090 return MCSymbolRefExpr::create(Symbol: MBB->getSymbol(), Ctx);
27091}
27092
27093bool RISCVTargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
27094 SDValue &Offset,
27095 ISD::MemIndexedMode &AM,
27096 SelectionDAG &DAG) const {
27097 // Target does not support indexed loads.
27098 if (!Subtarget.hasVendorXTHeadMemIdx())
27099 return false;
27100
27101 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
27102 return false;
27103
27104 Base = Op->getOperand(Num: 0);
27105 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1))) {
27106 int64_t RHSC = RHS->getSExtValue();
27107 if (Op->getOpcode() == ISD::SUB)
27108 RHSC = -(uint64_t)RHSC;
27109
27110 // The constants that can be encoded in the THeadMemIdx instructions
27111 // are of the form (sign_extend(imm5) << imm2).
27112 bool isLegalIndexedOffset = false;
27113 for (unsigned i = 0; i < 4; i++)
27114 if (isInt<5>(x: RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
27115 isLegalIndexedOffset = true;
27116 break;
27117 }
27118
27119 if (!isLegalIndexedOffset)
27120 return false;
27121
27122 Offset = Op->getOperand(Num: 1);
27123 return true;
27124 }
27125
27126 return false;
27127}
27128
27129bool RISCVTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
27130 SDValue &Offset,
27131 ISD::MemIndexedMode &AM,
27132 SelectionDAG &DAG) const {
27133 EVT VT;
27134 SDValue Ptr;
27135 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
27136 VT = LD->getMemoryVT();
27137 Ptr = LD->getBasePtr();
27138 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
27139 VT = ST->getMemoryVT();
27140 Ptr = ST->getBasePtr();
27141 } else
27142 return false;
27143
27144 if (!getIndexedAddressParts(Op: Ptr.getNode(), Base, Offset, AM, DAG))
27145 return false;
27146
27147 AM = ISD::PRE_INC;
27148 return true;
27149}
27150
27151bool RISCVTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
27152 SDValue &Base,
27153 SDValue &Offset,
27154 ISD::MemIndexedMode &AM,
27155 SelectionDAG &DAG) const {
27156 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
27157 if (Op->getOpcode() != ISD::ADD)
27158 return false;
27159
27160 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Val: N))
27161 Base = LS->getBasePtr();
27162 else
27163 return false;
27164
27165 if (Base == Op->getOperand(Num: 0))
27166 Offset = Op->getOperand(Num: 1);
27167 else if (Base == Op->getOperand(Num: 1))
27168 Offset = Op->getOperand(Num: 0);
27169 else
27170 return false;
27171
27172 AM = ISD::POST_INC;
27173 return true;
27174 }
27175
27176 EVT VT;
27177 SDValue Ptr;
27178 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
27179 VT = LD->getMemoryVT();
27180 Ptr = LD->getBasePtr();
27181 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
27182 VT = ST->getMemoryVT();
27183 Ptr = ST->getBasePtr();
27184 } else
27185 return false;
27186
27187 if (!getIndexedAddressParts(Op, Base, Offset, AM, DAG))
27188 return false;
27189 // Post-indexing updates the base, so it's not a valid transform
27190 // if that's not the same as the load's pointer.
27191 if (Ptr != Base)
27192 return false;
27193
27194 AM = ISD::POST_INC;
27195 return true;
27196}
27197
27198bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
27199 EVT VT) const {
27200 EVT SVT = VT.getScalarType();
27201
27202 if (!SVT.isSimple())
27203 return false;
27204
27205 switch (SVT.getSimpleVT().SimpleTy) {
27206 case MVT::f16:
27207 return VT.isVector() ? Subtarget.hasVInstructionsF16()
27208 : Subtarget.hasStdExtZfhOrZhinx();
27209 case MVT::f32:
27210 return Subtarget.hasStdExtFOrZfinx();
27211 case MVT::f64:
27212 return Subtarget.hasStdExtDOrZdinx();
27213 default:
27214 break;
27215 }
27216
27217 return false;
27218}
27219
27220ISD::NodeType RISCVTargetLowering::getExtendForAtomicCmpSwapArg() const {
27221 // Zacas will use amocas.w which does not require extension.
27222 return Subtarget.hasStdExtZacas() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
27223}
27224
27225ISD::NodeType RISCVTargetLowering::getExtendForAtomicRMWArg(unsigned Op) const {
27226 // Zaamo will use amo<op>.w which does not require extension.
27227 if (Subtarget.hasStdExtZaamo() || Subtarget.hasForcedAtomics())
27228 return ISD::ANY_EXTEND;
27229
27230 // Zalrsc pseudo expansions with comparison require sign-extension.
27231 assert(Subtarget.hasStdExtZalrsc());
27232 switch (Op) {
27233 case ISD::ATOMIC_LOAD_MIN:
27234 case ISD::ATOMIC_LOAD_MAX:
27235 case ISD::ATOMIC_LOAD_UMIN:
27236 case ISD::ATOMIC_LOAD_UMAX:
27237 return ISD::SIGN_EXTEND;
27238 default:
27239 break;
27240 }
27241 return ISD::ANY_EXTEND;
27242}
27243
27244Register RISCVTargetLowering::getExceptionPointerRegister(
27245 const Constant *PersonalityFn) const {
27246 return RISCV::X10;
27247}
27248
27249Register RISCVTargetLowering::getExceptionSelectorRegister(
27250 const Constant *PersonalityFn) const {
27251 return RISCV::X11;
27252}
27253
27254bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
27255 // Return false to suppress the unnecessary extensions if the LibCall
27256 // arguments or return value is a float narrower than XLEN on a soft FP ABI.
27257 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
27258 Type.getSizeInBits() < Subtarget.getXLen()))
27259 return false;
27260
27261 return true;
27262}
27263
27264bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(Type *Ty,
27265 bool IsSigned) const {
27266 if (Subtarget.is64Bit() && Ty->isIntegerTy(BitWidth: 32))
27267 return true;
27268
27269 return IsSigned;
27270}
27271
27272bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
27273 SDValue C) const {
27274 // Check integral scalar types.
27275 if (!VT.isScalarInteger())
27276 return false;
27277
27278 // Omit the optimization if the sub target has the M extension and the data
27279 // size exceeds XLen.
27280 const bool HasZmmul = Subtarget.hasStdExtZmmul();
27281 if (HasZmmul && VT.getSizeInBits() > Subtarget.getXLen())
27282 return false;
27283
27284 auto *ConstNode = cast<ConstantSDNode>(Val&: C);
27285 const APInt &Imm = ConstNode->getAPIntValue();
27286
27287 // Don't do this if the Xqciac extension is enabled and the Imm in simm12.
27288 if (Subtarget.hasVendorXqciac() && Imm.isSignedIntN(N: 12))
27289 return false;
27290
27291 // Break the MUL to a SLLI and an ADD/SUB.
27292 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
27293 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
27294 return true;
27295
27296 // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
27297 if (Subtarget.hasShlAdd(ShAmt: 3) && !Imm.isSignedIntN(N: 12) &&
27298 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
27299 (Imm - 8).isPowerOf2()))
27300 return true;
27301
27302 // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
27303 // a pair of LUI/ADDI.
27304 if (!Imm.isSignedIntN(N: 12) && Imm.countr_zero() < 12 &&
27305 ConstNode->hasOneUse()) {
27306 APInt ImmS = Imm.ashr(ShiftAmt: Imm.countr_zero());
27307 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
27308 (1 - ImmS).isPowerOf2())
27309 return true;
27310 }
27311
27312 return false;
27313}
27314
27315bool RISCVTargetLowering::isMulAddWithConstProfitable(SDValue AddNode,
27316 SDValue ConstNode) const {
27317 // Let the DAGCombiner decide for vectors.
27318 EVT VT = AddNode.getValueType();
27319 if (VT.isVector())
27320 return true;
27321
27322 // Let the DAGCombiner decide for larger types.
27323 if (VT.getScalarSizeInBits() > Subtarget.getXLen())
27324 return true;
27325
27326 // It is worse if c1 is simm12 while c1*c2 is not.
27327 ConstantSDNode *C1Node = cast<ConstantSDNode>(Val: AddNode.getOperand(i: 1));
27328 ConstantSDNode *C2Node = cast<ConstantSDNode>(Val&: ConstNode);
27329 const APInt &C1 = C1Node->getAPIntValue();
27330 const APInt &C2 = C2Node->getAPIntValue();
27331 if (C1.isSignedIntN(N: 12) && !(C1 * C2).isSignedIntN(N: 12))
27332 return false;
27333
27334 // Default to true and let the DAGCombiner decide.
27335 return true;
27336}
27337
27338bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
27339 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
27340 unsigned *Fast) const {
27341 if (!VT.isVector() || Subtarget.hasStdExtP()) {
27342 if (Fast)
27343 *Fast = Subtarget.enableUnalignedScalarMem();
27344 return Subtarget.enableUnalignedScalarMem();
27345 }
27346
27347 // All vector implementations must support element alignment
27348 EVT ElemVT = VT.getVectorElementType();
27349 if (Alignment >= ElemVT.getStoreSize()) {
27350 if (Fast)
27351 *Fast = 1;
27352 return true;
27353 }
27354
27355 // Note: We lower an unmasked unaligned vector access to an equally sized
27356 // e8 element type access. Given this, we effectively support all unmasked
27357 // misaligned accesses. TODO: Work through the codegen implications of
27358 // allowing such accesses to be formed, and considered fast.
27359 if (Fast)
27360 *Fast = Subtarget.enableUnalignedVectorMem();
27361 return Subtarget.enableUnalignedVectorMem();
27362}
27363
27364EVT RISCVTargetLowering::getOptimalMemOpType(
27365 LLVMContext &Context, const MemOp &Op,
27366 const AttributeList &FuncAttributes) const {
27367 if (!Subtarget.hasVInstructions())
27368 return MVT::Other;
27369
27370 if (FuncAttributes.hasFnAttr(Kind: Attribute::NoImplicitFloat))
27371 return MVT::Other;
27372
27373 // We use LMUL1 memory operations here for a non-obvious reason. Our caller
27374 // has an expansion threshold, and we want the number of hardware memory
27375 // operations to correspond roughly to that threshold. LMUL>1 operations
27376 // are typically expanded linearly internally, and thus correspond to more
27377 // than one actual memory operation. Note that store merging and load
27378 // combining will typically form larger LMUL operations from the LMUL1
27379 // operations emitted here, and that's okay because combining isn't
27380 // introducing new memory operations; it's just merging existing ones.
27381 // NOTE: We limit to 1024 bytes to avoid creating an invalid MVT.
27382 const unsigned MinVLenInBytes =
27383 std::min(a: Subtarget.getRealMinVLen() / 8, b: 1024U);
27384
27385 if (Op.size() < MinVLenInBytes)
27386 // TODO: Figure out short memops. For the moment, do the default thing
27387 // which ends up using scalar sequences.
27388 return MVT::Other;
27389
27390 // If the minimum VLEN is less than RISCV::RVVBitsPerBlock we don't support
27391 // fixed vectors.
27392 if (MinVLenInBytes <= RISCV::RVVBytesPerBlock)
27393 return MVT::Other;
27394
27395 // Prefer i8 for non-zero memset as it allows us to avoid materializing
27396 // a large scalar constant and instead use vmv.v.x/i to do the
27397 // broadcast. For everything else, prefer ELenVT to minimize VL and thus
27398 // maximize the chance we can encode the size in the vsetvli.
27399 MVT ELenVT = MVT::getIntegerVT(BitWidth: Subtarget.getELen());
27400 MVT PreferredVT = (Op.isMemset() && !Op.isZeroMemset()) ? MVT::i8 : ELenVT;
27401
27402 // Do we have sufficient alignment for our preferred VT? If not, revert
27403 // to largest size allowed by our alignment criteria.
27404 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
27405 Align RequiredAlign(PreferredVT.getStoreSize());
27406 if (Op.isFixedDstAlign())
27407 RequiredAlign = std::min(a: RequiredAlign, b: Op.getDstAlign());
27408 if (Op.isMemcpyOrMemmove())
27409 RequiredAlign = std::min(a: RequiredAlign, b: Op.getSrcAlign());
27410 PreferredVT = MVT::getIntegerVT(BitWidth: RequiredAlign.value() * 8);
27411 }
27412 return MVT::getVectorVT(VT: PreferredVT, NumElements: MinVLenInBytes/PreferredVT.getStoreSize());
27413}
27414
27415bool RISCVTargetLowering::splitValueIntoRegisterParts(
27416 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
27417 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
27418 bool IsABIRegCopy = CC.has_value();
27419 EVT ValueVT = Val.getValueType();
27420
27421 MVT PairVT = Subtarget.is64Bit() ? MVT::i128 : MVT::i64;
27422 if ((ValueVT == PairVT ||
27423 (!Subtarget.is64Bit() && Subtarget.hasStdExtZdinx() &&
27424 ValueVT == MVT::f64)) &&
27425 NumParts == 1 && PartVT == MVT::Untyped) {
27426 // Pairs in Inline Assembly, f64 in Inline assembly on rv32_zdinx
27427 MVT XLenVT = Subtarget.getXLenVT();
27428 if (ValueVT == MVT::f64)
27429 Val = DAG.getBitcast(VT: MVT::i64, V: Val);
27430 auto [Lo, Hi] = DAG.SplitScalar(N: Val, DL, LoVT: XLenVT, HiVT: XLenVT);
27431 // Always creating an MVT::Untyped part, so always use
27432 // RISCVISD::BuildGPRPair.
27433 Parts[0] = DAG.getNode(Opcode: RISCVISD::BuildGPRPair, DL, VT: PartVT, N1: Lo, N2: Hi);
27434 return true;
27435 }
27436
27437 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
27438 PartVT == MVT::f32) {
27439 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
27440 // nan, and cast to f32.
27441 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Val);
27442 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Val);
27443 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Val,
27444 N2: DAG.getConstant(Val: 0xFFFF0000, DL, VT: MVT::i32));
27445 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
27446 Parts[0] = Val;
27447 return true;
27448 }
27449
27450 if (ValueVT.isRISCVVectorTuple() && PartVT.isRISCVVectorTuple()) {
27451#ifndef NDEBUG
27452 unsigned ValNF = ValueVT.getRISCVVectorTupleNumFields();
27453 [[maybe_unused]] unsigned ValLMUL =
27454 divideCeil(ValueVT.getSizeInBits().getKnownMinValue(),
27455 ValNF * RISCV::RVVBitsPerBlock);
27456 unsigned PartNF = PartVT.getRISCVVectorTupleNumFields();
27457 [[maybe_unused]] unsigned PartLMUL =
27458 divideCeil(PartVT.getSizeInBits().getKnownMinValue(),
27459 PartNF * RISCV::RVVBitsPerBlock);
27460 assert(ValNF == PartNF && ValLMUL == PartLMUL &&
27461 "RISC-V vector tuple type only accepts same register class type "
27462 "TUPLE_INSERT");
27463#endif
27464
27465 Val = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
27466 N2: Val, N3: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
27467 Parts[0] = Val;
27468 return true;
27469 }
27470
27471 if (ValueVT.isFixedLengthVector() && PartVT.isScalableVector()) {
27472 ValueVT = getContainerForFixedLengthVector(VT: ValueVT.getSimpleVT());
27473 Val = convertToScalableVector(VT: ValueVT, V: Val, DAG, Subtarget);
27474
27475 LLVMContext &Context = *DAG.getContext();
27476 EVT ValueEltVT = ValueVT.getVectorElementType();
27477 EVT PartEltVT = PartVT.getVectorElementType();
27478 unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinValue();
27479 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinValue();
27480 if (PartVTBitSize % ValueVTBitSize == 0) {
27481 assert(PartVTBitSize >= ValueVTBitSize);
27482 // If the element types are different, bitcast to the same element type of
27483 // PartVT first.
27484 // Give an example here, we want copy a <vscale x 1 x i8> value to
27485 // <vscale x 4 x i16>.
27486 // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
27487 // subvector, then we can bitcast to <vscale x 4 x i16>.
27488 if (ValueEltVT != PartEltVT) {
27489 if (PartVTBitSize > ValueVTBitSize) {
27490 unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
27491 assert(Count != 0 && "The number of element should not be zero.");
27492 EVT SameEltTypeVT =
27493 EVT::getVectorVT(Context, VT: ValueEltVT, NumElements: Count, /*IsScalable=*/true);
27494 Val = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SameEltTypeVT), SubVec: Val, Idx: 0);
27495 }
27496 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
27497 } else {
27498 Val = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: PartVT), SubVec: Val, Idx: 0);
27499 }
27500 Parts[0] = Val;
27501 return true;
27502 }
27503 }
27504
27505 return false;
27506}
27507
27508SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
27509 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
27510 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
27511 bool IsABIRegCopy = CC.has_value();
27512
27513 MVT PairVT = Subtarget.is64Bit() ? MVT::i128 : MVT::i64;
27514 if ((ValueVT == PairVT ||
27515 (!Subtarget.is64Bit() && Subtarget.hasStdExtZdinx() &&
27516 ValueVT == MVT::f64)) &&
27517 NumParts == 1 && PartVT == MVT::Untyped) {
27518 // Pairs in Inline Assembly, f64 in Inline assembly on rv32_zdinx
27519 MVT XLenVT = Subtarget.getXLenVT();
27520
27521 SDValue Val = Parts[0];
27522 // Always starting with an MVT::Untyped part, so always use
27523 // RISCVISD::SplitGPRPair
27524 Val = DAG.getNode(Opcode: RISCVISD::SplitGPRPair, DL, VTList: DAG.getVTList(VT1: XLenVT, VT2: XLenVT),
27525 N: Val);
27526 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: PairVT, N1: Val.getValue(R: 0),
27527 N2: Val.getValue(R: 1));
27528 if (ValueVT == MVT::f64)
27529 Val = DAG.getBitcast(VT: ValueVT, V: Val);
27530 return Val;
27531 }
27532
27533 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
27534 PartVT == MVT::f32) {
27535 SDValue Val = Parts[0];
27536
27537 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
27538 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Val);
27539 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Val);
27540 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
27541 return Val;
27542 }
27543
27544 if (ValueVT.isFixedLengthVector() && PartVT.isScalableVector()) {
27545 LLVMContext &Context = *DAG.getContext();
27546 SDValue Val = Parts[0];
27547 EVT ValueEltVT = ValueVT.getVectorElementType();
27548 EVT PartEltVT = PartVT.getVectorElementType();
27549
27550 unsigned ValueVTBitSize =
27551 getContainerForFixedLengthVector(VT: ValueVT.getSimpleVT())
27552 .getSizeInBits()
27553 .getKnownMinValue();
27554
27555 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinValue();
27556 if (PartVTBitSize % ValueVTBitSize == 0) {
27557 assert(PartVTBitSize >= ValueVTBitSize);
27558 EVT SameEltTypeVT = ValueVT;
27559 // If the element types are different, convert it to the same element type
27560 // of PartVT.
27561 // Give an example here, we want copy a <vscale x 1 x i8> value from
27562 // <vscale x 4 x i16>.
27563 // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
27564 // then we can extract <vscale x 1 x i8>.
27565 if (ValueEltVT != PartEltVT) {
27566 unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
27567 assert(Count != 0 && "The number of element should not be zero.");
27568 SameEltTypeVT =
27569 EVT::getVectorVT(Context, VT: ValueEltVT, NumElements: Count, /*IsScalable=*/true);
27570 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: SameEltTypeVT, Operand: Val);
27571 }
27572 if (ValueVT.isFixedLengthVector())
27573 Val = convertFromScalableVector(VT: ValueVT, V: Val, DAG, Subtarget);
27574 else
27575 Val = DAG.getExtractSubvector(DL, VT: ValueVT, Vec: Val, Idx: 0);
27576 return Val;
27577 }
27578 }
27579 return SDValue();
27580}
27581
27582bool RISCVTargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
27583 // When aggressively optimizing for code size, we prefer to use a div
27584 // instruction, as it is usually smaller than the alternative sequence.
27585 // TODO: Add vector division?
27586 bool OptSize = Attr.hasFnAttr(Kind: Attribute::MinSize);
27587 return OptSize && !VT.isVector() &&
27588 VT.getSizeInBits() <= getMaxDivRemBitWidthSupported();
27589}
27590
27591void RISCVTargetLowering::finalizeLowering(MachineFunction &MF) const {
27592 MF.getFrameInfo().computeMaxCallFrameSize(MF);
27593 TargetLoweringBase::finalizeLowering(MF);
27594}
27595
27596bool RISCVTargetLowering::preferScalarizeSplat(SDNode *N) const {
27597 // Scalarize zero_ext and sign_ext might stop match to widening instruction in
27598 // some situation.
27599 unsigned Opc = N->getOpcode();
27600 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND)
27601 return false;
27602 return true;
27603}
27604
27605static Value *useTpOffset(IRBuilderBase &IRB, unsigned Offset) {
27606 Module *M = IRB.GetInsertBlock()->getModule();
27607 Function *ThreadPointerFunc = Intrinsic::getOrInsertDeclaration(
27608 M, id: Intrinsic::thread_pointer, OverloadTys: IRB.getPtrTy());
27609 return IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(),
27610 Ptr: IRB.CreateCall(Callee: ThreadPointerFunc), Idx0: Offset);
27611}
27612
27613Value *RISCVTargetLowering::getIRStackGuard(
27614 IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const {
27615 // Fuchsia provides a fixed TLS slot for the stack cookie.
27616 // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
27617 if (Subtarget.isTargetFuchsia())
27618 return useTpOffset(IRB, Offset: -0x10);
27619
27620 // Android provides a fixed TLS slot for the stack cookie. See the definition
27621 // of TLS_SLOT_STACK_GUARD in
27622 // https://android.googlesource.com/platform/bionic/+/main/libc/platform/bionic/tls_defines.h
27623 if (Subtarget.isTargetAndroid())
27624 return useTpOffset(IRB, Offset: -0x18);
27625
27626 Module *M = IRB.GetInsertBlock()->getModule();
27627
27628 if (M->getStackProtectorGuard() == "tls") {
27629 // Users must specify the offset explicitly
27630 int Offset = M->getStackProtectorGuardOffset();
27631 return useTpOffset(IRB, Offset);
27632 }
27633
27634 return TargetLowering::getIRStackGuard(IRB, Libcalls);
27635}
27636
27637bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
27638 Align Alignment) const {
27639 if (!Subtarget.hasVInstructions())
27640 return false;
27641
27642 // Only support fixed vectors if we know the minimum vector size.
27643 if (DataType.isFixedLengthVector() && !Subtarget.useRVVForFixedLengthVectors())
27644 return false;
27645
27646 EVT ScalarType = DataType.getScalarType();
27647 if (!isLegalElementTypeForRVV(ScalarTy: ScalarType))
27648 return false;
27649
27650 if (!Subtarget.enableUnalignedVectorMem() &&
27651 Alignment < ScalarType.getStoreSize())
27652 return false;
27653
27654 return true;
27655}
27656
27657bool RISCVTargetLowering::isLegalFirstFaultLoad(EVT DataType,
27658 Align Alignment) const {
27659 if (!Subtarget.hasVInstructions())
27660 return false;
27661
27662 EVT ScalarType = DataType.getScalarType();
27663 if (!isLegalElementTypeForRVV(ScalarTy: ScalarType))
27664 return false;
27665
27666 if (!Subtarget.enableUnalignedVectorMem() &&
27667 Alignment < ScalarType.getStoreSize())
27668 return false;
27669
27670 return true;
27671}
27672
27673MachineInstr *
27674RISCVTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
27675 MachineBasicBlock::instr_iterator &MBBI,
27676 const TargetInstrInfo *TII) const {
27677 assert(MBBI->isCall() && MBBI->getCFIType() &&
27678 "Invalid call instruction for a KCFI check");
27679 assert(is_contained({RISCV::PseudoCALLIndirect, RISCV::PseudoTAILIndirect},
27680 MBBI->getOpcode()));
27681
27682 MachineOperand &Target = MBBI->getOperand(i: 0);
27683 Target.setIsRenamable(false);
27684
27685 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: RISCV::KCFI_CHECK))
27686 .addReg(RegNo: Target.getReg())
27687 .addImm(Val: MBBI->getCFIType())
27688 .getInstr();
27689}
27690
27691#define GET_REGISTER_MATCHER
27692#include "RISCVGenAsmMatcher.inc"
27693
27694Register
27695RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
27696 const MachineFunction &MF) const {
27697 Register Reg = MatchRegisterAltName(Name: RegName);
27698 if (!Reg)
27699 Reg = MatchRegisterName(Name: RegName);
27700 if (!Reg)
27701 return Reg;
27702
27703 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
27704 if (!ReservedRegs.test(Idx: Reg) && !Subtarget.isRegisterReservedByUser(i: Reg))
27705 reportFatalUsageError(reason: Twine("Trying to obtain non-reserved register \"" +
27706 StringRef(RegName) + "\"."));
27707 return Reg;
27708}
27709
27710MachineMemOperand::Flags
27711RISCVTargetLowering::getTargetMMOFlags(const Instruction &I) const {
27712 const MDNode *NontemporalInfo = I.getMetadata(KindID: LLVMContext::MD_nontemporal);
27713
27714 if (NontemporalInfo == nullptr)
27715 return MachineMemOperand::MONone;
27716
27717 // 1 for default value work as __RISCV_NTLH_ALL
27718 // 2 -> __RISCV_NTLH_INNERMOST_PRIVATE
27719 // 3 -> __RISCV_NTLH_ALL_PRIVATE
27720 // 4 -> __RISCV_NTLH_INNERMOST_SHARED
27721 // 5 -> __RISCV_NTLH_ALL
27722 int NontemporalLevel = 5;
27723 const MDNode *RISCVNontemporalInfo =
27724 I.getMetadata(Kind: "riscv-nontemporal-domain");
27725 if (RISCVNontemporalInfo != nullptr)
27726 NontemporalLevel =
27727 cast<ConstantInt>(
27728 Val: cast<ConstantAsMetadata>(Val: RISCVNontemporalInfo->getOperand(I: 0))
27729 ->getValue())
27730 ->getZExtValue();
27731
27732 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
27733 "RISC-V target doesn't support this non-temporal domain.");
27734
27735 NontemporalLevel -= 2;
27736 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
27737 if (NontemporalLevel & 0b1)
27738 Flags |= MONontemporalBit0;
27739 if (NontemporalLevel & 0b10)
27740 Flags |= MONontemporalBit1;
27741
27742 return Flags;
27743}
27744
27745MachineMemOperand::Flags
27746RISCVTargetLowering::getTargetMMOFlags(const MemSDNode &Node) const {
27747
27748 MachineMemOperand::Flags NodeFlags = Node.getMemOperand()->getFlags();
27749 MachineMemOperand::Flags TargetFlags = MachineMemOperand::MONone;
27750 TargetFlags |= (NodeFlags & MONontemporalBit0);
27751 TargetFlags |= (NodeFlags & MONontemporalBit1);
27752 return TargetFlags;
27753}
27754
27755bool RISCVTargetLowering::areTwoSDNodeTargetMMOFlagsMergeable(
27756 const MemSDNode &NodeX, const MemSDNode &NodeY) const {
27757 return getTargetMMOFlags(Node: NodeX) == getTargetMMOFlags(Node: NodeY);
27758}
27759
27760bool RISCVTargetLowering::isCtpopFast(EVT VT) const {
27761 if (VT.isVector()) {
27762 EVT SVT = VT.getVectorElementType();
27763 // If the element type is legal we can use cpop.v if it is enabled.
27764 if (isLegalElementTypeForRVV(ScalarTy: SVT))
27765 return Subtarget.hasStdExtZvbb();
27766 // Don't consider it fast if the type needs to be legalized or scalarized.
27767 return false;
27768 }
27769
27770 return Subtarget.hasCPOPLike() && (VT == MVT::i32 || VT == MVT::i64);
27771}
27772
27773unsigned RISCVTargetLowering::getCustomCtpopCost(EVT VT,
27774 ISD::CondCode Cond) const {
27775 return isCtpopFast(VT) ? 0 : 1;
27776}
27777
27778bool RISCVTargetLowering::shouldInsertFencesForAtomic(
27779 const Instruction *I) const {
27780 if (Subtarget.hasStdExtZalasr()) {
27781 if (Subtarget.hasStdExtZtso()) {
27782 // Zalasr + TSO means that atomic_load_acquire and atomic_store_release
27783 // should be lowered to plain load/store. The easiest way to do this is
27784 // to say we should insert fences for them, and the fence insertion code
27785 // will just not insert any fences
27786 auto *LI = dyn_cast<LoadInst>(Val: I);
27787 auto *SI = dyn_cast<StoreInst>(Val: I);
27788 if ((LI &&
27789 (LI->getOrdering() == AtomicOrdering::SequentiallyConsistent)) ||
27790 (SI &&
27791 (SI->getOrdering() == AtomicOrdering::SequentiallyConsistent))) {
27792 // Here, this is a load or store which is seq_cst, and needs a .aq or
27793 // .rl therefore we shouldn't try to insert fences
27794 return false;
27795 }
27796 // Here, we are a TSO inst that isn't a seq_cst load/store
27797 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
27798 }
27799 return false;
27800 }
27801 // Note that one specific case requires fence insertion for an
27802 // AtomicCmpXchgInst but is handled via the RISCVZacasABIFix pass rather
27803 // than this hook due to limitations in the interface here.
27804 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
27805}
27806
27807bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
27808
27809 // GISel support is in progress or complete for these opcodes.
27810 unsigned Op = Inst.getOpcode();
27811 if (Op == Instruction::Add || Op == Instruction::Sub ||
27812 Op == Instruction::And || Op == Instruction::Or ||
27813 Op == Instruction::Xor || Op == Instruction::InsertElement ||
27814 Op == Instruction::ShuffleVector || Op == Instruction::Load ||
27815 Op == Instruction::Freeze || Op == Instruction::Store)
27816 return false;
27817
27818 if (auto *II = dyn_cast<IntrinsicInst>(Val: &Inst)) {
27819 // Mark RVV intrinsic as supported.
27820 if (RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: II->getIntrinsicID())) {
27821 // GISel doesn't support tuple types yet. It also doesn't suport returning
27822 // a struct containing a scalable vector like vleff.
27823 if (Inst.getType()->isRISCVVectorTupleTy() ||
27824 Inst.getType()->isStructTy())
27825 return true;
27826
27827 for (unsigned i = 0; i < II->arg_size(); ++i)
27828 if (II->getArgOperand(i)->getType()->isRISCVVectorTupleTy())
27829 return true;
27830
27831 return false;
27832 }
27833 if (II->getIntrinsicID() == Intrinsic::vector_extract ||
27834 II->getIntrinsicID() == Intrinsic::vector_insert)
27835 return false;
27836 }
27837
27838 if (Inst.getType()->isScalableTy())
27839 return true;
27840
27841 for (unsigned i = 0; i < Inst.getNumOperands(); ++i)
27842 if (Inst.getOperand(i)->getType()->isScalableTy() &&
27843 !isa<ReturnInst>(Val: &Inst))
27844 return true;
27845
27846 return false;
27847}
27848
27849SDValue
27850RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
27851 SelectionDAG &DAG,
27852 SmallVectorImpl<SDNode *> &Created) const {
27853 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
27854 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
27855 return SDValue(N, 0); // Lower SDIV as SDIV
27856
27857 // Only perform this transform if short forward branch opt is supported.
27858 if (!Subtarget.hasShortForwardBranchIALU())
27859 return SDValue();
27860 EVT VT = N->getValueType(ResNo: 0);
27861 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit())))
27862 return SDValue();
27863
27864 // Ensure 2**k-1 < 2048 so that we can just emit a single addi/addiw.
27865 if (Divisor.sgt(RHS: 2048) || Divisor.slt(RHS: -2048))
27866 return SDValue();
27867 return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
27868}
27869
27870bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
27871 EVT VT, const APInt &AndMask) const {
27872 if (Subtarget.hasCZEROLike() || Subtarget.hasVendorXTHeadCondMov())
27873 return !Subtarget.hasBEXTILike() && AndMask.ugt(RHS: 1024);
27874 return TargetLowering::shouldFoldSelectWithSingleBitTest(VT, AndMask);
27875}
27876
27877unsigned RISCVTargetLowering::getMinimumJumpTableEntries() const {
27878 return Subtarget.getMinimumJumpTableEntries();
27879}
27880
27881SDValue RISCVTargetLowering::expandIndirectJTBranch(const SDLoc &dl,
27882 SDValue Value, SDValue Addr,
27883 int JTI,
27884 SelectionDAG &DAG) const {
27885 const MachineFunction &MF = DAG.getMachineFunction();
27886 if (MF.getInfo<RISCVMachineFunctionInfo>()->hasCFProtectionBranch()) {
27887 // When cf-protection-branch enabled, we need to use software guarded
27888 // branch for jump table branch.
27889 SDValue Chain = Value;
27890 // Jump table debug info is only needed if CodeView is enabled.
27891 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF())
27892 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
27893 return DAG.getNode(Opcode: RISCVISD::SW_GUARDED_BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
27894 }
27895 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
27896}
27897
27898// If an output pattern produces multiple instructions tablegen may pick an
27899// arbitrary type from an instructions destination register class to use for the
27900// VT of that MachineSDNode. This VT may be used to look up the representative
27901// register class. If the type isn't legal, the default implementation will
27902// not find a register class.
27903//
27904// Some integer types smaller than XLen are listed in the GPR register class to
27905// support isel patterns for GISel, but are not legal in SelectionDAG. The
27906// arbitrary type tablegen picks may be one of these smaller types.
27907//
27908// f16 and bf16 are both valid for the FPR16 or GPRF16 register class. It's
27909// possible for tablegen to pick bf16 as the arbitrary type for an f16 pattern.
27910std::pair<const TargetRegisterClass *, uint8_t>
27911RISCVTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
27912 MVT VT) const {
27913 switch (VT.SimpleTy) {
27914 default:
27915 break;
27916 case MVT::i8:
27917 case MVT::i16:
27918 case MVT::i32:
27919 return TargetLowering::findRepresentativeClass(TRI, VT: Subtarget.getXLenVT());
27920 case MVT::bf16:
27921 case MVT::f16:
27922 return TargetLowering::findRepresentativeClass(TRI, VT: MVT::f32);
27923 }
27924
27925 return TargetLowering::findRepresentativeClass(TRI, VT);
27926}
27927
27928namespace llvm::RISCVVIntrinsicsTable {
27929
27930#define GET_RISCVVIntrinsicsTable_IMPL
27931#include "RISCVGenSearchableTables.inc"
27932
27933} // namespace llvm::RISCVVIntrinsicsTable
27934
27935bool RISCVTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
27936
27937 // If the function specifically requests inline stack probes, emit them.
27938 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
27939 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
27940 "inline-asm";
27941
27942 return false;
27943}
27944
27945unsigned RISCVTargetLowering::getStackProbeSize(const MachineFunction &MF,
27946 Align StackAlign) const {
27947 // The default stack probe size is 4096 if the function has no
27948 // stack-probe-size attribute.
27949 const Function &Fn = MF.getFunction();
27950 unsigned StackProbeSize =
27951 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
27952 // Round down to the stack alignment.
27953 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
27954 return StackProbeSize ? StackProbeSize : StackAlign.value();
27955}
27956
27957SDValue RISCVTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
27958 SelectionDAG &DAG) const {
27959 MachineFunction &MF = DAG.getMachineFunction();
27960 if (!hasInlineStackProbe(MF))
27961 return SDValue();
27962
27963 MVT XLenVT = Subtarget.getXLenVT();
27964 // Get the inputs.
27965 SDValue Chain = Op.getOperand(i: 0);
27966 SDValue Size = Op.getOperand(i: 1);
27967
27968 MaybeAlign Align =
27969 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
27970 SDLoc dl(Op);
27971 EVT VT = Op.getValueType();
27972
27973 // Construct the new SP value in a GPR.
27974 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: RISCV::X2, VT: XLenVT);
27975 Chain = SP.getValue(R: 1);
27976 SP = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: XLenVT, N1: SP, N2: Size);
27977 if (Align)
27978 SP = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SP.getValue(R: 0),
27979 N2: DAG.getSignedConstant(Val: -Align->value(), DL: dl, VT));
27980
27981 // Set the real SP to the new value with a probing loop.
27982 Chain = DAG.getNode(Opcode: RISCVISD::PROBED_ALLOCA, DL: dl, VT: MVT::Other, N1: Chain, N2: SP);
27983 return DAG.getMergeValues(Ops: {SP, Chain}, dl);
27984}
27985
27986MachineBasicBlock *
27987RISCVTargetLowering::emitDynamicProbedAlloc(MachineInstr &MI,
27988 MachineBasicBlock *MBB) const {
27989 MachineFunction &MF = *MBB->getParent();
27990 MachineBasicBlock::iterator MBBI = MI.getIterator();
27991 DebugLoc DL = MBB->findDebugLoc(MBBI);
27992 Register TargetReg = MI.getOperand(i: 0).getReg();
27993
27994 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
27995 bool IsRV64 = Subtarget.is64Bit();
27996 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
27997 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
27998 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
27999
28000 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB->getIterator());
28001 MachineBasicBlock *LoopTestMBB =
28002 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
28003 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
28004 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
28005 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
28006 Register SPReg = RISCV::X2;
28007 Register ScratchReg =
28008 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
28009
28010 // ScratchReg = ProbeSize
28011 TII->movImm(MBB&: *MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: MachineInstr::NoFlags);
28012
28013 // LoopTest:
28014 // SUB SP, SP, ProbeSize
28015 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
28016 .addReg(RegNo: SPReg)
28017 .addReg(RegNo: ScratchReg);
28018
28019 // s[d|w] zero, 0(sp)
28020 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
28021 MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
28022 .addReg(RegNo: RISCV::X0)
28023 .addReg(RegNo: SPReg)
28024 .addImm(Val: 0);
28025
28026 // BLTU TargetReg, SP, LoopTest
28027 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BLTU))
28028 .addReg(RegNo: TargetReg)
28029 .addReg(RegNo: SPReg)
28030 .addMBB(MBB: LoopTestMBB);
28031
28032 // Adjust with: MV SP, TargetReg.
28033 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: SPReg)
28034 .addReg(RegNo: TargetReg)
28035 .addImm(Val: 0);
28036
28037 ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end());
28038 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
28039
28040 LoopTestMBB->addSuccessor(Succ: ExitMBB);
28041 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
28042 MBB->addSuccessor(Succ: LoopTestMBB);
28043
28044 MI.eraseFromParent();
28045 MF.getInfo<RISCVMachineFunctionInfo>()->setDynamicAllocation();
28046 return ExitMBB->begin()->getParent();
28047}
28048
28049ArrayRef<MCPhysReg> RISCVTargetLowering::getRoundingControlRegisters() const {
28050 if (Subtarget.hasStdExtFOrZfinx()) {
28051 static const MCPhysReg RCRegs[] = {RISCV::FRM, RISCV::FFLAGS};
28052 return RCRegs;
28053 }
28054 return {};
28055}
28056
28057bool RISCVTargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
28058 EVT VT = Y.getValueType();
28059
28060 if (VT.isVector())
28061 return false;
28062
28063 return VT.getSizeInBits() <= Subtarget.getXLen();
28064}
28065
28066bool RISCVTargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
28067 SDValue N1) const {
28068 if (!N0.hasOneUse())
28069 return false;
28070
28071 // Avoid reassociating expressions that can be lowered to vector
28072 // multiply accumulate (i.e. add (mul x, y), z)
28073 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::MUL &&
28074 (N0.getValueType().isVector() && Subtarget.hasVInstructions()))
28075 return false;
28076
28077 return true;
28078}
28079