1//===-- RISCVISelLowering.cpp - RISC-V DAG Lowering Implementation -------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the interfaces that RISC-V uses to lower LLVM code into a
10// selection DAG.
11//
12//===----------------------------------------------------------------------===//
13
14#include "RISCVISelLowering.h"
15#include "MCTargetDesc/RISCVMatInt.h"
16#include "RISCV.h"
17#include "RISCVConstantPoolValue.h"
18#include "RISCVMachineFunctionInfo.h"
19#include "RISCVRegisterInfo.h"
20#include "RISCVSelectionDAGInfo.h"
21#include "RISCVSubtarget.h"
22#include "llvm/ADT/SmallSet.h"
23#include "llvm/ADT/SmallVector.h"
24#include "llvm/ADT/Statistic.h"
25#include "llvm/Analysis/MemoryLocation.h"
26#include "llvm/Analysis/ValueTracking.h"
27#include "llvm/Analysis/VectorUtils.h"
28#include "llvm/CodeGen/MachineFrameInfo.h"
29#include "llvm/CodeGen/MachineFunction.h"
30#include "llvm/CodeGen/MachineInstrBuilder.h"
31#include "llvm/CodeGen/MachineJumpTableInfo.h"
32#include "llvm/CodeGen/MachineRegisterInfo.h"
33#include "llvm/CodeGen/SDPatternMatch.h"
34#include "llvm/CodeGen/SelectionDAGAddressAnalysis.h"
35#include "llvm/CodeGen/TargetLoweringObjectFileImpl.h"
36#include "llvm/CodeGen/ValueTypes.h"
37#include "llvm/IR/DiagnosticInfo.h"
38#include "llvm/IR/DiagnosticPrinter.h"
39#include "llvm/IR/IRBuilder.h"
40#include "llvm/IR/Instructions.h"
41#include "llvm/IR/IntrinsicInst.h"
42#include "llvm/IR/IntrinsicsRISCV.h"
43#include "llvm/MC/MCCodeEmitter.h"
44#include "llvm/MC/MCInstBuilder.h"
45#include "llvm/Support/CommandLine.h"
46#include "llvm/Support/Debug.h"
47#include "llvm/Support/ErrorHandling.h"
48#include "llvm/Support/InstructionCost.h"
49#include "llvm/Support/KnownBits.h"
50#include "llvm/Support/MathExtras.h"
51#include "llvm/Support/raw_ostream.h"
52#include <optional>
53
54using namespace llvm;
55
56#define DEBUG_TYPE "riscv-lower"
57
58STATISTIC(NumTailCalls, "Number of tail calls");
59
60static cl::opt<unsigned> ExtensionMaxWebSize(
61 DEBUG_TYPE "-ext-max-web-size", cl::Hidden,
62 cl::desc("Give the maximum size (in number of nodes) of the web of "
63 "instructions that we will consider for VW expansion"),
64 cl::init(Val: 18));
65
66static cl::opt<bool>
67 AllowSplatInVW_W(DEBUG_TYPE "-form-vw-w-with-splat", cl::Hidden,
68 cl::desc("Allow the formation of VW_W operations (e.g., "
69 "VWADD_W) with splat constants"),
70 cl::init(Val: false));
71
72static cl::opt<unsigned> NumRepeatedDivisors(
73 DEBUG_TYPE "-fp-repeated-divisors", cl::Hidden,
74 cl::desc("Set the minimum number of repetitions of a divisor to allow "
75 "transformation to multiplications by the reciprocal"),
76 cl::init(Val: 2));
77
78static cl::opt<int>
79 FPImmCost(DEBUG_TYPE "-fpimm-cost", cl::Hidden,
80 cl::desc("Give the maximum number of instructions that we will "
81 "use for creating a floating-point immediate value"),
82 cl::init(Val: 3));
83
84static cl::opt<bool>
85 ReassocShlAddiAdd("reassoc-shl-addi-add", cl::Hidden,
86 cl::desc("Swap add and addi in cases where the add may "
87 "be combined with a shift"),
88 cl::init(Val: true));
89
90// TODO: Support more ops
91static const unsigned ZvfbfaVPOps[] = {
92 ISD::VP_FNEG, ISD::VP_FABS, ISD::VP_FCOPYSIGN};
93static const unsigned ZvfbfaOps[] = {
94 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FADD,
95 ISD::FSUB, ISD::FMUL, ISD::FMINNUM, ISD::FMAXNUM,
96 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUM, ISD::FMAXIMUM,
97 ISD::FMA, ISD::IS_FPCLASS, ISD::STRICT_FADD, ISD::STRICT_FSUB,
98 ISD::STRICT_FMUL, ISD::STRICT_FMA, ISD::SETCC};
99
100RISCVTargetLowering::RISCVTargetLowering(const TargetMachine &TM,
101 const RISCVSubtarget &STI)
102 : TargetLowering(TM, STI), Subtarget(STI) {
103
104 RISCVABI::ABI ABI = Subtarget.getTargetABI();
105 assert(ABI != RISCVABI::ABI_Unknown && "Improperly initialised target ABI");
106
107 if ((ABI == RISCVABI::ABI_ILP32F || ABI == RISCVABI::ABI_LP64F) &&
108 !Subtarget.hasStdExtF()) {
109 errs() << "Hard-float 'f' ABI can't be used for a target that "
110 "doesn't support the F instruction set extension (ignoring "
111 "target-abi)\n";
112 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
113 } else if ((ABI == RISCVABI::ABI_ILP32D || ABI == RISCVABI::ABI_LP64D) &&
114 !Subtarget.hasStdExtD()) {
115 errs() << "Hard-float 'd' ABI can't be used for a target that "
116 "doesn't support the D instruction set extension (ignoring "
117 "target-abi)\n";
118 ABI = Subtarget.is64Bit() ? RISCVABI::ABI_LP64 : RISCVABI::ABI_ILP32;
119 }
120
121 switch (ABI) {
122 default:
123 reportFatalUsageError(reason: "Don't know how to lower this ABI");
124 case RISCVABI::ABI_ILP32:
125 case RISCVABI::ABI_ILP32E:
126 case RISCVABI::ABI_LP64E:
127 case RISCVABI::ABI_ILP32F:
128 case RISCVABI::ABI_ILP32D:
129 case RISCVABI::ABI_LP64:
130 case RISCVABI::ABI_LP64F:
131 case RISCVABI::ABI_LP64D:
132 break;
133 }
134
135 MVT XLenVT = Subtarget.getXLenVT();
136
137 // Set up the register classes.
138 addRegisterClass(VT: XLenVT, RC: &RISCV::GPRRegClass);
139
140 if (Subtarget.hasStdExtZfhmin())
141 addRegisterClass(VT: MVT::f16, RC: &RISCV::FPR16RegClass);
142 if (Subtarget.hasStdExtZfbfmin() || Subtarget.hasVendorXAndesBFHCvt())
143 addRegisterClass(VT: MVT::bf16, RC: &RISCV::FPR16RegClass);
144 if (Subtarget.hasStdExtF())
145 addRegisterClass(VT: MVT::f32, RC: &RISCV::FPR32RegClass);
146 if (Subtarget.hasStdExtD())
147 addRegisterClass(VT: MVT::f64, RC: &RISCV::FPR64RegClass);
148 if (Subtarget.hasStdExtZhinxmin())
149 addRegisterClass(VT: MVT::f16, RC: &RISCV::GPRF16RegClass);
150 if (Subtarget.hasStdExtZfinx())
151 addRegisterClass(VT: MVT::f32, RC: &RISCV::GPRF32RegClass);
152 if (Subtarget.hasStdExtZdinx()) {
153 if (Subtarget.is64Bit())
154 addRegisterClass(VT: MVT::f64, RC: &RISCV::GPRRegClass);
155 else
156 addRegisterClass(VT: MVT::f64, RC: &RISCV::GPRPairRegClass);
157 }
158
159 static const MVT::SimpleValueType BoolVecVTs[] = {
160 MVT::nxv1i1, MVT::nxv2i1, MVT::nxv4i1, MVT::nxv8i1,
161 MVT::nxv16i1, MVT::nxv32i1, MVT::nxv64i1};
162 static const MVT::SimpleValueType IntVecVTs[] = {
163 MVT::nxv1i8, MVT::nxv2i8, MVT::nxv4i8, MVT::nxv8i8, MVT::nxv16i8,
164 MVT::nxv32i8, MVT::nxv64i8, MVT::nxv1i16, MVT::nxv2i16, MVT::nxv4i16,
165 MVT::nxv8i16, MVT::nxv16i16, MVT::nxv32i16, MVT::nxv1i32, MVT::nxv2i32,
166 MVT::nxv4i32, MVT::nxv8i32, MVT::nxv16i32, MVT::nxv1i64, MVT::nxv2i64,
167 MVT::nxv4i64, MVT::nxv8i64};
168 static const MVT::SimpleValueType F16VecVTs[] = {
169 MVT::nxv1f16, MVT::nxv2f16, MVT::nxv4f16,
170 MVT::nxv8f16, MVT::nxv16f16, MVT::nxv32f16};
171 static const MVT::SimpleValueType BF16VecVTs[] = {
172 MVT::nxv1bf16, MVT::nxv2bf16, MVT::nxv4bf16,
173 MVT::nxv8bf16, MVT::nxv16bf16, MVT::nxv32bf16};
174 static const MVT::SimpleValueType F32VecVTs[] = {
175 MVT::nxv1f32, MVT::nxv2f32, MVT::nxv4f32, MVT::nxv8f32, MVT::nxv16f32};
176 static const MVT::SimpleValueType F64VecVTs[] = {
177 MVT::nxv1f64, MVT::nxv2f64, MVT::nxv4f64, MVT::nxv8f64};
178 static const MVT::SimpleValueType VecTupleVTs[] = {
179 MVT::riscv_nxv1i8x2, MVT::riscv_nxv1i8x3, MVT::riscv_nxv1i8x4,
180 MVT::riscv_nxv1i8x5, MVT::riscv_nxv1i8x6, MVT::riscv_nxv1i8x7,
181 MVT::riscv_nxv1i8x8, MVT::riscv_nxv2i8x2, MVT::riscv_nxv2i8x3,
182 MVT::riscv_nxv2i8x4, MVT::riscv_nxv2i8x5, MVT::riscv_nxv2i8x6,
183 MVT::riscv_nxv2i8x7, MVT::riscv_nxv2i8x8, MVT::riscv_nxv4i8x2,
184 MVT::riscv_nxv4i8x3, MVT::riscv_nxv4i8x4, MVT::riscv_nxv4i8x5,
185 MVT::riscv_nxv4i8x6, MVT::riscv_nxv4i8x7, MVT::riscv_nxv4i8x8,
186 MVT::riscv_nxv8i8x2, MVT::riscv_nxv8i8x3, MVT::riscv_nxv8i8x4,
187 MVT::riscv_nxv8i8x5, MVT::riscv_nxv8i8x6, MVT::riscv_nxv8i8x7,
188 MVT::riscv_nxv8i8x8, MVT::riscv_nxv16i8x2, MVT::riscv_nxv16i8x3,
189 MVT::riscv_nxv16i8x4, MVT::riscv_nxv32i8x2};
190
191 if (Subtarget.hasVInstructions()) {
192 auto addRegClassForRVV = [this](MVT VT) {
193 // Disable the smallest fractional LMUL types if ELEN is less than
194 // RVVBitsPerBlock.
195 unsigned MinElts = RISCV::RVVBitsPerBlock / Subtarget.getELen();
196 if (VT.getVectorMinNumElements() < MinElts)
197 return;
198
199 unsigned Size = VT.getSizeInBits().getKnownMinValue();
200 const TargetRegisterClass *RC;
201 if (Size <= RISCV::RVVBitsPerBlock)
202 RC = &RISCV::VRRegClass;
203 else if (Size == 2 * RISCV::RVVBitsPerBlock)
204 RC = &RISCV::VRM2RegClass;
205 else if (Size == 4 * RISCV::RVVBitsPerBlock)
206 RC = &RISCV::VRM4RegClass;
207 else if (Size == 8 * RISCV::RVVBitsPerBlock)
208 RC = &RISCV::VRM8RegClass;
209 else
210 llvm_unreachable("Unexpected size");
211
212 addRegisterClass(VT, RC);
213 };
214
215 for (MVT VT : BoolVecVTs)
216 addRegClassForRVV(VT);
217 for (MVT VT : IntVecVTs) {
218 if (VT.getVectorElementType() == MVT::i64 &&
219 !Subtarget.hasVInstructionsI64())
220 continue;
221 addRegClassForRVV(VT);
222 }
223
224 if (Subtarget.hasVInstructionsF16Minimal() ||
225 Subtarget.hasVendorXAndesVPackFPH())
226 for (MVT VT : F16VecVTs)
227 addRegClassForRVV(VT);
228
229 if (Subtarget.hasVInstructionsBF16Minimal() ||
230 Subtarget.hasVendorXAndesVBFHCvt())
231 for (MVT VT : BF16VecVTs)
232 addRegClassForRVV(VT);
233
234 if (Subtarget.hasVInstructionsF32())
235 for (MVT VT : F32VecVTs)
236 addRegClassForRVV(VT);
237
238 if (Subtarget.hasVInstructionsF64())
239 for (MVT VT : F64VecVTs)
240 addRegClassForRVV(VT);
241
242 if (Subtarget.useRVVForFixedLengthVectors()) {
243 auto addRegClassForFixedVectors = [this](MVT VT) {
244 MVT ContainerVT = getContainerForFixedLengthVector(VT);
245 unsigned RCID = getRegClassIDForVecVT(VT: ContainerVT);
246 const RISCVRegisterInfo &TRI = *Subtarget.getRegisterInfo();
247 addRegisterClass(VT, RC: TRI.getRegClass(i: RCID));
248 };
249 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes())
250 if (useRVVForFixedLengthVectorVT(VT))
251 addRegClassForFixedVectors(VT);
252
253 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes())
254 if (useRVVForFixedLengthVectorVT(VT))
255 addRegClassForFixedVectors(VT);
256 }
257
258 addRegisterClass(VT: MVT::riscv_nxv1i8x2, RC: &RISCV::VRN2M1RegClass);
259 addRegisterClass(VT: MVT::riscv_nxv1i8x3, RC: &RISCV::VRN3M1RegClass);
260 addRegisterClass(VT: MVT::riscv_nxv1i8x4, RC: &RISCV::VRN4M1RegClass);
261 addRegisterClass(VT: MVT::riscv_nxv1i8x5, RC: &RISCV::VRN5M1RegClass);
262 addRegisterClass(VT: MVT::riscv_nxv1i8x6, RC: &RISCV::VRN6M1RegClass);
263 addRegisterClass(VT: MVT::riscv_nxv1i8x7, RC: &RISCV::VRN7M1RegClass);
264 addRegisterClass(VT: MVT::riscv_nxv1i8x8, RC: &RISCV::VRN8M1RegClass);
265 addRegisterClass(VT: MVT::riscv_nxv2i8x2, RC: &RISCV::VRN2M1RegClass);
266 addRegisterClass(VT: MVT::riscv_nxv2i8x3, RC: &RISCV::VRN3M1RegClass);
267 addRegisterClass(VT: MVT::riscv_nxv2i8x4, RC: &RISCV::VRN4M1RegClass);
268 addRegisterClass(VT: MVT::riscv_nxv2i8x5, RC: &RISCV::VRN5M1RegClass);
269 addRegisterClass(VT: MVT::riscv_nxv2i8x6, RC: &RISCV::VRN6M1RegClass);
270 addRegisterClass(VT: MVT::riscv_nxv2i8x7, RC: &RISCV::VRN7M1RegClass);
271 addRegisterClass(VT: MVT::riscv_nxv2i8x8, RC: &RISCV::VRN8M1RegClass);
272 addRegisterClass(VT: MVT::riscv_nxv4i8x2, RC: &RISCV::VRN2M1RegClass);
273 addRegisterClass(VT: MVT::riscv_nxv4i8x3, RC: &RISCV::VRN3M1RegClass);
274 addRegisterClass(VT: MVT::riscv_nxv4i8x4, RC: &RISCV::VRN4M1RegClass);
275 addRegisterClass(VT: MVT::riscv_nxv4i8x5, RC: &RISCV::VRN5M1RegClass);
276 addRegisterClass(VT: MVT::riscv_nxv4i8x6, RC: &RISCV::VRN6M1RegClass);
277 addRegisterClass(VT: MVT::riscv_nxv4i8x7, RC: &RISCV::VRN7M1RegClass);
278 addRegisterClass(VT: MVT::riscv_nxv4i8x8, RC: &RISCV::VRN8M1RegClass);
279 addRegisterClass(VT: MVT::riscv_nxv8i8x2, RC: &RISCV::VRN2M1RegClass);
280 addRegisterClass(VT: MVT::riscv_nxv8i8x3, RC: &RISCV::VRN3M1RegClass);
281 addRegisterClass(VT: MVT::riscv_nxv8i8x4, RC: &RISCV::VRN4M1RegClass);
282 addRegisterClass(VT: MVT::riscv_nxv8i8x5, RC: &RISCV::VRN5M1RegClass);
283 addRegisterClass(VT: MVT::riscv_nxv8i8x6, RC: &RISCV::VRN6M1RegClass);
284 addRegisterClass(VT: MVT::riscv_nxv8i8x7, RC: &RISCV::VRN7M1RegClass);
285 addRegisterClass(VT: MVT::riscv_nxv8i8x8, RC: &RISCV::VRN8M1RegClass);
286 addRegisterClass(VT: MVT::riscv_nxv16i8x2, RC: &RISCV::VRN2M2RegClass);
287 addRegisterClass(VT: MVT::riscv_nxv16i8x3, RC: &RISCV::VRN3M2RegClass);
288 addRegisterClass(VT: MVT::riscv_nxv16i8x4, RC: &RISCV::VRN4M2RegClass);
289 addRegisterClass(VT: MVT::riscv_nxv32i8x2, RC: &RISCV::VRN2M4RegClass);
290 }
291
292 // fixed vector is stored in GPRs for P extension packed operations
293 if (Subtarget.hasStdExtP()) {
294 if (Subtarget.is64Bit()) {
295 addRegisterClass(VT: MVT::v2i32, RC: &RISCV::GPRRegClass);
296 addRegisterClass(VT: MVT::v4i16, RC: &RISCV::GPRRegClass);
297 addRegisterClass(VT: MVT::v8i8, RC: &RISCV::GPRRegClass);
298 } else {
299 addRegisterClass(VT: MVT::v2i16, RC: &RISCV::GPRRegClass);
300 addRegisterClass(VT: MVT::v4i8, RC: &RISCV::GPRRegClass);
301 }
302 }
303
304 // Compute derived properties from the register classes.
305 computeRegisterProperties(TRI: STI.getRegisterInfo());
306
307 setStackPointerRegisterToSaveRestore(RISCV::X2);
308
309 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: XLenVT,
310 MemVT: MVT::i1, Action: Promote);
311 // DAGCombiner can call isLoadExtLegal for types that aren't legal.
312 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: MVT::i32,
313 MemVT: MVT::i1, Action: Promote);
314
315 // TODO: add all necessary setOperationAction calls.
316 setOperationAction(Op: ISD::DYNAMIC_STACKALLOC, VT: XLenVT, Action: Custom);
317
318 setOperationAction(Op: ISD::BR_JT, VT: MVT::Other, Action: Expand);
319 setOperationAction(Op: ISD::BR_CC, VT: XLenVT, Action: Expand);
320 setOperationAction(Op: ISD::BRCOND, VT: MVT::Other, Action: Custom);
321 setOperationAction(Op: ISD::SELECT_CC, VT: XLenVT, Action: Expand);
322
323 setCondCodeAction(CCs: ISD::SETGT, VT: XLenVT, Action: Custom);
324 setCondCodeAction(CCs: ISD::SETGE, VT: XLenVT, Action: Expand);
325 setCondCodeAction(CCs: ISD::SETUGT, VT: XLenVT, Action: Custom);
326 setCondCodeAction(CCs: ISD::SETUGE, VT: XLenVT, Action: Expand);
327 if (!(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
328 setCondCodeAction(CCs: ISD::SETULE, VT: XLenVT, Action: Expand);
329 setCondCodeAction(CCs: ISD::SETLE, VT: XLenVT, Action: Expand);
330 }
331
332 setOperationAction(Ops: {ISD::STACKSAVE, ISD::STACKRESTORE}, VT: MVT::Other, Action: Expand);
333
334 setOperationAction(Op: ISD::VASTART, VT: MVT::Other, Action: Custom);
335 setOperationAction(Ops: {ISD::VAARG, ISD::VACOPY, ISD::VAEND}, VT: MVT::Other, Action: Expand);
336
337 if (!Subtarget.hasVendorXTHeadBb() && !Subtarget.hasVendorXqcibm() &&
338 !Subtarget.hasVendorXAndesPerf())
339 setOperationAction(Op: ISD::SIGN_EXTEND_INREG, VT: MVT::i1, Action: Expand);
340
341 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i32, Action: Custom);
342
343 if (!Subtarget.hasStdExtZbb() && !Subtarget.hasVendorXTHeadBb() &&
344 !Subtarget.hasVendorXqcibm() && !Subtarget.hasVendorXAndesPerf() &&
345 !(Subtarget.hasVendorXCValu() && !Subtarget.is64Bit()))
346 setOperationAction(Ops: ISD::SIGN_EXTEND_INREG, VTs: {MVT::i8, MVT::i16}, Action: Expand);
347
348 if (Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit()) {
349 setOperationAction(Op: ISD::LOAD, VT: MVT::i64, Action: Custom);
350 setOperationAction(Op: ISD::STORE, VT: MVT::i64, Action: Custom);
351 }
352
353 if (Subtarget.is64Bit()) {
354 setOperationAction(Op: ISD::EH_DWARF_CFA, VT: MVT::i64, Action: Custom);
355
356 setOperationAction(Op: ISD::LOAD, VT: MVT::i32, Action: Custom);
357 setOperationAction(Ops: {ISD::ADD, ISD::SUB, ISD::SHL, ISD::SRA, ISD::SRL},
358 VT: MVT::i32, Action: Custom);
359 setOperationAction(Ops: {ISD::UADDO, ISD::USUBO}, VT: MVT::i32, Action: Custom);
360 setOperationAction(Ops: {ISD::SADDO, ISD::SSUBO}, VT: MVT::i32, Action: Custom);
361 } else if (Subtarget.hasStdExtP()) {
362 // Custom legalize i64 ADD/SUB/SHL/SRL/SRA for RV32+P.
363 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VT: MVT::i64, Action: Custom);
364 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VT: MVT::i64, Action: Custom);
365 }
366 if (!Subtarget.hasStdExtZmmul()) {
367 setOperationAction(Ops: {ISD::MUL, ISD::MULHS, ISD::MULHU}, VT: XLenVT, Action: Expand);
368 } else if (Subtarget.is64Bit()) {
369 setOperationAction(Op: ISD::MUL, VT: MVT::i128, Action: Custom);
370 setOperationAction(Op: ISD::MUL, VT: MVT::i32, Action: Custom);
371 } else {
372 setOperationAction(Op: ISD::MUL, VT: MVT::i64, Action: Custom);
373 }
374
375 if (!Subtarget.hasStdExtM()) {
376 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::SREM, ISD::UREM}, VT: XLenVT,
377 Action: Expand);
378 } else if (Subtarget.is64Bit()) {
379 setOperationAction(Ops: {ISD::SDIV, ISD::UDIV, ISD::UREM},
380 VTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Custom);
381 }
382
383 setOperationAction(Ops: {ISD::SDIVREM, ISD::UDIVREM}, VT: XLenVT, Action: Expand);
384
385 // On RV32, the P extension has a WMUL(U) instruction we can use for
386 // (S/U)MUL_LOHI.
387 // FIXME: Does P imply Zmmul?
388 if (!Subtarget.hasStdExtP() || !Subtarget.hasStdExtZmmul() ||
389 Subtarget.is64Bit())
390 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT: XLenVT, Action: Expand);
391
392 setOperationAction(Ops: {ISD::SHL_PARTS, ISD::SRL_PARTS, ISD::SRA_PARTS}, VT: XLenVT,
393 Action: Custom);
394
395 if (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) {
396 if (Subtarget.is64Bit())
397 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: MVT::i32, Action: Custom);
398 } else if (Subtarget.hasVendorXTHeadBb()) {
399 if (Subtarget.is64Bit())
400 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: MVT::i32, Action: Custom);
401 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: XLenVT, Action: Custom);
402 } else if (Subtarget.hasVendorXCVbitmanip() && !Subtarget.is64Bit()) {
403 setOperationAction(Op: ISD::ROTL, VT: XLenVT, Action: Expand);
404 } else {
405 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT: XLenVT, Action: Expand);
406 }
407
408 if (Subtarget.hasStdExtP())
409 setOperationAction(Ops: {ISD::FSHL, ISD::FSHR}, VT: XLenVT, Action: Legal);
410
411 setOperationAction(Op: ISD::BSWAP, VT: XLenVT,
412 Action: Subtarget.hasREV8Like() ? Legal : Expand);
413
414 if (Subtarget.hasREVLike()) {
415 setOperationAction(Op: ISD::BITREVERSE, VT: XLenVT, Action: Legal);
416 } else {
417 // Zbkb can use rev8+brev8 to implement bitreverse.
418 setOperationAction(Op: ISD::BITREVERSE, VT: XLenVT,
419 Action: Subtarget.hasStdExtZbkb() ? Custom : Expand);
420 if (Subtarget.hasStdExtZbkb())
421 setOperationAction(Op: ISD::BITREVERSE, VT: MVT::i8, Action: Custom);
422 }
423
424 if (Subtarget.hasStdExtZbb() ||
425 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
426 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT: XLenVT,
427 Action: Legal);
428 }
429
430 if (Subtarget.hasCTZLike()) {
431 if (Subtarget.is64Bit())
432 setOperationAction(Ops: {ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF}, VT: MVT::i32, Action: Custom);
433 } else {
434 setOperationAction(Op: ISD::CTTZ, VT: XLenVT, Action: Expand);
435 }
436
437 if (!Subtarget.hasCPOPLike()) {
438 // TODO: These should be set to LibCall, but this currently breaks
439 // the Linux kernel build. See #101786. Lacks i128 tests, too.
440 if (Subtarget.is64Bit())
441 setOperationAction(Op: ISD::CTPOP, VT: MVT::i128, Action: Expand);
442 else
443 setOperationAction(Op: ISD::CTPOP, VT: MVT::i32, Action: Expand);
444 setOperationAction(Op: ISD::CTPOP, VT: MVT::i64, Action: Expand);
445 }
446
447 if (Subtarget.hasCLZLike()) {
448 // We need the custom lowering to make sure that the resulting sequence
449 // for the 32bit case is efficient on 64bit targets.
450 // Use default promotion for i32 without Zbb.
451 if (Subtarget.is64Bit() &&
452 (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtP()))
453 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF}, VT: MVT::i32, Action: Custom);
454 } else {
455 setOperationAction(Op: ISD::CTLZ, VT: XLenVT, Action: Expand);
456 }
457
458 if (Subtarget.hasStdExtP()) {
459 setOperationAction(Op: ISD::CTLS, VT: XLenVT, Action: Legal);
460 if (Subtarget.is64Bit())
461 setOperationAction(Op: ISD::CTLS, VT: MVT::i32, Action: Custom);
462 }
463
464 if (Subtarget.hasStdExtP() ||
465 (Subtarget.hasVendorXCValu() && !Subtarget.is64Bit())) {
466 setOperationAction(Op: ISD::ABS, VT: XLenVT, Action: Legal);
467 if (Subtarget.is64Bit())
468 setOperationAction(Op: ISD::ABS, VT: MVT::i32, Action: Custom);
469 } else if (Subtarget.hasShortForwardBranchIALU()) {
470 // We can use PseudoCCSUB to implement ABS.
471 setOperationAction(Op: ISD::ABS, VT: XLenVT, Action: Legal);
472 } else if (Subtarget.is64Bit()) {
473 setOperationAction(Op: ISD::ABS, VT: MVT::i32, Action: Custom);
474 }
475
476 if (!Subtarget.useMIPSCCMovInsn() && !Subtarget.hasVendorXTHeadCondMov())
477 setOperationAction(Op: ISD::SELECT, VT: XLenVT, Action: Custom);
478
479 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
480 !Subtarget.is64Bit()) {
481 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
482 VT: MVT::i32, Action: Legal);
483 } else if (Subtarget.hasStdExtP() && Subtarget.is64Bit()) {
484 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
485 VT: MVT::i32, Action: Custom);
486 } else if (!Subtarget.hasStdExtZbb() && Subtarget.is64Bit()) {
487 setOperationAction(Ops: {ISD::SADDSAT, ISD::SSUBSAT, ISD::UADDSAT, ISD::USUBSAT},
488 VT: MVT::i32, Action: Custom);
489 }
490
491 if (Subtarget.hasVendorXqcia() && !Subtarget.is64Bit()) {
492 setOperationAction(Op: ISD::USHLSAT, VT: MVT::i32, Action: Legal);
493 }
494
495 if ((Subtarget.hasStdExtP() || Subtarget.hasVendorXqcia()) &&
496 !Subtarget.is64Bit()) {
497 // FIXME: Support i32 on RV64+P by inserting into a v2i32 vector, doing
498 // pssha.w and extracting.
499 setOperationAction(Op: ISD::SSHLSAT, VT: MVT::i32, Action: Legal);
500 }
501
502 if (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc())
503 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT: XLenVT, Action: Legal);
504 if (Subtarget.hasStdExtZbc())
505 setOperationAction(Op: ISD::CLMULR, VT: XLenVT, Action: Legal);
506
507 static const unsigned FPLegalNodeTypes[] = {
508 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMINIMUMNUM,
509 ISD::FMAXIMUMNUM, ISD::LRINT, ISD::LLRINT,
510 ISD::LROUND, ISD::LLROUND, ISD::STRICT_LRINT,
511 ISD::STRICT_LLRINT, ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
512 ISD::STRICT_FMA, ISD::STRICT_FADD, ISD::STRICT_FSUB,
513 ISD::STRICT_FMUL, ISD::STRICT_FDIV, ISD::STRICT_FSQRT,
514 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::FCANONICALIZE};
515
516 static const ISD::CondCode FPCCToExpand[] = {
517 ISD::SETOGT, ISD::SETOGE, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
518 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUNE, ISD::SETGT,
519 ISD::SETGE, ISD::SETNE, ISD::SETO, ISD::SETUO};
520
521 static const unsigned FPOpToExpand[] = {ISD::FSIN, ISD::FCOS, ISD::FSINCOS,
522 ISD::FPOW};
523 static const unsigned FPOpToLibCall[] = {ISD::FREM};
524
525 static const unsigned FPRndMode[] = {
526 ISD::FCEIL, ISD::FFLOOR, ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
527 ISD::FROUNDEVEN};
528
529 static const unsigned ZfhminZfbfminPromoteOps[] = {
530 ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM,
531 ISD::FMINIMUMNUM, ISD::FADD, ISD::FSUB,
532 ISD::FMUL, ISD::FMA, ISD::FDIV,
533 ISD::FSQRT, ISD::STRICT_FMA, ISD::STRICT_FADD,
534 ISD::STRICT_FSUB, ISD::STRICT_FMUL, ISD::STRICT_FDIV,
535 ISD::STRICT_FSQRT, ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
536 ISD::SETCC, ISD::FCEIL, ISD::FFLOOR,
537 ISD::FTRUNC, ISD::FRINT, ISD::FROUND,
538 ISD::FROUNDEVEN, ISD::FCANONICALIZE};
539
540 if (Subtarget.hasStdExtP()) {
541 setTargetDAGCombine(ISD::TRUNCATE);
542 static const MVT RV32VTs[] = {MVT::v2i16, MVT::v4i8};
543 static const MVT RV64VTs[] = {MVT::v2i32, MVT::v4i16, MVT::v8i8};
544 ArrayRef<MVT> VTs;
545 if (Subtarget.is64Bit()) {
546 VTs = RV64VTs;
547 // There's no instruction for vector shamt in P extension so we unroll to
548 // scalar instructions. Vector VTs that are 32-bit are widened to 64-bit
549 // vector, e.g. v2i16 -> v4i16, before getting unrolled, so we need custom
550 // widen for those operations that will be unrolled.
551 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA},
552 VTs: {MVT::v2i16, MVT::v4i8}, Action: Custom);
553 } else {
554 VTs = RV32VTs;
555 }
556 // By default everything must be expanded.
557 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
558 setOperationAction(Ops: Op, VTs, Action: Expand);
559
560 for (MVT VT : VTs) {
561 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
562 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
563 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
564 MemVT: OtherVT, Action: Expand);
565 }
566 }
567
568 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VTs, Action: Legal);
569 setOperationAction(Ops: {ISD::ADD, ISD::SUB}, VTs, Action: Legal);
570 setOperationAction(Ops: {ISD::AND, ISD::OR, ISD::XOR}, VTs, Action: Legal);
571 setOperationAction(Ops: {ISD::MUL, ISD::MULHS, ISD::MULHU}, VTs, Action: Legal);
572 setOperationAction(Ops: ISD::UADDSAT, VTs, Action: Legal);
573 setOperationAction(Ops: ISD::SADDSAT, VTs, Action: Legal);
574 setOperationAction(Ops: ISD::USUBSAT, VTs, Action: Legal);
575 setOperationAction(Ops: ISD::SSUBSAT, VTs, Action: Legal);
576 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU}, VTs, Action: Legal);
577 for (MVT VT : VTs) {
578 if (VT != MVT::v2i32)
579 setOperationAction(Ops: {ISD::ABS, ISD::ABDS, ISD::ABDU}, VT, Action: Legal);
580 if (VT.getVectorElementType() != MVT::i8)
581 setOperationAction(Op: ISD::SSHLSAT, VT, Action: Custom);
582 }
583 setOperationAction(Ops: ISD::SPLAT_VECTOR, VTs, Action: Legal);
584 setOperationAction(Ops: ISD::BUILD_VECTOR, VTs, Action: Legal);
585 setOperationAction(Ops: ISD::SCALAR_TO_VECTOR, VTs, Action: Legal);
586 setOperationAction(Ops: {ISD::SHL, ISD::SRL, ISD::SRA}, VTs, Action: Custom);
587 setOperationAction(Ops: ISD::BITCAST, VTs, Action: Custom);
588 setOperationAction(Ops: {ISD::EXTRACT_VECTOR_ELT, ISD::INSERT_VECTOR_ELT}, VTs,
589 Action: Custom);
590 setOperationAction(Ops: {ISD::SMIN, ISD::UMIN, ISD::SMAX, ISD::UMAX}, VTs,
591 Action: Legal);
592 setOperationAction(Ops: ISD::SELECT, VTs, Action: Custom);
593 setOperationAction(Ops: ISD::VSELECT, VTs, Action: Legal);
594 setOperationAction(Ops: ISD::SETCC, VTs, Action: Legal);
595 setCondCodeAction(CCs: {ISD::SETNE, ISD::SETGT, ISD::SETGE, ISD::SETUGT,
596 ISD::SETUGE, ISD::SETULE, ISD::SETLE},
597 VTs, Action: Expand);
598
599 if (!Subtarget.is64Bit())
600 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::v4i8, Action: Custom);
601
602 // P extension vector comparisons produce all 1s for true, all 0s for false
603 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
604 }
605
606 if (Subtarget.hasStdExtZfbfmin()) {
607 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
608 setOperationAction(Op: ISD::ConstantFP, VT: MVT::bf16, Action: Expand);
609 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::bf16, Action: Expand);
610 setOperationAction(Op: ISD::SELECT, VT: MVT::bf16, Action: Custom);
611 setOperationAction(Op: ISD::BR_CC, VT: MVT::bf16, Action: Expand);
612 setOperationAction(Ops: ZfhminZfbfminPromoteOps, VT: MVT::bf16, Action: Promote);
613 setOperationAction(Op: ISD::FREM, VT: MVT::bf16, Action: Promote);
614 setOperationAction(Op: ISD::FABS, VT: MVT::bf16, Action: Custom);
615 setOperationAction(Op: ISD::FNEG, VT: MVT::bf16, Action: Custom);
616 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::bf16, Action: Custom);
617 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: XLenVT, Action: Custom);
618 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: XLenVT, Action: Custom);
619 }
620
621 if (Subtarget.hasStdExtZfhminOrZhinxmin()) {
622 if (Subtarget.hasStdExtZfhOrZhinx()) {
623 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f16, Action: Legal);
624 setOperationAction(Ops: FPRndMode, VT: MVT::f16,
625 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
626 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f16, Action: Custom);
627 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16,
628 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
629 if (Subtarget.hasStdExtZfa())
630 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f16, Action: Custom);
631 } else {
632 setOperationAction(Ops: ZfhminZfbfminPromoteOps, VT: MVT::f16, Action: Promote);
633 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f16, Action: Promote);
634 for (auto Op : {ISD::LROUND, ISD::LLROUND, ISD::LRINT, ISD::LLRINT,
635 ISD::STRICT_LROUND, ISD::STRICT_LLROUND,
636 ISD::STRICT_LRINT, ISD::STRICT_LLRINT})
637 setOperationAction(Op, VT: MVT::f16, Action: Custom);
638 setOperationAction(Op: ISD::FABS, VT: MVT::f16, Action: Custom);
639 setOperationAction(Op: ISD::FNEG, VT: MVT::f16, Action: Custom);
640 setOperationAction(Op: ISD::FCOPYSIGN, VT: MVT::f16, Action: Custom);
641 setOperationAction(Ops: {ISD::FP_TO_SINT, ISD::FP_TO_UINT}, VT: XLenVT, Action: Custom);
642 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP}, VT: XLenVT, Action: Custom);
643 }
644
645 if (!Subtarget.hasStdExtD()) {
646 // FIXME: handle f16 fma when f64 is not legal. Using an f32 fma
647 // instruction runs into double rounding issues, so this is wrong.
648 // Normally we'd use an f64 fma, but without the D extension the f64 type
649 // is not legal. This should probably be a libcall.
650 AddPromotedToType(Opc: ISD::FMA, OrigVT: MVT::f16, DestVT: MVT::f32);
651 AddPromotedToType(Opc: ISD::STRICT_FMA, OrigVT: MVT::f16, DestVT: MVT::f32);
652 }
653
654 setOperationAction(Op: ISD::BITCAST, VT: MVT::i16, Action: Custom);
655
656 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f16, Action: Legal);
657 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f32, Action: Legal);
658 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f16, Action: Expand);
659 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f16, Action: Expand);
660 setOperationAction(Op: ISD::SELECT, VT: MVT::f16, Action: Custom);
661 setOperationAction(Op: ISD::BR_CC, VT: MVT::f16, Action: Expand);
662
663 setOperationAction(
664 Op: ISD::FNEARBYINT, VT: MVT::f16,
665 Action: Subtarget.hasStdExtZfh() && Subtarget.hasStdExtZfa() ? Legal : Promote);
666 setOperationAction(Ops: {ISD::FREM, ISD::FPOW, ISD::FPOWI,
667 ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
668 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2,
669 ISD::FLOG10, ISD::FLDEXP, ISD::FFREXP, ISD::FMODF},
670 VT: MVT::f16, Action: Promote);
671
672 // FIXME: Need to promote f16 STRICT_* to f32 libcalls, but we don't have
673 // complete support for all operations in LegalizeDAG.
674 setOperationAction(Ops: {ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR,
675 ISD::STRICT_FNEARBYINT, ISD::STRICT_FRINT,
676 ISD::STRICT_FROUND, ISD::STRICT_FROUNDEVEN,
677 ISD::STRICT_FTRUNC, ISD::STRICT_FLDEXP},
678 VT: MVT::f16, Action: Promote);
679
680 // We need to custom promote this.
681 if (Subtarget.is64Bit())
682 setOperationAction(Op: ISD::FPOWI, VT: MVT::i32, Action: Custom);
683 }
684
685 if (Subtarget.hasStdExtFOrZfinx()) {
686 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f32, Action: Legal);
687 setOperationAction(Ops: FPRndMode, VT: MVT::f32,
688 Action: Subtarget.hasStdExtZfa() ? Legal : Custom);
689 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f32, Action: Expand);
690 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f32, Action: Expand);
691 setOperationAction(Op: ISD::SELECT, VT: MVT::f32, Action: Custom);
692 setOperationAction(Op: ISD::BR_CC, VT: MVT::f32, Action: Expand);
693 setOperationAction(Ops: FPOpToExpand, VT: MVT::f32, Action: Expand);
694 setOperationAction(Ops: FPOpToLibCall, VT: MVT::f32, Action: LibCall);
695 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
696 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::f16, Action: Expand);
697 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
698 setTruncStoreAction(ValVT: MVT::f32, MemVT: MVT::bf16, Action: Expand);
699 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f32, Action: Custom);
700 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f32, Action: Custom);
701 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f32,
702 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
703 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f32, Action: Custom);
704 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f32, Action: Custom);
705 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f32, Action: Custom);
706 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f32, Action: Custom);
707
708 if (Subtarget.hasStdExtZfa()) {
709 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f32, Action: Custom);
710 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f32, Action: Legal);
711 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Legal);
712 } else {
713 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f32, Action: Custom);
714 }
715 }
716
717 if (Subtarget.hasStdExtFOrZfinx() && Subtarget.is64Bit())
718 setOperationAction(Op: ISD::BITCAST, VT: MVT::i32, Action: Custom);
719
720 if (Subtarget.hasStdExtDOrZdinx()) {
721 setOperationAction(Ops: FPLegalNodeTypes, VT: MVT::f64, Action: Legal);
722
723 if (!Subtarget.is64Bit())
724 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
725
726 if (Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
727 !Subtarget.is64Bit()) {
728 setOperationAction(Op: ISD::LOAD, VT: MVT::f64, Action: Custom);
729 setOperationAction(Op: ISD::STORE, VT: MVT::f64, Action: Custom);
730 }
731
732 if (Subtarget.hasStdExtZfa()) {
733 setOperationAction(Op: ISD::ConstantFP, VT: MVT::f64, Action: Custom);
734 setOperationAction(Ops: FPRndMode, VT: MVT::f64, Action: Legal);
735 setOperationAction(Op: ISD::FNEARBYINT, VT: MVT::f64, Action: Legal);
736 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f64, Action: Legal);
737 } else {
738 if (Subtarget.is64Bit())
739 setOperationAction(Ops: FPRndMode, VT: MVT::f64, Action: Custom);
740
741 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT: MVT::f64, Action: Custom);
742 }
743
744 setOperationAction(Op: ISD::STRICT_FP_ROUND, VT: MVT::f32, Action: Legal);
745 setOperationAction(Op: ISD::STRICT_FP_EXTEND, VT: MVT::f64, Action: Legal);
746 setCondCodeAction(CCs: FPCCToExpand, VT: MVT::f64, Action: Expand);
747 setOperationAction(Op: ISD::SELECT_CC, VT: MVT::f64, Action: Expand);
748 setOperationAction(Op: ISD::SELECT, VT: MVT::f64, Action: Custom);
749 setOperationAction(Op: ISD::BR_CC, VT: MVT::f64, Action: Expand);
750 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
751 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f32, Action: Expand);
752 setOperationAction(Ops: FPOpToExpand, VT: MVT::f64, Action: Expand);
753 setOperationAction(Ops: FPOpToLibCall, VT: MVT::f64, Action: LibCall);
754 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
755 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::f16, Action: Expand);
756 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
757 setTruncStoreAction(ValVT: MVT::f64, MemVT: MVT::bf16, Action: Expand);
758 setOperationAction(Op: ISD::IS_FPCLASS, VT: MVT::f64, Action: Custom);
759 setOperationAction(Op: ISD::BF16_TO_FP, VT: MVT::f64, Action: Custom);
760 setOperationAction(Op: ISD::FP_TO_BF16, VT: MVT::f64,
761 Action: Subtarget.isSoftFPABI() ? LibCall : Custom);
762 setOperationAction(Op: ISD::FP_TO_FP16, VT: MVT::f64, Action: Custom);
763 setOperationAction(Op: ISD::FP16_TO_FP, VT: MVT::f64, Action: Expand);
764 setOperationAction(Op: ISD::STRICT_FP_TO_FP16, VT: MVT::f64, Action: Custom);
765 setOperationAction(Op: ISD::STRICT_FP16_TO_FP, VT: MVT::f64, Action: Expand);
766 }
767
768 if (Subtarget.is64Bit()) {
769 setOperationAction(Ops: {ISD::FP_TO_UINT, ISD::FP_TO_SINT,
770 ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT},
771 VT: MVT::i32, Action: Custom);
772 setOperationAction(Op: ISD::LROUND, VT: MVT::i32, Action: Custom);
773 }
774
775 if (Subtarget.hasStdExtFOrZfinx()) {
776 setOperationAction(Ops: {ISD::FP_TO_UINT_SAT, ISD::FP_TO_SINT_SAT}, VT: XLenVT,
777 Action: Custom);
778
779 // f16/bf16 require custom handling.
780 setOperationAction(Ops: {ISD::STRICT_FP_TO_UINT, ISD::STRICT_FP_TO_SINT}, VT: XLenVT,
781 Action: Custom);
782 setOperationAction(Ops: {ISD::STRICT_UINT_TO_FP, ISD::STRICT_SINT_TO_FP}, VT: XLenVT,
783 Action: Custom);
784
785 setOperationAction(Op: ISD::GET_ROUNDING, VT: XLenVT, Action: Custom);
786 setOperationAction(Op: ISD::SET_ROUNDING, VT: MVT::Other, Action: Custom);
787 setOperationAction(Op: ISD::GET_FPENV, VT: XLenVT, Action: Custom);
788 setOperationAction(Op: ISD::SET_FPENV, VT: XLenVT, Action: Custom);
789 setOperationAction(Op: ISD::RESET_FPENV, VT: MVT::Other, Action: Custom);
790 setOperationAction(Op: ISD::GET_FPMODE, VT: XLenVT, Action: Custom);
791 setOperationAction(Op: ISD::SET_FPMODE, VT: XLenVT, Action: Custom);
792 setOperationAction(Op: ISD::RESET_FPMODE, VT: MVT::Other, Action: Custom);
793 }
794
795 setOperationAction(Ops: {ISD::GlobalAddress, ISD::BlockAddress, ISD::ConstantPool,
796 ISD::JumpTable},
797 VT: XLenVT, Action: Custom);
798
799 setOperationAction(Op: ISD::GlobalTLSAddress, VT: XLenVT, Action: Custom);
800
801 if (Subtarget.is64Bit())
802 setOperationAction(Op: ISD::Constant, VT: MVT::i64, Action: Custom);
803
804 // TODO: On M-mode only targets, the cycle[h]/time[h] CSR may not be present.
805 // Unfortunately this can't be determined just from the ISA naming string.
806 setOperationAction(Op: ISD::READCYCLECOUNTER, VT: MVT::i64,
807 Action: Subtarget.is64Bit() ? Legal : Custom);
808 setOperationAction(Op: ISD::READSTEADYCOUNTER, VT: MVT::i64,
809 Action: Subtarget.is64Bit() ? Legal : Custom);
810
811 if (Subtarget.is64Bit()) {
812 setOperationAction(Op: ISD::INIT_TRAMPOLINE, VT: MVT::Other, Action: Custom);
813 setOperationAction(Op: ISD::ADJUST_TRAMPOLINE, VT: MVT::Other, Action: Custom);
814 }
815
816 setOperationAction(Ops: {ISD::TRAP, ISD::DEBUGTRAP}, VT: MVT::Other, Action: Legal);
817 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::Other, Action: Custom);
818 if (Subtarget.is64Bit())
819 setOperationAction(Op: ISD::INTRINSIC_WO_CHAIN, VT: MVT::i32, Action: Custom);
820
821 if (Subtarget.hasVendorXMIPSCBOP())
822 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Custom);
823 else if (Subtarget.hasStdExtZicbop())
824 setOperationAction(Op: ISD::PREFETCH, VT: MVT::Other, Action: Legal);
825
826 if (Subtarget.hasStdExtZalrsc()) {
827 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
828 if (Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas())
829 setMinCmpXchgSizeInBits(8);
830 else
831 setMinCmpXchgSizeInBits(32);
832 } else if (Subtarget.hasForcedAtomics()) {
833 setMaxAtomicSizeInBitsSupported(Subtarget.getXLen());
834 } else {
835 setMaxAtomicSizeInBitsSupported(0);
836 }
837
838 setOperationAction(Op: ISD::ATOMIC_FENCE, VT: MVT::Other, Action: Custom);
839
840 setBooleanContents(ZeroOrOneBooleanContent);
841
842 if (getTargetMachine().getTargetTriple().isOSLinux()) {
843 // Custom lowering of llvm.clear_cache.
844 setOperationAction(Op: ISD::CLEAR_CACHE, VT: MVT::Other, Action: Custom);
845 }
846
847 if (Subtarget.hasVInstructions()) {
848 setBooleanVectorContents(ZeroOrNegativeOneBooleanContent);
849
850 setOperationAction(Op: ISD::VSCALE, VT: XLenVT, Action: Custom);
851
852 // RVV intrinsics may have illegal operands.
853 // We also need to custom legalize vmv.x.s.
854 setOperationAction(Ops: {ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN,
855 ISD::INTRINSIC_VOID},
856 VTs: {MVT::i8, MVT::i16}, Action: Custom);
857 if (Subtarget.is64Bit())
858 setOperationAction(Ops: {ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
859 VT: MVT::i32, Action: Custom);
860 else
861 setOperationAction(Ops: {ISD::INTRINSIC_WO_CHAIN, ISD::INTRINSIC_W_CHAIN},
862 VT: MVT::i64, Action: Custom);
863
864 setOperationAction(Ops: {ISD::INTRINSIC_W_CHAIN, ISD::INTRINSIC_VOID},
865 VT: MVT::Other, Action: Custom);
866
867 static const unsigned IntegerVPOps[] = {
868 ISD::VP_ADD, ISD::VP_SUB, ISD::VP_MUL,
869 ISD::VP_SDIV, ISD::VP_UDIV, ISD::VP_SREM,
870 ISD::VP_UREM, ISD::VP_AND, ISD::VP_OR,
871 ISD::VP_XOR, ISD::VP_SRA, ISD::VP_SRL,
872 ISD::VP_SHL, ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
873 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR, ISD::VP_REDUCE_SMAX,
874 ISD::VP_REDUCE_SMIN, ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN,
875 ISD::VP_MERGE, ISD::VP_SELECT, ISD::VP_FP_TO_SINT,
876 ISD::VP_FP_TO_UINT, ISD::VP_SETCC, ISD::VP_SIGN_EXTEND,
877 ISD::VP_ZERO_EXTEND, ISD::VP_TRUNCATE, ISD::VP_SMIN,
878 ISD::VP_SMAX, ISD::VP_UMIN, ISD::VP_UMAX,
879 ISD::VP_ABS, ISD::EXPERIMENTAL_VP_REVERSE, ISD::EXPERIMENTAL_VP_SPLICE,
880 ISD::VP_SADDSAT, ISD::VP_UADDSAT, ISD::VP_SSUBSAT,
881 ISD::VP_USUBSAT, ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF};
882
883 static const unsigned FloatingPointVPOps[] = {
884 ISD::VP_FADD, ISD::VP_FSUB, ISD::VP_FMUL,
885 ISD::VP_FDIV, ISD::VP_FNEG, ISD::VP_FABS,
886 ISD::VP_FMA, ISD::VP_REDUCE_FADD, ISD::VP_REDUCE_SEQ_FADD,
887 ISD::VP_REDUCE_FMIN, ISD::VP_REDUCE_FMAX, ISD::VP_MERGE,
888 ISD::VP_SELECT, ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP,
889 ISD::VP_SETCC, ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND,
890 ISD::VP_SQRT, ISD::VP_FMINNUM, ISD::VP_FMAXNUM,
891 ISD::VP_FCEIL, ISD::VP_FFLOOR, ISD::VP_FROUND,
892 ISD::VP_FROUNDEVEN, ISD::VP_FCOPYSIGN, ISD::VP_FROUNDTOZERO,
893 ISD::VP_FRINT, ISD::VP_FNEARBYINT, ISD::VP_IS_FPCLASS,
894 ISD::VP_FMINIMUM, ISD::VP_FMAXIMUM, ISD::VP_LRINT,
895 ISD::VP_LLRINT, ISD::VP_REDUCE_FMINIMUM,
896 ISD::VP_REDUCE_FMAXIMUM};
897
898 static const unsigned IntegerVecReduceOps[] = {
899 ISD::VECREDUCE_ADD, ISD::VECREDUCE_AND, ISD::VECREDUCE_OR,
900 ISD::VECREDUCE_XOR, ISD::VECREDUCE_SMAX, ISD::VECREDUCE_SMIN,
901 ISD::VECREDUCE_UMAX, ISD::VECREDUCE_UMIN};
902
903 static const unsigned FloatingPointVecReduceOps[] = {
904 ISD::VECREDUCE_FADD, ISD::VECREDUCE_SEQ_FADD, ISD::VECREDUCE_FMIN,
905 ISD::VECREDUCE_FMAX, ISD::VECREDUCE_FMINIMUM, ISD::VECREDUCE_FMAXIMUM};
906
907 static const unsigned FloatingPointLibCallOps[] = {
908 ISD::FREM, ISD::FPOW, ISD::FCOS, ISD::FSIN, ISD::FSINCOS, ISD::FEXP,
909 ISD::FEXP2, ISD::FEXP10, ISD::FLOG, ISD::FLOG2, ISD::FLOG10};
910
911 if (!Subtarget.is64Bit()) {
912 // We must custom-lower certain vXi64 operations on RV32 due to the vector
913 // element type being illegal.
914 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
915 VT: MVT::i64, Action: Custom);
916
917 setOperationAction(Ops: IntegerVecReduceOps, VT: MVT::i64, Action: Custom);
918
919 setOperationAction(Ops: {ISD::VP_REDUCE_ADD, ISD::VP_REDUCE_AND,
920 ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR,
921 ISD::VP_REDUCE_SMAX, ISD::VP_REDUCE_SMIN,
922 ISD::VP_REDUCE_UMAX, ISD::VP_REDUCE_UMIN},
923 VT: MVT::i64, Action: Custom);
924 }
925
926 for (MVT VT : BoolVecVTs) {
927 if (!isTypeLegal(VT))
928 continue;
929
930 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Custom);
931
932 // Mask VTs are custom-expanded into a series of standard nodes
933 setOperationAction(Ops: {ISD::TRUNCATE, ISD::CONCAT_VECTORS,
934 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
935 ISD::SCALAR_TO_VECTOR},
936 VT, Action: Custom);
937
938 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
939 Action: Custom);
940
941 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
942 setOperationAction(Ops: {ISD::SELECT_CC, ISD::VSELECT, ISD::VP_SELECT}, VT,
943 Action: Expand);
944 setOperationAction(Op: ISD::VP_MERGE, VT, Action: Custom);
945
946 setOperationAction(Ops: {ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON,
947 ISD::VP_CTTZ_ELTS, ISD::VP_CTTZ_ELTS_ZERO_UNDEF},
948 VT, Action: Custom);
949
950 setOperationAction(Ops: {ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR}, VT, Action: Custom);
951
952 setOperationAction(
953 Ops: {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
954 Action: Custom);
955
956 setOperationAction(
957 Ops: {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
958 Action: Custom);
959
960 // RVV has native int->float & float->int conversions where the
961 // element type sizes are within one power-of-two of each other. Any
962 // wider distances between type sizes have to be lowered as sequences
963 // which progressively narrow the gap in stages.
964 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
965 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
966 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
967 ISD::STRICT_FP_TO_UINT},
968 VT, Action: Custom);
969 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
970 Action: Custom);
971
972 // Expand all extending loads to types larger than this, and truncating
973 // stores from types larger than this.
974 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
975 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
976 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
977 MemVT: OtherVT, Action: Expand);
978 }
979
980 setOperationAction(Ops: {ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
981 ISD::VP_TRUNCATE, ISD::VP_SETCC},
982 VT, Action: Custom);
983
984 setOperationAction(Op: ISD::VECTOR_DEINTERLEAVE, VT, Action: Custom);
985 setOperationAction(Op: ISD::VECTOR_INTERLEAVE, VT, Action: Custom);
986
987 setOperationAction(Op: ISD::VECTOR_REVERSE, VT, Action: Custom);
988
989 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
990 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
991
992 setOperationPromotedToType(
993 Ops: {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT}, OrigVT: VT,
994 DestVT: MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount()));
995 }
996
997 for (MVT VT : IntVecVTs) {
998 if (!isTypeLegal(VT))
999 continue;
1000
1001 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1002 setOperationAction(Op: ISD::SPLAT_VECTOR_PARTS, VT, Action: Custom);
1003
1004 // Vectors implement MULHS/MULHU.
1005 setOperationAction(Ops: {ISD::SMUL_LOHI, ISD::UMUL_LOHI}, VT, Action: Expand);
1006
1007 // nxvXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1008 if (VT.getVectorElementType() == MVT::i64 && !Subtarget.hasStdExtV())
1009 setOperationAction(Ops: {ISD::MULHU, ISD::MULHS}, VT, Action: Expand);
1010
1011 setOperationAction(Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX}, VT,
1012 Action: Legal);
1013
1014 if (Subtarget.hasStdExtZvabd()) {
1015 setOperationAction(Op: ISD::ABS, VT, Action: Legal);
1016 // Only SEW=8/16 are supported in Zvabd.
1017 if (VT.getVectorElementType() == MVT::i8 ||
1018 VT.getVectorElementType() == MVT::i16)
1019 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Legal);
1020 else
1021 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1022 } else
1023 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1024
1025 // Custom-lower extensions and truncations from/to mask types.
1026 setOperationAction(Ops: {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND},
1027 VT, Action: Custom);
1028
1029 // RVV has native int->float & float->int conversions where the
1030 // element type sizes are within one power-of-two of each other. Any
1031 // wider distances between type sizes have to be lowered as sequences
1032 // which progressively narrow the gap in stages.
1033 setOperationAction(Ops: {ISD::SINT_TO_FP, ISD::UINT_TO_FP, ISD::FP_TO_SINT,
1034 ISD::FP_TO_UINT, ISD::STRICT_SINT_TO_FP,
1035 ISD::STRICT_UINT_TO_FP, ISD::STRICT_FP_TO_SINT,
1036 ISD::STRICT_FP_TO_UINT},
1037 VT, Action: Custom);
1038 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1039 Action: Custom);
1040 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1041 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1042 ISD::SSUBSAT, ISD::USUBSAT},
1043 VT, Action: Legal);
1044
1045 // Integer VTs are lowered as a series of "RISCVISD::TRUNCATE_VECTOR_VL"
1046 // nodes which truncate by one power of two at a time.
1047 setOperationAction(
1048 Ops: {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1049 Action: Custom);
1050
1051 // Custom-lower insert/extract operations to simplify patterns.
1052 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1053 Action: Custom);
1054
1055 // Custom-lower reduction operations to set up the corresponding custom
1056 // nodes' operands.
1057 setOperationAction(Ops: IntegerVecReduceOps, VT, Action: Custom);
1058
1059 setOperationAction(Ops: IntegerVPOps, VT, Action: Custom);
1060
1061 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1062
1063 setOperationAction(Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1064 VT, Action: Custom);
1065
1066 setOperationAction(
1067 Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1068 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1069 VT, Action: Custom);
1070 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1071
1072 setOperationAction(Ops: {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1073 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1074 VT, Action: Custom);
1075
1076 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1077 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1078
1079 setOperationAction(Ops: {ISD::STEP_VECTOR, ISD::VECTOR_REVERSE}, VT, Action: Custom);
1080
1081 for (MVT OtherVT : MVT::integer_scalable_vector_valuetypes()) {
1082 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1083 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1084 MemVT: OtherVT, Action: Expand);
1085 }
1086
1087 setOperationAction(Op: ISD::VECTOR_DEINTERLEAVE, VT, Action: Custom);
1088 setOperationAction(Op: ISD::VECTOR_INTERLEAVE, VT, Action: Custom);
1089
1090 setOperationAction(Ops: {ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT},
1091 VT, Action: Custom);
1092
1093 if (Subtarget.hasStdExtZvkb()) {
1094 setOperationAction(Op: ISD::BSWAP, VT, Action: Legal);
1095 setOperationAction(Op: ISD::VP_BSWAP, VT, Action: Custom);
1096 } else {
1097 setOperationAction(Ops: {ISD::BSWAP, ISD::VP_BSWAP}, VT, Action: Expand);
1098 setOperationAction(Ops: {ISD::ROTL, ISD::ROTR}, VT, Action: Expand);
1099 }
1100
1101 if (Subtarget.hasStdExtZvbb()) {
1102 setOperationAction(Op: ISD::BITREVERSE, VT, Action: Legal);
1103 setOperationAction(Op: ISD::VP_BITREVERSE, VT, Action: Custom);
1104 setOperationAction(Ops: {ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,
1105 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
1106 VT, Action: Custom);
1107 } else {
1108 setOperationAction(Ops: {ISD::BITREVERSE, ISD::VP_BITREVERSE}, VT, Action: Expand);
1109 setOperationAction(Ops: {ISD::CTLZ, ISD::CTTZ, ISD::CTPOP}, VT, Action: Expand);
1110 setOperationAction(Ops: {ISD::VP_CTLZ, ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ,
1111 ISD::VP_CTTZ_ZERO_UNDEF, ISD::VP_CTPOP},
1112 VT, Action: Expand);
1113
1114 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
1115 // range of f32.
1116 EVT FloatVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1117 if (isTypeLegal(VT: FloatVT)) {
1118 setOperationAction(Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
1119 ISD::CTTZ_ZERO_UNDEF, ISD::VP_CTLZ,
1120 ISD::VP_CTLZ_ZERO_UNDEF, ISD::VP_CTTZ_ZERO_UNDEF},
1121 VT, Action: Custom);
1122 }
1123 }
1124
1125 if (VT.getVectorElementType() == MVT::i64) {
1126 if (Subtarget.hasStdExtZvbc())
1127 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Legal);
1128 } else {
1129 if (Subtarget.hasStdExtZvbc32e()) {
1130 setOperationAction(Ops: {ISD::CLMUL, ISD::CLMULH}, VT, Action: Legal);
1131 } else if (Subtarget.hasStdExtZvbc()) {
1132 // Promote to i64 if the lmul is small enough.
1133 // FIXME: Split if necessary to widen.
1134 // FIXME: Promote clmulh directly without legalizing to clmul first.
1135 MVT I64VecVT = MVT::getVectorVT(VT: MVT::i64, EC: VT.getVectorElementCount());
1136 if (isTypeLegal(VT: I64VecVT))
1137 setOperationAction(Op: ISD::CLMUL, VT, Action: Custom);
1138 }
1139 }
1140
1141 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1142 }
1143
1144 for (MVT VT : VecTupleVTs) {
1145 if (!isTypeLegal(VT))
1146 continue;
1147
1148 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1149 }
1150
1151 // Expand various CCs to best match the RVV ISA, which natively supports UNE
1152 // but no other unordered comparisons, and supports all ordered comparisons
1153 // except ONE. Additionally, we expand GT,OGT,GE,OGE for optimization
1154 // purposes; they are expanded to their swapped-operand CCs (LT,OLT,LE,OLE),
1155 // and we pattern-match those back to the "original", swapping operands once
1156 // more. This way we catch both operations and both "vf" and "fv" forms with
1157 // fewer patterns.
1158 static const ISD::CondCode VFPCCToExpand[] = {
1159 ISD::SETO, ISD::SETONE, ISD::SETUEQ, ISD::SETUGT,
1160 ISD::SETUGE, ISD::SETULT, ISD::SETULE, ISD::SETUO,
1161 ISD::SETGT, ISD::SETOGT, ISD::SETGE, ISD::SETOGE,
1162 };
1163
1164 // TODO: support more ops.
1165 static const unsigned ZvfhminZvfbfminPromoteOps[] = {
1166 ISD::FMINNUM,
1167 ISD::FMAXNUM,
1168 ISD::FMINIMUMNUM,
1169 ISD::FMAXIMUMNUM,
1170 ISD::FADD,
1171 ISD::FSUB,
1172 ISD::FMUL,
1173 ISD::FMA,
1174 ISD::FDIV,
1175 ISD::FSQRT,
1176 ISD::FCEIL,
1177 ISD::FTRUNC,
1178 ISD::FFLOOR,
1179 ISD::FROUND,
1180 ISD::FROUNDEVEN,
1181 ISD::FRINT,
1182 ISD::FNEARBYINT,
1183 ISD::IS_FPCLASS,
1184 ISD::SETCC,
1185 ISD::FMAXIMUM,
1186 ISD::FMINIMUM,
1187 ISD::STRICT_FADD,
1188 ISD::STRICT_FSUB,
1189 ISD::STRICT_FMUL,
1190 ISD::STRICT_FDIV,
1191 ISD::STRICT_FSQRT,
1192 ISD::STRICT_FMA,
1193 ISD::VECREDUCE_FMIN,
1194 ISD::VECREDUCE_FMAX,
1195 ISD::VECREDUCE_FMINIMUM,
1196 ISD::VECREDUCE_FMAXIMUM};
1197
1198 // TODO: Make more of these ops legal.
1199 static const unsigned ZvfbfaPromoteOps[] = {ISD::FDIV,
1200 ISD::FSQRT,
1201 ISD::FCEIL,
1202 ISD::FTRUNC,
1203 ISD::FFLOOR,
1204 ISD::FROUND,
1205 ISD::FROUNDEVEN,
1206 ISD::FRINT,
1207 ISD::FNEARBYINT,
1208 ISD::STRICT_FDIV,
1209 ISD::STRICT_FSQRT,
1210 ISD::VECREDUCE_FMIN,
1211 ISD::VECREDUCE_FMAX,
1212 ISD::VECREDUCE_FMINIMUM,
1213 ISD::VECREDUCE_FMAXIMUM};
1214
1215 // TODO: support more vp ops.
1216 static const unsigned ZvfhminZvfbfminPromoteVPOps[] = {
1217 ISD::VP_FADD,
1218 ISD::VP_FSUB,
1219 ISD::VP_FMUL,
1220 ISD::VP_FDIV,
1221 ISD::VP_FMA,
1222 ISD::VP_REDUCE_FMIN,
1223 ISD::VP_REDUCE_FMAX,
1224 ISD::VP_SQRT,
1225 ISD::VP_FMINNUM,
1226 ISD::VP_FMAXNUM,
1227 ISD::VP_FCEIL,
1228 ISD::VP_FFLOOR,
1229 ISD::VP_FROUND,
1230 ISD::VP_FROUNDEVEN,
1231 ISD::VP_FROUNDTOZERO,
1232 ISD::VP_FRINT,
1233 ISD::VP_FNEARBYINT,
1234 ISD::VP_SETCC,
1235 ISD::VP_FMINIMUM,
1236 ISD::VP_FMAXIMUM,
1237 ISD::VP_REDUCE_FMINIMUM,
1238 ISD::VP_REDUCE_FMAXIMUM};
1239
1240 // Sets common operation actions on RVV floating-point vector types.
1241 const auto SetCommonVFPActions = [&](MVT VT) {
1242 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1243 // RVV has native FP_ROUND & FP_EXTEND conversions where the element type
1244 // sizes are within one power-of-two of each other. Therefore conversions
1245 // between vXf16 and vXf64 must be lowered as sequences which convert via
1246 // vXf32.
1247 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1248 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1249 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1250 // Custom-lower insert/extract operations to simplify patterns.
1251 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT}, VT,
1252 Action: Custom);
1253 // Expand various condition codes (explained above).
1254 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1255
1256 setOperationAction(
1257 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1258 Action: Legal);
1259 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Action: Custom);
1260
1261 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1262 ISD::FROUNDEVEN, ISD::FRINT, ISD::FNEARBYINT,
1263 ISD::IS_FPCLASS},
1264 VT, Action: Custom);
1265
1266 setOperationAction(Ops: FloatingPointVecReduceOps, VT, Action: Custom);
1267
1268 // Expand FP operations that need libcalls.
1269 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1270
1271 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Legal);
1272
1273 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1274
1275 setOperationAction(Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER},
1276 VT, Action: Custom);
1277
1278 setOperationAction(
1279 Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1280 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER, ISD::VP_SCATTER},
1281 VT, Action: Custom);
1282 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1283
1284 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1285 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1286
1287 setOperationAction(Ops: {ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1288 ISD::EXTRACT_SUBVECTOR, ISD::SCALAR_TO_VECTOR},
1289 VT, Action: Custom);
1290
1291 setOperationAction(Op: ISD::VECTOR_DEINTERLEAVE, VT, Action: Custom);
1292 setOperationAction(Op: ISD::VECTOR_INTERLEAVE, VT, Action: Custom);
1293
1294 setOperationAction(Ops: {ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1295 ISD::VECTOR_SPLICE_RIGHT},
1296 VT, Action: Custom);
1297 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1298 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1299
1300 setOperationAction(Ops: FloatingPointVPOps, VT, Action: Custom);
1301
1302 setOperationAction(Ops: {ISD::STRICT_FP_EXTEND, ISD::STRICT_FP_ROUND}, VT,
1303 Action: Custom);
1304 setOperationAction(Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1305 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA},
1306 VT, Action: Legal);
1307 setOperationAction(Ops: {ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS,
1308 ISD::STRICT_FTRUNC, ISD::STRICT_FCEIL,
1309 ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1310 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1311 VT, Action: Custom);
1312
1313 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1314 };
1315
1316 // Sets common extload/truncstore actions on RVV floating-point vector
1317 // types.
1318 const auto SetCommonVFPExtLoadTruncStoreActions =
1319 [&](MVT VT, ArrayRef<MVT::SimpleValueType> SmallerVTs) {
1320 for (auto SmallVT : SmallerVTs) {
1321 setTruncStoreAction(ValVT: VT, MemVT: SmallVT, Action: Expand);
1322 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: SmallVT, Action: Expand);
1323 }
1324 };
1325
1326 // Sets common actions for f16 and bf16 for when there's only
1327 // zvfhmin/zvfbfmin and we need to promote to f32 for most operations.
1328 const auto SetCommonPromoteToF32Actions = [&](MVT VT) {
1329 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1330 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1331 Action: Custom);
1332 setOperationAction(Ops: {ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Action: Custom);
1333 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1334 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1335 setOperationAction(Ops: {ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
1336 Action: Custom);
1337 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1338 setOperationAction(Ops: {ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP}, VT, Action: Custom);
1339 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::CONCAT_VECTORS,
1340 ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR,
1341 ISD::VECTOR_DEINTERLEAVE, ISD::VECTOR_INTERLEAVE,
1342 ISD::VECTOR_REVERSE, ISD::VECTOR_SPLICE_LEFT,
1343 ISD::VECTOR_SPLICE_RIGHT, ISD::VECTOR_COMPRESS},
1344 VT, Action: Custom);
1345 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1346 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1347 MVT EltVT = VT.getVectorElementType();
1348 if (isTypeLegal(VT: EltVT))
1349 setOperationAction(Ops: {ISD::SPLAT_VECTOR, ISD::EXTRACT_VECTOR_ELT},
1350 VT, Action: Custom);
1351 else
1352 setOperationAction(Op: ISD::SPLAT_VECTOR, VT: EltVT, Action: Custom);
1353 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1354 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1355 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1356 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1357 ISD::VP_SCATTER},
1358 VT, Action: Custom);
1359 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1360
1361 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
1362 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
1363 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Expand);
1364
1365 // Expand FP operations that need libcalls.
1366 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1367
1368 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1369 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1370 setOperationAction(Ops: ZvfhminZvfbfminPromoteOps, VT, Action: Custom);
1371 setOperationAction(Ops: ZvfhminZvfbfminPromoteVPOps, VT, Action: Custom);
1372 } else {
1373 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1374 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1375 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1376 }
1377 };
1378
1379 // Sets common actions for zvfbfa, some of instructions are supported
1380 // natively so that we don't need to promote them.
1381 const auto SetZvfbfaActions = [&](MVT VT) {
1382 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1383 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1384 Action: Custom);
1385 setOperationAction(Ops: {ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Action: Custom);
1386 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1387 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1388 setOperationAction(Ops: {ISD::VP_MERGE, ISD::VP_SELECT, ISD::SELECT}, VT,
1389 Action: Custom);
1390 setOperationAction(Op: ISD::SELECT_CC, VT, Action: Expand);
1391 setOperationAction(Ops: {ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP}, VT, Action: Custom);
1392 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1393 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1394 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_DEINTERLEAVE,
1395 ISD::VECTOR_INTERLEAVE, ISD::VECTOR_REVERSE,
1396 ISD::VECTOR_SPLICE_LEFT, ISD::VECTOR_SPLICE_RIGHT,
1397 ISD::VECTOR_COMPRESS},
1398 VT, Action: Custom);
1399 setOperationAction(
1400 Ops: {ISD::FMINNUM, ISD::FMAXNUM, ISD::FMAXIMUMNUM, ISD::FMINIMUMNUM}, VT,
1401 Action: Legal);
1402 setOperationAction(Ops: {ISD::FMAXIMUM, ISD::FMINIMUM}, VT, Action: Custom);
1403 setOperationAction(Op: ISD::IS_FPCLASS, VT, Action: Custom);
1404 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1405 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1406
1407 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Legal);
1408 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1409 setOperationAction(Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1410 ISD::STRICT_FMA},
1411 VT, Action: Legal);
1412 setOperationAction(Ops: ZvfbfaVPOps, VT, Action: Custom);
1413 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1414
1415 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1416 ISD::MGATHER, ISD::MSCATTER, ISD::VP_LOAD,
1417 ISD::VP_STORE, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1418 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1419 ISD::VP_SCATTER},
1420 VT, Action: Custom);
1421 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1422
1423 // Expand FP operations that need libcalls.
1424 setOperationAction(Ops: FloatingPointLibCallOps, VT, Action: Expand);
1425
1426 // Custom split nxv32[b]f16 since nxv32[b]f32 is not legal.
1427 if (getLMUL(VT) == RISCVVType::LMUL_8) {
1428 setOperationAction(Ops: ZvfbfaPromoteOps, VT, Action: Custom);
1429 setOperationAction(Ops: ZvfhminZvfbfminPromoteVPOps, VT, Action: Custom);
1430 } else {
1431 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1432 setOperationPromotedToType(Ops: ZvfbfaPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1433 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1434 }
1435 };
1436
1437 if (Subtarget.hasVInstructionsF16()) {
1438 for (MVT VT : F16VecVTs) {
1439 if (!isTypeLegal(VT))
1440 continue;
1441 SetCommonVFPActions(VT);
1442 }
1443 } else if (Subtarget.hasVInstructionsF16Minimal()) {
1444 for (MVT VT : F16VecVTs) {
1445 if (!isTypeLegal(VT))
1446 continue;
1447 SetCommonPromoteToF32Actions(VT);
1448 }
1449 }
1450
1451 if (Subtarget.hasVInstructionsBF16()) {
1452 for (MVT VT : BF16VecVTs) {
1453 if (!isTypeLegal(VT))
1454 continue;
1455 SetZvfbfaActions(VT);
1456 }
1457 } else if (Subtarget.hasVInstructionsBF16Minimal()) {
1458 for (MVT VT : BF16VecVTs) {
1459 if (!isTypeLegal(VT))
1460 continue;
1461 SetCommonPromoteToF32Actions(VT);
1462 }
1463 }
1464
1465 if (Subtarget.hasVInstructionsF32()) {
1466 for (MVT VT : F32VecVTs) {
1467 if (!isTypeLegal(VT))
1468 continue;
1469 SetCommonVFPActions(VT);
1470 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1471 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1472 }
1473 }
1474
1475 if (Subtarget.hasVInstructionsF64()) {
1476 for (MVT VT : F64VecVTs) {
1477 if (!isTypeLegal(VT))
1478 continue;
1479 SetCommonVFPActions(VT);
1480 SetCommonVFPExtLoadTruncStoreActions(VT, F16VecVTs);
1481 SetCommonVFPExtLoadTruncStoreActions(VT, BF16VecVTs);
1482 SetCommonVFPExtLoadTruncStoreActions(VT, F32VecVTs);
1483 }
1484 }
1485
1486 if (Subtarget.useRVVForFixedLengthVectors()) {
1487 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1488 if (!useRVVForFixedLengthVectorVT(VT))
1489 continue;
1490
1491 // By default everything must be expanded.
1492 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1493 setOperationAction(Op, VT, Action: Expand);
1494 for (MVT OtherVT : MVT::integer_fixedlen_vector_valuetypes()) {
1495 setTruncStoreAction(ValVT: VT, MemVT: OtherVT, Action: Expand);
1496 setLoadExtAction(ExtTypes: {ISD::EXTLOAD, ISD::SEXTLOAD, ISD::ZEXTLOAD}, ValVT: VT,
1497 MemVT: OtherVT, Action: Expand);
1498 }
1499
1500 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1501 // expansion to a build_vector of 0s.
1502 setOperationAction(Op: ISD::UNDEF, VT, Action: Custom);
1503
1504 // We use EXTRACT_SUBVECTOR as a "cast" from scalable to fixed.
1505 setOperationAction(Ops: {ISD::INSERT_SUBVECTOR, ISD::EXTRACT_SUBVECTOR}, VT,
1506 Action: Custom);
1507
1508 setOperationAction(
1509 Ops: {ISD::BUILD_VECTOR, ISD::CONCAT_VECTORS, ISD::VECTOR_REVERSE}, VT,
1510 Action: Custom);
1511
1512 setOperationAction(Ops: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1513 VT, Action: Custom);
1514
1515 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT},
1516 VT, Action: Custom);
1517
1518 setOperationAction(Op: ISD::SCALAR_TO_VECTOR, VT, Action: Custom);
1519
1520 setOperationAction(Ops: {ISD::LOAD, ISD::STORE}, VT, Action: Custom);
1521
1522 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1523
1524 setOperationAction(Op: ISD::SELECT, VT, Action: Custom);
1525
1526 setOperationAction(
1527 Ops: {ISD::TRUNCATE, ISD::TRUNCATE_SSAT_S, ISD::TRUNCATE_USAT_U}, VT,
1528 Action: Custom);
1529
1530 setOperationAction(Op: ISD::BITCAST, VT, Action: Custom);
1531
1532 setOperationAction(
1533 Ops: {ISD::VECREDUCE_AND, ISD::VECREDUCE_OR, ISD::VECREDUCE_XOR}, VT,
1534 Action: Custom);
1535
1536 setOperationAction(
1537 Ops: {ISD::VP_REDUCE_AND, ISD::VP_REDUCE_OR, ISD::VP_REDUCE_XOR}, VT,
1538 Action: Custom);
1539
1540 setOperationAction(
1541 Ops: {
1542 ISD::SINT_TO_FP,
1543 ISD::UINT_TO_FP,
1544 ISD::FP_TO_SINT,
1545 ISD::FP_TO_UINT,
1546 ISD::STRICT_SINT_TO_FP,
1547 ISD::STRICT_UINT_TO_FP,
1548 ISD::STRICT_FP_TO_SINT,
1549 ISD::STRICT_FP_TO_UINT,
1550 },
1551 VT, Action: Custom);
1552 setOperationAction(Ops: {ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT}, VT,
1553 Action: Custom);
1554
1555 setOperationAction(Op: ISD::VECTOR_SHUFFLE, VT, Action: Custom);
1556
1557 // Operations below are different for between masks and other vectors.
1558 if (VT.getVectorElementType() == MVT::i1) {
1559 setOperationAction(Ops: {ISD::VP_AND, ISD::VP_OR, ISD::VP_XOR, ISD::AND,
1560 ISD::OR, ISD::XOR},
1561 VT, Action: Custom);
1562
1563 setOperationAction(Ops: {ISD::VP_FP_TO_SINT, ISD::VP_FP_TO_UINT,
1564 ISD::VP_SETCC, ISD::VP_TRUNCATE},
1565 VT, Action: Custom);
1566
1567 setOperationAction(Op: ISD::VP_MERGE, VT, Action: Custom);
1568
1569 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1570 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1571
1572 setOperationAction(Ops: {ISD::CTTZ_ELTS, ISD::CTTZ_ELTS_ZERO_POISON}, VT,
1573 Action: Custom);
1574 continue;
1575 }
1576
1577 // Make SPLAT_VECTOR Legal so DAGCombine will convert splat vectors to
1578 // it before type legalization for i64 vectors on RV32. It will then be
1579 // type legalized to SPLAT_VECTOR_PARTS which we need to Custom handle.
1580 // FIXME: Use SPLAT_VECTOR for all types? DAGCombine probably needs
1581 // improvements first.
1582 if (!Subtarget.is64Bit() && VT.getVectorElementType() == MVT::i64) {
1583 setOperationAction(Op: ISD::SPLAT_VECTOR, VT, Action: Legal);
1584 setOperationAction(Op: ISD::SPLAT_VECTOR_PARTS, VT, Action: Custom);
1585
1586 // Lower BUILD_VECTOR with i64 type to VID on RV32 if possible.
1587 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::i64, Action: Custom);
1588 }
1589
1590 setOperationAction(
1591 Ops: {ISD::MLOAD, ISD::MSTORE, ISD::MGATHER, ISD::MSCATTER}, VT, Action: Custom);
1592
1593 setOperationAction(Ops: {ISD::VP_LOAD, ISD::VP_STORE,
1594 ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1595 ISD::EXPERIMENTAL_VP_STRIDED_STORE, ISD::VP_GATHER,
1596 ISD::VP_SCATTER},
1597 VT, Action: Custom);
1598 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1599
1600 setOperationAction(Ops: {ISD::ADD, ISD::MUL, ISD::SUB, ISD::AND, ISD::OR,
1601 ISD::XOR, ISD::SDIV, ISD::SREM, ISD::UDIV,
1602 ISD::UREM, ISD::SHL, ISD::SRA, ISD::SRL},
1603 VT, Action: Custom);
1604
1605 setOperationAction(
1606 Ops: {ISD::SMIN, ISD::SMAX, ISD::UMIN, ISD::UMAX, ISD::ABS}, VT, Action: Custom);
1607
1608 setOperationAction(Ops: {ISD::ABDS, ISD::ABDU}, VT, Action: Custom);
1609
1610 // vXi64 MULHS/MULHU requires the V extension instead of Zve64*.
1611 if (VT.getVectorElementType() != MVT::i64 || Subtarget.hasStdExtV())
1612 setOperationAction(Ops: {ISD::MULHS, ISD::MULHU}, VT, Action: Custom);
1613
1614 setOperationAction(Ops: {ISD::AVGFLOORS, ISD::AVGFLOORU, ISD::AVGCEILS,
1615 ISD::AVGCEILU, ISD::SADDSAT, ISD::UADDSAT,
1616 ISD::SSUBSAT, ISD::USUBSAT},
1617 VT, Action: Custom);
1618
1619 setOperationAction(Op: ISD::VSELECT, VT, Action: Custom);
1620
1621 setOperationAction(
1622 Ops: {ISD::ANY_EXTEND, ISD::SIGN_EXTEND, ISD::ZERO_EXTEND}, VT, Action: Custom);
1623
1624 // Custom-lower reduction operations to set up the corresponding custom
1625 // nodes' operands.
1626 setOperationAction(Ops: {ISD::VECREDUCE_ADD, ISD::VECREDUCE_SMAX,
1627 ISD::VECREDUCE_SMIN, ISD::VECREDUCE_UMAX,
1628 ISD::VECREDUCE_UMIN},
1629 VT, Action: Custom);
1630
1631 setOperationAction(Ops: IntegerVPOps, VT, Action: Custom);
1632
1633 if (Subtarget.hasStdExtZvkb())
1634 setOperationAction(Ops: {ISD::BSWAP, ISD::ROTL, ISD::ROTR}, VT, Action: Custom);
1635
1636 if (Subtarget.hasStdExtZvbb()) {
1637 setOperationAction(Ops: {ISD::BITREVERSE, ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF,
1638 ISD::CTTZ, ISD::CTTZ_ZERO_UNDEF, ISD::CTPOP},
1639 VT, Action: Custom);
1640 } else {
1641 // Lower CTLZ_ZERO_UNDEF and CTTZ_ZERO_UNDEF if element of VT in the
1642 // range of f32.
1643 EVT FloatVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1644 if (isTypeLegal(VT: FloatVT))
1645 setOperationAction(
1646 Ops: {ISD::CTLZ, ISD::CTLZ_ZERO_UNDEF, ISD::CTTZ_ZERO_UNDEF}, VT,
1647 Action: Custom);
1648 }
1649
1650 setOperationAction(Op: ISD::VECTOR_COMPRESS, VT, Action: Custom);
1651 }
1652
1653 for (MVT VT : MVT::fp_fixedlen_vector_valuetypes()) {
1654 // There are no extending loads or truncating stores.
1655 for (MVT InnerVT : MVT::fp_fixedlen_vector_valuetypes()) {
1656 setLoadExtAction(ExtType: ISD::EXTLOAD, ValVT: VT, MemVT: InnerVT, Action: Expand);
1657 setTruncStoreAction(ValVT: VT, MemVT: InnerVT, Action: Expand);
1658 }
1659
1660 if (!useRVVForFixedLengthVectorVT(VT))
1661 continue;
1662
1663 // By default everything must be expanded.
1664 for (unsigned Op = 0; Op < ISD::BUILTIN_OP_END; ++Op)
1665 setOperationAction(Op, VT, Action: Expand);
1666
1667 // Custom lower fixed vector undefs to scalable vector undefs to avoid
1668 // expansion to a build_vector of 0s.
1669 setOperationAction(Op: ISD::UNDEF, VT, Action: Custom);
1670
1671 setOperationAction(Ops: {ISD::INSERT_VECTOR_ELT, ISD::EXTRACT_VECTOR_ELT,
1672 ISD::CONCAT_VECTORS, ISD::INSERT_SUBVECTOR,
1673 ISD::EXTRACT_SUBVECTOR, ISD::VECTOR_REVERSE,
1674 ISD::VECTOR_SHUFFLE, ISD::VECTOR_COMPRESS},
1675 VT, Action: Custom);
1676 setOperationAction(Op: ISD::EXPERIMENTAL_VP_SPLICE, VT, Action: Custom);
1677 setOperationAction(Op: ISD::EXPERIMENTAL_VP_REVERSE, VT, Action: Custom);
1678
1679 setOperationAction(Ops: {ISD::VECTOR_INTERLEAVE, ISD::VECTOR_DEINTERLEAVE},
1680 VT, Action: Custom);
1681
1682 setOperationAction(Ops: {ISD::LOAD, ISD::STORE, ISD::MLOAD, ISD::MSTORE,
1683 ISD::MGATHER, ISD::MSCATTER},
1684 VT, Action: Custom);
1685 setOperationAction(Ops: {ISD::VP_LOAD, ISD::VP_STORE, ISD::VP_GATHER,
1686 ISD::VP_SCATTER, ISD::EXPERIMENTAL_VP_STRIDED_LOAD,
1687 ISD::EXPERIMENTAL_VP_STRIDED_STORE},
1688 VT, Action: Custom);
1689 setOperationAction(Op: ISD::VP_LOAD_FF, VT, Action: Custom);
1690
1691 setOperationAction(Ops: {ISD::FP_ROUND, ISD::FP_EXTEND}, VT, Action: Custom);
1692 setOperationAction(Ops: {ISD::STRICT_FP_ROUND, ISD::STRICT_FP_EXTEND}, VT,
1693 Action: Custom);
1694
1695 setOperationAction(Op: ISD::BITCAST, VT, Action: Custom);
1696
1697 if (VT.getVectorElementType() == MVT::f16 &&
1698 !Subtarget.hasVInstructionsF16()) {
1699 setOperationAction(Ops: {ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Action: Custom);
1700 setOperationAction(
1701 Ops: {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1702 Action: Custom);
1703 setOperationAction(Ops: {ISD::VP_SINT_TO_FP, ISD::VP_UINT_TO_FP}, VT,
1704 Action: Custom);
1705 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1706 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1707 if (Subtarget.hasStdExtZfhmin()) {
1708 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
1709 } else {
1710 // We need to custom legalize f16 build vectors if Zfhmin isn't
1711 // available.
1712 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::f16, Action: Custom);
1713 }
1714 setOperationAction(Op: ISD::FNEG, VT, Action: Expand);
1715 setOperationAction(Op: ISD::FABS, VT, Action: Expand);
1716 setOperationAction(Op: ISD::FCOPYSIGN, VT, Action: Expand);
1717 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1718 // Don't promote f16 vector operations to f32 if f32 vector type is
1719 // not legal.
1720 // TODO: could split the f16 vector into two vectors and do promotion.
1721 if (!isTypeLegal(VT: F32VecVT))
1722 continue;
1723 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1724 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1725 continue;
1726 }
1727
1728 if (VT.getVectorElementType() == MVT::bf16) {
1729 setOperationAction(Ops: {ISD::VP_FP_ROUND, ISD::VP_FP_EXTEND}, VT, Action: Custom);
1730 setOperationAction(Ops: {ISD::LRINT, ISD::LLRINT}, VT, Action: Custom);
1731 setOperationAction(Ops: {ISD::LROUND, ISD::LLROUND}, VT, Action: Custom);
1732 if (Subtarget.hasStdExtZfbfmin()) {
1733 setOperationAction(Op: ISD::BUILD_VECTOR, VT, Action: Custom);
1734 } else {
1735 // We need to custom legalize bf16 build vectors if Zfbfmin isn't
1736 // available.
1737 setOperationAction(Op: ISD::BUILD_VECTOR, VT: MVT::bf16, Action: Custom);
1738 }
1739 if (Subtarget.hasStdExtZvfbfa()) {
1740 setOperationAction(Ops: ZvfbfaOps, VT, Action: Custom);
1741 setOperationAction(Ops: ZvfbfaVPOps, VT, Action: Custom);
1742 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1743 }
1744 setOperationAction(
1745 Ops: {ISD::VP_MERGE, ISD::VP_SELECT, ISD::VSELECT, ISD::SELECT}, VT,
1746 Action: Custom);
1747 MVT F32VecVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
1748 // Don't promote f16 vector operations to f32 if f32 vector type is
1749 // not legal.
1750 // TODO: could split the f16 vector into two vectors and do promotion.
1751 if (!isTypeLegal(VT: F32VecVT))
1752 continue;
1753
1754 if (Subtarget.hasStdExtZvfbfa())
1755 setOperationPromotedToType(Ops: ZvfbfaPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1756 else
1757 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteOps, OrigVT: VT, DestVT: F32VecVT);
1758 setOperationPromotedToType(Ops: ZvfhminZvfbfminPromoteVPOps, OrigVT: VT, DestVT: F32VecVT);
1759 continue;
1760 }
1761
1762 setOperationAction(Ops: {ISD::BUILD_VECTOR, ISD::SCALAR_TO_VECTOR}, VT,
1763 Action: Custom);
1764
1765 setOperationAction(Ops: {ISD::FADD, ISD::FSUB, ISD::FMUL, ISD::FDIV,
1766 ISD::FNEG, ISD::FABS, ISD::FCOPYSIGN, ISD::FSQRT,
1767 ISD::FMA, ISD::FMINNUM, ISD::FMAXNUM,
1768 ISD::FMINIMUMNUM, ISD::FMAXIMUMNUM, ISD::IS_FPCLASS,
1769 ISD::FMAXIMUM, ISD::FMINIMUM},
1770 VT, Action: Custom);
1771
1772 setOperationAction(Ops: {ISD::FTRUNC, ISD::FCEIL, ISD::FFLOOR, ISD::FROUND,
1773 ISD::FROUNDEVEN, ISD::FRINT, ISD::LRINT,
1774 ISD::LLRINT, ISD::LROUND, ISD::LLROUND,
1775 ISD::FNEARBYINT},
1776 VT, Action: Custom);
1777
1778 setCondCodeAction(CCs: VFPCCToExpand, VT, Action: Expand);
1779
1780 setOperationAction(Op: ISD::SETCC, VT, Action: Custom);
1781 setOperationAction(Ops: {ISD::VSELECT, ISD::SELECT}, VT, Action: Custom);
1782
1783 setOperationAction(Ops: FloatingPointVecReduceOps, VT, Action: Custom);
1784
1785 setOperationAction(Ops: FloatingPointVPOps, VT, Action: Custom);
1786
1787 setOperationAction(
1788 Ops: {ISD::STRICT_FADD, ISD::STRICT_FSUB, ISD::STRICT_FMUL,
1789 ISD::STRICT_FDIV, ISD::STRICT_FSQRT, ISD::STRICT_FMA,
1790 ISD::STRICT_FSETCC, ISD::STRICT_FSETCCS, ISD::STRICT_FTRUNC,
1791 ISD::STRICT_FCEIL, ISD::STRICT_FFLOOR, ISD::STRICT_FROUND,
1792 ISD::STRICT_FROUNDEVEN, ISD::STRICT_FNEARBYINT},
1793 VT, Action: Custom);
1794 }
1795
1796 // Custom-legalize bitcasts from fixed-length vectors to scalar types.
1797 setOperationAction(Ops: ISD::BITCAST, VTs: {MVT::i8, MVT::i16, MVT::i32}, Action: Custom);
1798 if (Subtarget.is64Bit())
1799 setOperationAction(Op: ISD::BITCAST, VT: MVT::i64, Action: Custom);
1800 if (Subtarget.hasStdExtZfhminOrZhinxmin())
1801 setOperationAction(Op: ISD::BITCAST, VT: MVT::f16, Action: Custom);
1802 if (Subtarget.hasStdExtZfbfmin())
1803 setOperationAction(Op: ISD::BITCAST, VT: MVT::bf16, Action: Custom);
1804 if (Subtarget.hasStdExtFOrZfinx())
1805 setOperationAction(Op: ISD::BITCAST, VT: MVT::f32, Action: Custom);
1806 if (Subtarget.hasStdExtDOrZdinx())
1807 setOperationAction(Op: ISD::BITCAST, VT: MVT::f64, Action: Custom);
1808 }
1809 }
1810
1811 if (Subtarget.hasStdExtZaamo())
1812 setOperationAction(Op: ISD::ATOMIC_LOAD_SUB, VT: XLenVT, Action: Expand);
1813
1814 if (Subtarget.hasForcedAtomics()) {
1815 // Force __sync libcalls to be emitted for atomic rmw/cas operations.
1816 setOperationAction(
1817 Ops: {ISD::ATOMIC_CMP_SWAP, ISD::ATOMIC_SWAP, ISD::ATOMIC_LOAD_ADD,
1818 ISD::ATOMIC_LOAD_SUB, ISD::ATOMIC_LOAD_AND, ISD::ATOMIC_LOAD_OR,
1819 ISD::ATOMIC_LOAD_XOR, ISD::ATOMIC_LOAD_NAND, ISD::ATOMIC_LOAD_MIN,
1820 ISD::ATOMIC_LOAD_MAX, ISD::ATOMIC_LOAD_UMIN, ISD::ATOMIC_LOAD_UMAX},
1821 VT: XLenVT, Action: LibCall);
1822 }
1823
1824 if (Subtarget.hasVendorXTHeadMemIdx()) {
1825 for (unsigned im : {ISD::PRE_INC, ISD::POST_INC}) {
1826 setIndexedLoadAction(IdxModes: im, VT: MVT::i8, Action: Legal);
1827 setIndexedStoreAction(IdxModes: im, VT: MVT::i8, Action: Legal);
1828 setIndexedLoadAction(IdxModes: im, VT: MVT::i16, Action: Legal);
1829 setIndexedStoreAction(IdxModes: im, VT: MVT::i16, Action: Legal);
1830 setIndexedLoadAction(IdxModes: im, VT: MVT::i32, Action: Legal);
1831 setIndexedStoreAction(IdxModes: im, VT: MVT::i32, Action: Legal);
1832
1833 if (Subtarget.is64Bit()) {
1834 setIndexedLoadAction(IdxModes: im, VT: MVT::i64, Action: Legal);
1835 setIndexedStoreAction(IdxModes: im, VT: MVT::i64, Action: Legal);
1836 }
1837 }
1838 }
1839
1840 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
1841 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i8, Action: Legal);
1842 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i16, Action: Legal);
1843 setIndexedLoadAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
1844
1845 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i8, Action: Legal);
1846 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i16, Action: Legal);
1847 setIndexedStoreAction(IdxModes: ISD::POST_INC, VT: MVT::i32, Action: Legal);
1848 }
1849
1850 // zve32x is broken for partial_reduce_umla, but let's not make it worse.
1851 if (Subtarget.hasStdExtZvdot4a8i() && Subtarget.getELen() >= 64) {
1852 static const unsigned MLAOps[] = {ISD::PARTIAL_REDUCE_SMLA,
1853 ISD::PARTIAL_REDUCE_UMLA,
1854 ISD::PARTIAL_REDUCE_SUMLA};
1855 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv1i32, InputVT: MVT::nxv4i8, Action: Custom);
1856 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv2i32, InputVT: MVT::nxv8i8, Action: Custom);
1857 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv4i32, InputVT: MVT::nxv16i8, Action: Custom);
1858 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv8i32, InputVT: MVT::nxv32i8, Action: Custom);
1859 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: MVT::nxv16i32, InputVT: MVT::nxv64i8, Action: Custom);
1860
1861 if (Subtarget.useRVVForFixedLengthVectors()) {
1862 for (MVT VT : MVT::integer_fixedlen_vector_valuetypes()) {
1863 if (VT.getVectorElementType() != MVT::i32 ||
1864 !useRVVForFixedLengthVectorVT(VT))
1865 continue;
1866 ElementCount EC = VT.getVectorElementCount();
1867 MVT ArgVT = MVT::getVectorVT(VT: MVT::i8, EC: EC.multiplyCoefficientBy(RHS: 4));
1868 setPartialReduceMLAAction(Opcodes: MLAOps, AccVT: VT, InputVT: ArgVT, Action: Custom);
1869 }
1870 }
1871 }
1872
1873 // Customize load and store operation for bf16 if zfh isn't enabled.
1874 if (Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh()) {
1875 setOperationAction(Op: ISD::LOAD, VT: MVT::bf16, Action: Custom);
1876 setOperationAction(Op: ISD::STORE, VT: MVT::bf16, Action: Custom);
1877 }
1878
1879 // Function alignments.
1880 const Align FunctionAlignment(Subtarget.hasStdExtZca() ? 2 : 4);
1881 setMinFunctionAlignment(FunctionAlignment);
1882 // Set preferred alignments.
1883 setPrefFunctionAlignment(Subtarget.getPrefFunctionAlignment());
1884 setPrefLoopAlignment(Subtarget.getPrefLoopAlignment());
1885
1886 setTargetDAGCombine({ISD::INTRINSIC_VOID, ISD::INTRINSIC_W_CHAIN,
1887 ISD::INTRINSIC_WO_CHAIN, ISD::ADD, ISD::SUB, ISD::MUL,
1888 ISD::AND, ISD::OR, ISD::XOR, ISD::SETCC, ISD::SELECT,
1889 ISD::SRA});
1890 setTargetDAGCombine(ISD::SIGN_EXTEND_INREG);
1891
1892 if (Subtarget.hasStdExtFOrZfinx())
1893 setTargetDAGCombine({ISD::FADD, ISD::FMAXNUM, ISD::FMINNUM, ISD::FMUL});
1894
1895 if (Subtarget.hasStdExtZbb())
1896 setTargetDAGCombine({ISD::UMAX, ISD::UMIN, ISD::SMAX, ISD::SMIN});
1897
1898 if ((Subtarget.hasStdExtZbs() && Subtarget.is64Bit()) ||
1899 Subtarget.hasVInstructions())
1900 setTargetDAGCombine(ISD::TRUNCATE);
1901
1902 if (Subtarget.hasStdExtZbkb())
1903 setTargetDAGCombine(ISD::BITREVERSE);
1904
1905 if (Subtarget.hasStdExtFOrZfinx())
1906 setTargetDAGCombine({ISD::ZERO_EXTEND, ISD::FP_TO_SINT, ISD::FP_TO_UINT,
1907 ISD::FP_TO_SINT_SAT, ISD::FP_TO_UINT_SAT});
1908 if (Subtarget.hasVInstructions())
1909 setTargetDAGCombine({ISD::FCOPYSIGN,
1910 ISD::MGATHER,
1911 ISD::MSCATTER,
1912 ISD::VP_GATHER,
1913 ISD::VP_SCATTER,
1914 ISD::SRL,
1915 ISD::SHL,
1916 ISD::STORE,
1917 ISD::SPLAT_VECTOR,
1918 ISD::BUILD_VECTOR,
1919 ISD::CONCAT_VECTORS,
1920 ISD::VP_STORE,
1921 ISD::VP_TRUNCATE,
1922 ISD::EXPERIMENTAL_VP_REVERSE,
1923 ISD::SDIV,
1924 ISD::UDIV,
1925 ISD::SREM,
1926 ISD::UREM,
1927 ISD::INSERT_VECTOR_ELT,
1928 ISD::ABS,
1929 ISD::CTPOP,
1930 ISD::VECTOR_SHUFFLE,
1931 ISD::FMA,
1932 ISD::VSELECT,
1933 ISD::VECREDUCE_ADD});
1934
1935 if (Subtarget.hasVendorXTHeadMemPair())
1936 setTargetDAGCombine({ISD::LOAD, ISD::STORE});
1937 if (Subtarget.useRVVForFixedLengthVectors())
1938 setTargetDAGCombine(ISD::BITCAST);
1939
1940 setMaxDivRemBitWidthSupported(Subtarget.is64Bit() ? 128 : 64);
1941
1942 // Disable strict node mutation.
1943 IsStrictFPEnabled = true;
1944 EnableExtLdPromotion = true;
1945
1946 // Let the subtarget decide if a predictable select is more expensive than the
1947 // corresponding branch. This information is used in CGP/SelectOpt to decide
1948 // when to convert selects into branches.
1949 PredictableSelectIsExpensive = Subtarget.predictableSelectIsExpensive();
1950
1951 MaxStoresPerMemsetOptSize = Subtarget.getMaxStoresPerMemset(/*OptSize=*/true);
1952 MaxStoresPerMemset = Subtarget.getMaxStoresPerMemset(/*OptSize=*/false);
1953
1954 MaxGluedStoresPerMemcpy = Subtarget.getMaxGluedStoresPerMemcpy();
1955 MaxStoresPerMemcpyOptSize = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/true);
1956 MaxStoresPerMemcpy = Subtarget.getMaxStoresPerMemcpy(/*OptSize=*/false);
1957
1958 MaxStoresPerMemmoveOptSize =
1959 Subtarget.getMaxStoresPerMemmove(/*OptSize=*/true);
1960 MaxStoresPerMemmove = Subtarget.getMaxStoresPerMemmove(/*OptSize=*/false);
1961
1962 MaxLoadsPerMemcmpOptSize = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/true);
1963 MaxLoadsPerMemcmp = Subtarget.getMaxLoadsPerMemcmp(/*OptSize=*/false);
1964}
1965
1966TargetLoweringBase::LegalizeTypeAction
1967RISCVTargetLowering::getPreferredVectorAction(MVT VT) const {
1968 if (Subtarget.is64Bit() && Subtarget.hasStdExtP())
1969 if (VT == MVT::v2i16 || VT == MVT::v4i8)
1970 return TypeWidenVector;
1971
1972 return TargetLoweringBase::getPreferredVectorAction(VT);
1973}
1974
1975EVT RISCVTargetLowering::getSetCCResultType(const DataLayout &DL,
1976 LLVMContext &Context,
1977 EVT VT) const {
1978 if (!VT.isVector())
1979 return getPointerTy(DL);
1980 if (Subtarget.hasVInstructions() &&
1981 (VT.isScalableVector() || Subtarget.useRVVForFixedLengthVectors()))
1982 return EVT::getVectorVT(Context, VT: MVT::i1, EC: VT.getVectorElementCount());
1983 return VT.changeVectorElementTypeToInteger();
1984}
1985
1986MVT RISCVTargetLowering::getVPExplicitVectorLengthTy() const {
1987 return Subtarget.getXLenVT();
1988}
1989
1990// Return false if we can lower get_vector_length to a vsetvli intrinsic.
1991bool RISCVTargetLowering::shouldExpandGetVectorLength(EVT TripCountVT,
1992 unsigned VF,
1993 bool IsScalable) const {
1994 if (!Subtarget.hasVInstructions())
1995 return true;
1996
1997 if (!IsScalable)
1998 return true;
1999
2000 if (TripCountVT != MVT::i32 && TripCountVT != Subtarget.getXLenVT())
2001 return true;
2002
2003 // Don't allow VF=1 if those types are't legal.
2004 if (VF < RISCV::RVVBitsPerBlock / Subtarget.getELen())
2005 return true;
2006
2007 // VLEN=32 support is incomplete.
2008 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
2009 return true;
2010
2011 // The maximum VF is for the smallest element width with LMUL=8.
2012 // VF must be a power of 2.
2013 unsigned MaxVF = RISCV::RVVBytesPerBlock * 8;
2014 return VF > MaxVF || !isPowerOf2_32(Value: VF);
2015}
2016
2017bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const {
2018 return !Subtarget.hasVInstructions() ||
2019 VT.getVectorElementType() != MVT::i1 || !isTypeLegal(VT);
2020}
2021
2022void RISCVTargetLowering::getTgtMemIntrinsic(
2023 SmallVectorImpl<IntrinsicInfo> &Infos, const CallBase &I,
2024 MachineFunction &MF, unsigned Intrinsic) const {
2025 IntrinsicInfo Info;
2026 auto &DL = I.getDataLayout();
2027
2028 auto SetRVVLoadStoreInfo = [&](unsigned PtrOp, bool IsStore,
2029 bool IsUnitStrided, bool UsePtrVal = false) {
2030 Info.opc = IsStore ? ISD::INTRINSIC_VOID : ISD::INTRINSIC_W_CHAIN;
2031 // We can't use ptrVal if the intrinsic can access memory before the
2032 // pointer. This means we can't use it for strided or indexed intrinsics.
2033 if (UsePtrVal)
2034 Info.ptrVal = I.getArgOperand(i: PtrOp);
2035 else
2036 Info.fallbackAddressSpace =
2037 I.getArgOperand(i: PtrOp)->getType()->getPointerAddressSpace();
2038 Type *MemTy;
2039 if (IsStore) {
2040 // Store value is the first operand.
2041 MemTy = I.getArgOperand(i: 0)->getType();
2042 } else {
2043 // Use return type. If it's segment load, return type is a struct.
2044 MemTy = I.getType();
2045 if (MemTy->isStructTy())
2046 MemTy = MemTy->getStructElementType(N: 0);
2047 }
2048 if (!IsUnitStrided)
2049 MemTy = MemTy->getScalarType();
2050
2051 Info.memVT = getValueType(DL, Ty: MemTy);
2052 if (MemTy->isTargetExtTy()) {
2053 // RISC-V vector tuple type's alignment type should be its element type.
2054 if (cast<TargetExtType>(Val: MemTy)->getName() == "riscv.vector.tuple")
2055 MemTy = Type::getIntNTy(
2056 C&: MemTy->getContext(),
2057 N: 1 << cast<ConstantInt>(Val: I.getArgOperand(i: I.arg_size() - 1))
2058 ->getZExtValue());
2059 Info.align = DL.getABITypeAlign(Ty: MemTy);
2060 } else {
2061 Info.align = Align(DL.getTypeStoreSize(Ty: MemTy->getScalarType()));
2062 }
2063 Info.size = MemoryLocation::UnknownSize;
2064 Info.flags |=
2065 IsStore ? MachineMemOperand::MOStore : MachineMemOperand::MOLoad;
2066 Infos.push_back(Elt: Info);
2067 };
2068
2069 if (I.hasMetadata(KindID: LLVMContext::MD_nontemporal))
2070 Info.flags |= MachineMemOperand::MONonTemporal;
2071
2072 Info.flags |= RISCVTargetLowering::getTargetMMOFlags(I);
2073 switch (Intrinsic) {
2074 default:
2075 return;
2076 case Intrinsic::riscv_masked_atomicrmw_xchg:
2077 case Intrinsic::riscv_masked_atomicrmw_add:
2078 case Intrinsic::riscv_masked_atomicrmw_sub:
2079 case Intrinsic::riscv_masked_atomicrmw_nand:
2080 case Intrinsic::riscv_masked_atomicrmw_max:
2081 case Intrinsic::riscv_masked_atomicrmw_min:
2082 case Intrinsic::riscv_masked_atomicrmw_umax:
2083 case Intrinsic::riscv_masked_atomicrmw_umin:
2084 case Intrinsic::riscv_masked_cmpxchg:
2085 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
2086 // narrow atomic operation. These will be expanded to an LR/SC loop that
2087 // reads/writes to/from an aligned 4 byte location. And, or, shift, etc.
2088 // will be used to modify the appropriate part of the 4 byte data and
2089 // preserve the rest.
2090 Info.opc = ISD::INTRINSIC_W_CHAIN;
2091 Info.memVT = MVT::i32;
2092 Info.ptrVal = I.getArgOperand(i: 0);
2093 Info.offset = 0;
2094 Info.align = Align(4);
2095 Info.flags = MachineMemOperand::MOLoad | MachineMemOperand::MOStore |
2096 MachineMemOperand::MOVolatile;
2097 Infos.push_back(Elt: Info);
2098 return;
2099 case Intrinsic::riscv_seg2_load_mask:
2100 case Intrinsic::riscv_seg3_load_mask:
2101 case Intrinsic::riscv_seg4_load_mask:
2102 case Intrinsic::riscv_seg5_load_mask:
2103 case Intrinsic::riscv_seg6_load_mask:
2104 case Intrinsic::riscv_seg7_load_mask:
2105 case Intrinsic::riscv_seg8_load_mask:
2106 case Intrinsic::riscv_sseg2_load_mask:
2107 case Intrinsic::riscv_sseg3_load_mask:
2108 case Intrinsic::riscv_sseg4_load_mask:
2109 case Intrinsic::riscv_sseg5_load_mask:
2110 case Intrinsic::riscv_sseg6_load_mask:
2111 case Intrinsic::riscv_sseg7_load_mask:
2112 case Intrinsic::riscv_sseg8_load_mask:
2113 SetRVVLoadStoreInfo(/*PtrOp*/ 0, /*IsStore*/ false,
2114 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2115 return;
2116 case Intrinsic::riscv_seg2_store_mask:
2117 case Intrinsic::riscv_seg3_store_mask:
2118 case Intrinsic::riscv_seg4_store_mask:
2119 case Intrinsic::riscv_seg5_store_mask:
2120 case Intrinsic::riscv_seg6_store_mask:
2121 case Intrinsic::riscv_seg7_store_mask:
2122 case Intrinsic::riscv_seg8_store_mask:
2123 // Operands are (vec, ..., vec, ptr, mask, vl)
2124 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2125 /*IsStore*/ true,
2126 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2127 return;
2128 case Intrinsic::riscv_sseg2_store_mask:
2129 case Intrinsic::riscv_sseg3_store_mask:
2130 case Intrinsic::riscv_sseg4_store_mask:
2131 case Intrinsic::riscv_sseg5_store_mask:
2132 case Intrinsic::riscv_sseg6_store_mask:
2133 case Intrinsic::riscv_sseg7_store_mask:
2134 case Intrinsic::riscv_sseg8_store_mask:
2135 // Operands are (vec, ..., vec, ptr, offset, mask, vl)
2136 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2137 /*IsStore*/ true,
2138 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2139 return;
2140 case Intrinsic::riscv_vlm:
2141 SetRVVLoadStoreInfo(/*PtrOp*/ 0,
2142 /*IsStore*/ false,
2143 /*IsUnitStrided*/ true,
2144 /*UsePtrVal*/ true);
2145 return;
2146 case Intrinsic::riscv_vle:
2147 case Intrinsic::riscv_vle_mask:
2148 case Intrinsic::riscv_vleff:
2149 case Intrinsic::riscv_vleff_mask:
2150 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2151 /*IsStore*/ false,
2152 /*IsUnitStrided*/ true,
2153 /*UsePtrVal*/ true);
2154 return;
2155 case Intrinsic::riscv_vsm:
2156 case Intrinsic::riscv_vse:
2157 case Intrinsic::riscv_vse_mask:
2158 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2159 /*IsStore*/ true,
2160 /*IsUnitStrided*/ true,
2161 /*UsePtrVal*/ true);
2162 return;
2163 case Intrinsic::riscv_vlse:
2164 case Intrinsic::riscv_vlse_mask:
2165 case Intrinsic::riscv_vloxei:
2166 case Intrinsic::riscv_vloxei_mask:
2167 case Intrinsic::riscv_vluxei:
2168 case Intrinsic::riscv_vluxei_mask:
2169 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2170 /*IsStore*/ false,
2171 /*IsUnitStrided*/ false);
2172 return;
2173 case Intrinsic::riscv_vsse:
2174 case Intrinsic::riscv_vsse_mask:
2175 case Intrinsic::riscv_vsoxei:
2176 case Intrinsic::riscv_vsoxei_mask:
2177 case Intrinsic::riscv_vsuxei:
2178 case Intrinsic::riscv_vsuxei_mask:
2179 SetRVVLoadStoreInfo(/*PtrOp*/ 1,
2180 /*IsStore*/ true,
2181 /*IsUnitStrided*/ false);
2182 return;
2183 case Intrinsic::riscv_vlseg2:
2184 case Intrinsic::riscv_vlseg3:
2185 case Intrinsic::riscv_vlseg4:
2186 case Intrinsic::riscv_vlseg5:
2187 case Intrinsic::riscv_vlseg6:
2188 case Intrinsic::riscv_vlseg7:
2189 case Intrinsic::riscv_vlseg8:
2190 case Intrinsic::riscv_vlseg2ff:
2191 case Intrinsic::riscv_vlseg3ff:
2192 case Intrinsic::riscv_vlseg4ff:
2193 case Intrinsic::riscv_vlseg5ff:
2194 case Intrinsic::riscv_vlseg6ff:
2195 case Intrinsic::riscv_vlseg7ff:
2196 case Intrinsic::riscv_vlseg8ff:
2197 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2198 /*IsStore*/ false,
2199 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2200 return;
2201 case Intrinsic::riscv_vlseg2_mask:
2202 case Intrinsic::riscv_vlseg3_mask:
2203 case Intrinsic::riscv_vlseg4_mask:
2204 case Intrinsic::riscv_vlseg5_mask:
2205 case Intrinsic::riscv_vlseg6_mask:
2206 case Intrinsic::riscv_vlseg7_mask:
2207 case Intrinsic::riscv_vlseg8_mask:
2208 case Intrinsic::riscv_vlseg2ff_mask:
2209 case Intrinsic::riscv_vlseg3ff_mask:
2210 case Intrinsic::riscv_vlseg4ff_mask:
2211 case Intrinsic::riscv_vlseg5ff_mask:
2212 case Intrinsic::riscv_vlseg6ff_mask:
2213 case Intrinsic::riscv_vlseg7ff_mask:
2214 case Intrinsic::riscv_vlseg8ff_mask:
2215 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2216 /*IsStore*/ false,
2217 /*IsUnitStrided*/ false, /*UsePtrVal*/ true);
2218 return;
2219 case Intrinsic::riscv_vlsseg2:
2220 case Intrinsic::riscv_vlsseg3:
2221 case Intrinsic::riscv_vlsseg4:
2222 case Intrinsic::riscv_vlsseg5:
2223 case Intrinsic::riscv_vlsseg6:
2224 case Intrinsic::riscv_vlsseg7:
2225 case Intrinsic::riscv_vlsseg8:
2226 case Intrinsic::riscv_vloxseg2:
2227 case Intrinsic::riscv_vloxseg3:
2228 case Intrinsic::riscv_vloxseg4:
2229 case Intrinsic::riscv_vloxseg5:
2230 case Intrinsic::riscv_vloxseg6:
2231 case Intrinsic::riscv_vloxseg7:
2232 case Intrinsic::riscv_vloxseg8:
2233 case Intrinsic::riscv_vluxseg2:
2234 case Intrinsic::riscv_vluxseg3:
2235 case Intrinsic::riscv_vluxseg4:
2236 case Intrinsic::riscv_vluxseg5:
2237 case Intrinsic::riscv_vluxseg6:
2238 case Intrinsic::riscv_vluxseg7:
2239 case Intrinsic::riscv_vluxseg8:
2240 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2241 /*IsStore*/ false,
2242 /*IsUnitStrided*/ false);
2243 return;
2244 case Intrinsic::riscv_vlsseg2_mask:
2245 case Intrinsic::riscv_vlsseg3_mask:
2246 case Intrinsic::riscv_vlsseg4_mask:
2247 case Intrinsic::riscv_vlsseg5_mask:
2248 case Intrinsic::riscv_vlsseg6_mask:
2249 case Intrinsic::riscv_vlsseg7_mask:
2250 case Intrinsic::riscv_vlsseg8_mask:
2251 case Intrinsic::riscv_vloxseg2_mask:
2252 case Intrinsic::riscv_vloxseg3_mask:
2253 case Intrinsic::riscv_vloxseg4_mask:
2254 case Intrinsic::riscv_vloxseg5_mask:
2255 case Intrinsic::riscv_vloxseg6_mask:
2256 case Intrinsic::riscv_vloxseg7_mask:
2257 case Intrinsic::riscv_vloxseg8_mask:
2258 case Intrinsic::riscv_vluxseg2_mask:
2259 case Intrinsic::riscv_vluxseg3_mask:
2260 case Intrinsic::riscv_vluxseg4_mask:
2261 case Intrinsic::riscv_vluxseg5_mask:
2262 case Intrinsic::riscv_vluxseg6_mask:
2263 case Intrinsic::riscv_vluxseg7_mask:
2264 case Intrinsic::riscv_vluxseg8_mask:
2265 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 6,
2266 /*IsStore*/ false,
2267 /*IsUnitStrided*/ false);
2268 return;
2269 case Intrinsic::riscv_vsseg2:
2270 case Intrinsic::riscv_vsseg3:
2271 case Intrinsic::riscv_vsseg4:
2272 case Intrinsic::riscv_vsseg5:
2273 case Intrinsic::riscv_vsseg6:
2274 case Intrinsic::riscv_vsseg7:
2275 case Intrinsic::riscv_vsseg8:
2276 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 3,
2277 /*IsStore*/ true,
2278 /*IsUnitStrided*/ false);
2279 return;
2280 case Intrinsic::riscv_vsseg2_mask:
2281 case Intrinsic::riscv_vsseg3_mask:
2282 case Intrinsic::riscv_vsseg4_mask:
2283 case Intrinsic::riscv_vsseg5_mask:
2284 case Intrinsic::riscv_vsseg6_mask:
2285 case Intrinsic::riscv_vsseg7_mask:
2286 case Intrinsic::riscv_vsseg8_mask:
2287 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2288 /*IsStore*/ true,
2289 /*IsUnitStrided*/ false);
2290 return;
2291 case Intrinsic::riscv_vssseg2:
2292 case Intrinsic::riscv_vssseg3:
2293 case Intrinsic::riscv_vssseg4:
2294 case Intrinsic::riscv_vssseg5:
2295 case Intrinsic::riscv_vssseg6:
2296 case Intrinsic::riscv_vssseg7:
2297 case Intrinsic::riscv_vssseg8:
2298 case Intrinsic::riscv_vsoxseg2:
2299 case Intrinsic::riscv_vsoxseg3:
2300 case Intrinsic::riscv_vsoxseg4:
2301 case Intrinsic::riscv_vsoxseg5:
2302 case Intrinsic::riscv_vsoxseg6:
2303 case Intrinsic::riscv_vsoxseg7:
2304 case Intrinsic::riscv_vsoxseg8:
2305 case Intrinsic::riscv_vsuxseg2:
2306 case Intrinsic::riscv_vsuxseg3:
2307 case Intrinsic::riscv_vsuxseg4:
2308 case Intrinsic::riscv_vsuxseg5:
2309 case Intrinsic::riscv_vsuxseg6:
2310 case Intrinsic::riscv_vsuxseg7:
2311 case Intrinsic::riscv_vsuxseg8:
2312 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 4,
2313 /*IsStore*/ true,
2314 /*IsUnitStrided*/ false);
2315 return;
2316 case Intrinsic::riscv_vssseg2_mask:
2317 case Intrinsic::riscv_vssseg3_mask:
2318 case Intrinsic::riscv_vssseg4_mask:
2319 case Intrinsic::riscv_vssseg5_mask:
2320 case Intrinsic::riscv_vssseg6_mask:
2321 case Intrinsic::riscv_vssseg7_mask:
2322 case Intrinsic::riscv_vssseg8_mask:
2323 case Intrinsic::riscv_vsoxseg2_mask:
2324 case Intrinsic::riscv_vsoxseg3_mask:
2325 case Intrinsic::riscv_vsoxseg4_mask:
2326 case Intrinsic::riscv_vsoxseg5_mask:
2327 case Intrinsic::riscv_vsoxseg6_mask:
2328 case Intrinsic::riscv_vsoxseg7_mask:
2329 case Intrinsic::riscv_vsoxseg8_mask:
2330 case Intrinsic::riscv_vsuxseg2_mask:
2331 case Intrinsic::riscv_vsuxseg3_mask:
2332 case Intrinsic::riscv_vsuxseg4_mask:
2333 case Intrinsic::riscv_vsuxseg5_mask:
2334 case Intrinsic::riscv_vsuxseg6_mask:
2335 case Intrinsic::riscv_vsuxseg7_mask:
2336 case Intrinsic::riscv_vsuxseg8_mask:
2337 SetRVVLoadStoreInfo(/*PtrOp*/ I.arg_size() - 5,
2338 /*IsStore*/ true,
2339 /*IsUnitStrided*/ false);
2340 return;
2341 case Intrinsic::riscv_sf_vlte8:
2342 case Intrinsic::riscv_sf_vlte16:
2343 case Intrinsic::riscv_sf_vlte32:
2344 case Intrinsic::riscv_sf_vlte64:
2345 Info.opc = ISD::INTRINSIC_VOID;
2346 Info.ptrVal = I.getArgOperand(i: 1);
2347 switch (Intrinsic) {
2348 case Intrinsic::riscv_sf_vlte8:
2349 Info.memVT = MVT::i8;
2350 Info.align = Align(1);
2351 break;
2352 case Intrinsic::riscv_sf_vlte16:
2353 Info.memVT = MVT::i16;
2354 Info.align = Align(2);
2355 break;
2356 case Intrinsic::riscv_sf_vlte32:
2357 Info.memVT = MVT::i32;
2358 Info.align = Align(4);
2359 break;
2360 case Intrinsic::riscv_sf_vlte64:
2361 Info.memVT = MVT::i64;
2362 Info.align = Align(8);
2363 break;
2364 }
2365 Info.size = MemoryLocation::UnknownSize;
2366 Info.flags |= MachineMemOperand::MOLoad;
2367 Infos.push_back(Elt: Info);
2368 return;
2369 case Intrinsic::riscv_sf_vste8:
2370 case Intrinsic::riscv_sf_vste16:
2371 case Intrinsic::riscv_sf_vste32:
2372 case Intrinsic::riscv_sf_vste64:
2373 Info.opc = ISD::INTRINSIC_VOID;
2374 Info.ptrVal = I.getArgOperand(i: 1);
2375 switch (Intrinsic) {
2376 case Intrinsic::riscv_sf_vste8:
2377 Info.memVT = MVT::i8;
2378 Info.align = Align(1);
2379 break;
2380 case Intrinsic::riscv_sf_vste16:
2381 Info.memVT = MVT::i16;
2382 Info.align = Align(2);
2383 break;
2384 case Intrinsic::riscv_sf_vste32:
2385 Info.memVT = MVT::i32;
2386 Info.align = Align(4);
2387 break;
2388 case Intrinsic::riscv_sf_vste64:
2389 Info.memVT = MVT::i64;
2390 Info.align = Align(8);
2391 break;
2392 }
2393 Info.size = MemoryLocation::UnknownSize;
2394 Info.flags |= MachineMemOperand::MOStore;
2395 Infos.push_back(Elt: Info);
2396 return;
2397 }
2398}
2399
2400bool RISCVTargetLowering::isLegalAddressingMode(const DataLayout &DL,
2401 const AddrMode &AM, Type *Ty,
2402 unsigned AS,
2403 Instruction *I) const {
2404 // No global is ever allowed as a base.
2405 if (AM.BaseGV)
2406 return false;
2407
2408 // None of our addressing modes allows a scalable offset
2409 if (AM.ScalableOffset)
2410 return false;
2411
2412 // RVV instructions only support register addressing.
2413 if (Subtarget.hasVInstructions() && isa<VectorType>(Val: Ty))
2414 return AM.HasBaseReg && AM.Scale == 0 && !AM.BaseOffs;
2415
2416 // Require a 12-bit signed offset.
2417 if (!isInt<12>(x: AM.BaseOffs))
2418 return false;
2419
2420 switch (AM.Scale) {
2421 case 0: // "r+i" or just "i", depending on HasBaseReg.
2422 break;
2423 case 1:
2424 if (!AM.HasBaseReg) // allow "r+i".
2425 break;
2426 return false; // disallow "r+r" or "r+r+i".
2427 default:
2428 return false;
2429 }
2430
2431 return true;
2432}
2433
2434bool RISCVTargetLowering::isLegalICmpImmediate(int64_t Imm) const {
2435 return isInt<12>(x: Imm);
2436}
2437
2438bool RISCVTargetLowering::isLegalAddImmediate(int64_t Imm) const {
2439 return isInt<12>(x: Imm);
2440}
2441
2442// On RV32, 64-bit integers are split into their high and low parts and held
2443// in two different registers, so the trunc is free since the low register can
2444// just be used.
2445// FIXME: Should we consider i64->i32 free on RV64 to match the EVT version of
2446// isTruncateFree?
2447bool RISCVTargetLowering::isTruncateFree(Type *SrcTy, Type *DstTy) const {
2448 if (Subtarget.is64Bit() || !SrcTy->isIntegerTy() || !DstTy->isIntegerTy())
2449 return false;
2450 unsigned SrcBits = SrcTy->getPrimitiveSizeInBits();
2451 unsigned DestBits = DstTy->getPrimitiveSizeInBits();
2452 return (SrcBits == 64 && DestBits == 32);
2453}
2454
2455bool RISCVTargetLowering::isTruncateFree(EVT SrcVT, EVT DstVT) const {
2456 // We consider i64->i32 free on RV64 since we have good selection of W
2457 // instructions that make promoting operations back to i64 free in many cases.
2458 if (SrcVT.isVector() || DstVT.isVector() || !SrcVT.isInteger() ||
2459 !DstVT.isInteger())
2460 return false;
2461 unsigned SrcBits = SrcVT.getSizeInBits();
2462 unsigned DestBits = DstVT.getSizeInBits();
2463 return (SrcBits == 64 && DestBits == 32);
2464}
2465
2466bool RISCVTargetLowering::isTruncateFree(SDValue Val, EVT VT2) const {
2467 EVT SrcVT = Val.getValueType();
2468 // free truncate from vnsrl and vnsra
2469 if (Subtarget.hasVInstructions() &&
2470 (Val.getOpcode() == ISD::SRL || Val.getOpcode() == ISD::SRA) &&
2471 SrcVT.isVector() && VT2.isVector()) {
2472 unsigned SrcBits = SrcVT.getVectorElementType().getSizeInBits();
2473 unsigned DestBits = VT2.getVectorElementType().getSizeInBits();
2474 if (SrcBits == DestBits * 2) {
2475 return true;
2476 }
2477 }
2478 return TargetLowering::isTruncateFree(Val, VT2);
2479}
2480
2481bool RISCVTargetLowering::isZExtFree(SDValue Val, EVT VT2) const {
2482 // Zexts are free if they can be combined with a load.
2483 // Don't advertise i32->i64 zextload as being free for RV64. It interacts
2484 // poorly with type legalization of compares preferring sext.
2485 if (auto *LD = dyn_cast<LoadSDNode>(Val)) {
2486 EVT MemVT = LD->getMemoryVT();
2487 if ((MemVT == MVT::i8 || MemVT == MVT::i16) &&
2488 (LD->getExtensionType() == ISD::NON_EXTLOAD ||
2489 LD->getExtensionType() == ISD::ZEXTLOAD))
2490 return true;
2491 }
2492
2493 return TargetLowering::isZExtFree(Val, VT2);
2494}
2495
2496bool RISCVTargetLowering::isSExtCheaperThanZExt(EVT SrcVT, EVT DstVT) const {
2497 return Subtarget.is64Bit() && SrcVT == MVT::i32 && DstVT == MVT::i64;
2498}
2499
2500bool RISCVTargetLowering::signExtendConstant(const ConstantInt *CI) const {
2501 return Subtarget.is64Bit() && CI->getType()->isIntegerTy(Bitwidth: 32);
2502}
2503
2504bool RISCVTargetLowering::isCheapToSpeculateCttz(Type *Ty) const {
2505 return Subtarget.hasCTZLike();
2506}
2507
2508bool RISCVTargetLowering::isCheapToSpeculateCtlz(Type *Ty) const {
2509 return Subtarget.hasCLZLike();
2510}
2511
2512bool RISCVTargetLowering::isMaskAndCmp0FoldingBeneficial(
2513 const Instruction &AndI) const {
2514 // We expect to be able to match a bit extraction instruction if the Zbs
2515 // extension is supported and the mask is a power of two. However, we
2516 // conservatively return false if the mask would fit in an ANDI instruction,
2517 // on the basis that it's possible the sinking+duplication of the AND in
2518 // CodeGenPrepare triggered by this hook wouldn't decrease the instruction
2519 // count and would increase code size (e.g. ANDI+BNEZ => BEXTI+BNEZ).
2520 if (!Subtarget.hasBEXTILike())
2521 return false;
2522 ConstantInt *Mask = dyn_cast<ConstantInt>(Val: AndI.getOperand(i: 1));
2523 if (!Mask)
2524 return false;
2525 return !Mask->getValue().isSignedIntN(N: 12) && Mask->getValue().isPowerOf2();
2526}
2527
2528bool RISCVTargetLowering::hasAndNotCompare(SDValue Y) const {
2529 EVT VT = Y.getValueType();
2530
2531 if (VT.isVector())
2532 return false;
2533
2534 return (Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
2535 (!isa<ConstantSDNode>(Val: Y) || cast<ConstantSDNode>(Val&: Y)->isOpaque());
2536}
2537
2538bool RISCVTargetLowering::hasAndNot(SDValue Y) const {
2539 EVT VT = Y.getValueType();
2540
2541 if (!VT.isVector())
2542 return hasAndNotCompare(Y);
2543
2544 return Subtarget.hasStdExtZvkb();
2545}
2546
2547bool RISCVTargetLowering::hasBitTest(SDValue X, SDValue Y) const {
2548 // Zbs provides BEXT[_I], which can be used with SEQZ/SNEZ as a bit test.
2549 if (Subtarget.hasStdExtZbs())
2550 return X.getValueType().isScalarInteger();
2551 auto *C = dyn_cast<ConstantSDNode>(Val&: Y);
2552 // XTheadBs provides th.tst (similar to bexti), if Y is a constant
2553 if (Subtarget.hasVendorXTHeadBs())
2554 return C != nullptr;
2555 // We can use ANDI+SEQZ/SNEZ as a bit test. Y contains the bit position.
2556 return C && C->getAPIntValue().ule(RHS: 10);
2557}
2558
2559bool RISCVTargetLowering::shouldFoldSelectWithIdentityConstant(
2560 unsigned BinOpcode, EVT VT, unsigned SelectOpcode, SDValue X,
2561 SDValue Y) const {
2562 if (SelectOpcode != ISD::VSELECT)
2563 return false;
2564
2565 // Only enable for rvv.
2566 if (!VT.isVector() || !Subtarget.hasVInstructions())
2567 return false;
2568
2569 if (VT.isFixedLengthVector() && !isTypeLegal(VT))
2570 return false;
2571
2572 return true;
2573}
2574
2575bool RISCVTargetLowering::shouldConvertConstantLoadToIntImm(const APInt &Imm,
2576 Type *Ty) const {
2577 assert(Ty->isIntegerTy());
2578
2579 unsigned BitSize = Ty->getIntegerBitWidth();
2580 if (BitSize > Subtarget.getXLen())
2581 return false;
2582
2583 // Fast path, assume 32-bit immediates are cheap.
2584 int64_t Val = Imm.getSExtValue();
2585 if (isInt<32>(x: Val))
2586 return true;
2587
2588 // A constant pool entry may be more aligned than the load we're trying to
2589 // replace. If we don't support unaligned scalar mem, prefer the constant
2590 // pool.
2591 // TODO: Can the caller pass down the alignment?
2592 if (!Subtarget.enableUnalignedScalarMem())
2593 return true;
2594
2595 // Prefer to keep the load if it would require many instructions.
2596 // This uses the same threshold we use for constant pools but doesn't
2597 // check useConstantPoolForLargeInts.
2598 // TODO: Should we keep the load only when we're definitely going to emit a
2599 // constant pool?
2600
2601 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val, STI: Subtarget);
2602 return Seq.size() <= Subtarget.getMaxBuildIntsCost();
2603}
2604
2605bool RISCVTargetLowering::
2606 shouldProduceAndByConstByHoistingConstFromShiftsLHSOfAnd(
2607 SDValue X, ConstantSDNode *XC, ConstantSDNode *CC, SDValue Y,
2608 unsigned OldShiftOpcode, unsigned NewShiftOpcode,
2609 SelectionDAG &DAG) const {
2610 // One interesting pattern that we'd want to form is 'bit extract':
2611 // ((1 >> Y) & 1) ==/!= 0
2612 // But we also need to be careful not to try to reverse that fold.
2613
2614 // Is this '((1 >> Y) & 1)'?
2615 if (XC && OldShiftOpcode == ISD::SRL && XC->isOne())
2616 return false; // Keep the 'bit extract' pattern.
2617
2618 // Will this be '((1 >> Y) & 1)' after the transform?
2619 if (NewShiftOpcode == ISD::SRL && CC->isOne())
2620 return true; // Do form the 'bit extract' pattern.
2621
2622 // If 'X' is a constant, and we transform, then we will immediately
2623 // try to undo the fold, thus causing endless combine loop.
2624 // So only do the transform if X is not a constant. This matches the default
2625 // implementation of this function.
2626 return !XC;
2627}
2628
2629bool RISCVTargetLowering::shouldScalarizeBinop(SDValue VecOp) const {
2630 unsigned Opc = VecOp.getOpcode();
2631
2632 // Assume target opcodes can't be scalarized.
2633 // TODO - do we have any exceptions?
2634 if (Opc >= ISD::BUILTIN_OP_END || !isBinOp(Opcode: Opc))
2635 return false;
2636
2637 // If the vector op is not supported, try to convert to scalar.
2638 EVT VecVT = VecOp.getValueType();
2639 if (!isOperationLegalOrCustomOrPromote(Op: Opc, VT: VecVT))
2640 return true;
2641
2642 // If the vector op is supported, but the scalar op is not, the transform may
2643 // not be worthwhile.
2644 // Permit a vector binary operation can be converted to scalar binary
2645 // operation which is custom lowered with illegal type.
2646 EVT ScalarVT = VecVT.getScalarType();
2647 return isOperationLegalOrCustomOrPromote(Op: Opc, VT: ScalarVT) ||
2648 isOperationCustom(Op: Opc, VT: ScalarVT);
2649}
2650
2651bool RISCVTargetLowering::isOffsetFoldingLegal(
2652 const GlobalAddressSDNode *GA) const {
2653 // In order to maximise the opportunity for common subexpression elimination,
2654 // keep a separate ADD node for the global address offset instead of folding
2655 // it in the global address node. Later peephole optimisations may choose to
2656 // fold it back in when profitable.
2657 return false;
2658}
2659
2660// Returns 0-31 if the fli instruction is available for the type and this is
2661// legal FP immediate for the type. Returns -1 otherwise.
2662int RISCVTargetLowering::getLegalZfaFPImm(const APFloat &Imm, EVT VT) const {
2663 if (!Subtarget.hasStdExtZfa())
2664 return -1;
2665
2666 bool IsSupportedVT = false;
2667 if (VT == MVT::f16) {
2668 IsSupportedVT = Subtarget.hasStdExtZfh() || Subtarget.hasStdExtZvfh();
2669 } else if (VT == MVT::f32) {
2670 IsSupportedVT = true;
2671 } else if (VT == MVT::f64) {
2672 assert(Subtarget.hasStdExtD() && "Expect D extension");
2673 IsSupportedVT = true;
2674 }
2675
2676 if (!IsSupportedVT)
2677 return -1;
2678
2679 return RISCVLoadFPImm::getLoadFPImm(FPImm: Imm);
2680}
2681
2682bool RISCVTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT,
2683 bool ForCodeSize) const {
2684 bool IsLegalVT = false;
2685 if (VT == MVT::f16)
2686 IsLegalVT = Subtarget.hasStdExtZfhminOrZhinxmin();
2687 else if (VT == MVT::f32)
2688 IsLegalVT = Subtarget.hasStdExtFOrZfinx();
2689 else if (VT == MVT::f64)
2690 IsLegalVT = Subtarget.hasStdExtDOrZdinx();
2691 else if (VT == MVT::bf16)
2692 IsLegalVT = Subtarget.hasStdExtZfbfmin();
2693
2694 if (!IsLegalVT)
2695 return false;
2696
2697 if (getLegalZfaFPImm(Imm, VT) >= 0)
2698 return true;
2699
2700 // Some constants can be produced by fli+fneg.
2701 if (Imm.isNegative() && getLegalZfaFPImm(Imm: -Imm, VT) >= 0)
2702 return true;
2703
2704 // Cannot create a 64 bit floating-point immediate value for rv32.
2705 if (Subtarget.getXLen() < VT.getScalarSizeInBits()) {
2706 // td can handle +0.0 or -0.0 already.
2707 // -0.0 can be created by fmv + fneg.
2708 return Imm.isZero();
2709 }
2710
2711 // Special case: fmv + fneg
2712 if (Imm.isNegZero())
2713 return true;
2714
2715 // Building an integer and then converting requires a fmv at the end of
2716 // the integer sequence. The fmv is not required for Zfinx.
2717 const int FmvCost = Subtarget.hasStdExtZfinx() ? 0 : 1;
2718 const int Cost =
2719 FmvCost + RISCVMatInt::getIntMatCost(Val: Imm.bitcastToAPInt(),
2720 Size: Subtarget.getXLen(), STI: Subtarget);
2721 return Cost <= FPImmCost;
2722}
2723
2724// TODO: This is very conservative.
2725bool RISCVTargetLowering::isExtractSubvectorCheap(EVT ResVT, EVT SrcVT,
2726 unsigned Index) const {
2727 if (!isOperationLegalOrCustom(Op: ISD::EXTRACT_SUBVECTOR, VT: ResVT))
2728 return false;
2729
2730 // Extracts from index 0 are just subreg extracts.
2731 if (Index == 0)
2732 return true;
2733
2734 // Only support extracting a fixed from a fixed vector for now.
2735 if (ResVT.isScalableVector() || SrcVT.isScalableVector())
2736 return false;
2737
2738 EVT EltVT = ResVT.getVectorElementType();
2739 assert(EltVT == SrcVT.getVectorElementType() && "Should hold for node");
2740
2741 // The smallest type we can slide is i8.
2742 if (EltVT == MVT::i1)
2743 return false;
2744
2745 unsigned ResElts = ResVT.getVectorNumElements();
2746 unsigned SrcElts = SrcVT.getVectorNumElements();
2747
2748 unsigned MinVLen = Subtarget.getRealMinVLen();
2749 unsigned MinVLMAX = MinVLen / EltVT.getSizeInBits();
2750
2751 // If we're extracting only data from the first VLEN bits of the source
2752 // then we can always do this with an m1 vslidedown.vx. Restricting the
2753 // Index ensures we can use a vslidedown.vi.
2754 // TODO: We can generalize this when the exact VLEN is known.
2755 if (Index + ResElts <= MinVLMAX && Index < 31)
2756 return true;
2757
2758 // Convervatively only handle extracting half of a vector.
2759 // TODO: We can do arbitrary slidedowns, but for now only support extracting
2760 // the upper half of a vector until we have more test coverage.
2761 // TODO: For sizes which aren't multiples of VLEN sizes, this may not be
2762 // a cheap extract. However, this case is important in practice for
2763 // shuffled extracts of longer vectors. How resolve?
2764 return (ResElts * 2) == SrcElts && Index == ResElts;
2765}
2766
2767MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
2768 CallingConv::ID CC,
2769 EVT VT) const {
2770 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2771 // We might still end up using a GPR but that will be decided based on ABI.
2772 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2773 !Subtarget.hasStdExtZfhminOrZhinxmin())
2774 return MVT::f32;
2775
2776 return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
2777}
2778
2779unsigned
2780RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
2781 std::optional<MVT> RegisterVT) const {
2782 // Pair inline assembly operand
2783 if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
2784 *RegisterVT == MVT::Untyped)
2785 return 1;
2786
2787 return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
2788}
2789
2790unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
2791 CallingConv::ID CC,
2792 EVT VT) const {
2793 // Use f32 to pass f16 if it is legal and Zfh/Zfhmin is not enabled.
2794 // We might still end up using a GPR but that will be decided based on ABI.
2795 if (VT == MVT::f16 && Subtarget.hasStdExtFOrZfinx() &&
2796 !Subtarget.hasStdExtZfhminOrZhinxmin())
2797 return 1;
2798
2799 return TargetLowering::getNumRegistersForCallingConv(Context, CC, VT);
2800}
2801
2802// Changes the condition code and swaps operands if necessary, so the SetCC
2803// operation matches one of the comparisons supported directly by branches
2804// in the RISC-V ISA. May adjust compares to favor compare with 0 over compare
2805// with 1/-1.
2806static void translateSetCCForBranch(const SDLoc &DL, SDValue &LHS, SDValue &RHS,
2807 ISD::CondCode &CC, SelectionDAG &DAG,
2808 const RISCVSubtarget &Subtarget) {
2809 // If this is a single bit test that can't be handled by ANDI, shift the
2810 // bit to be tested to the MSB and perform a signed compare with 0.
2811 if (isIntEqualitySetCC(Code: CC) && isNullConstant(V: RHS) &&
2812 LHS.getOpcode() == ISD::AND && LHS.hasOneUse() &&
2813 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) &&
2814 // XAndesPerf supports branch on test bit.
2815 !Subtarget.hasVendorXAndesPerf()) {
2816 uint64_t Mask = LHS.getConstantOperandVal(i: 1);
2817 if ((isPowerOf2_64(Value: Mask) || isMask_64(Value: Mask)) && !isInt<12>(x: Mask)) {
2818 unsigned ShAmt = 0;
2819 if (isPowerOf2_64(Value: Mask)) {
2820 CC = CC == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
2821 ShAmt = LHS.getValueSizeInBits() - 1 - Log2_64(Value: Mask);
2822 } else {
2823 ShAmt = LHS.getValueSizeInBits() - llvm::bit_width(Value: Mask);
2824 }
2825
2826 LHS = LHS.getOperand(i: 0);
2827 if (ShAmt != 0)
2828 LHS = DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS,
2829 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
2830 return;
2831 }
2832 }
2833
2834 if (auto *RHSC = dyn_cast<ConstantSDNode>(Val&: RHS)) {
2835 int64_t C = RHSC->getSExtValue();
2836 switch (CC) {
2837 default: break;
2838 case ISD::SETGT:
2839 // Convert X > -1 to X >= 0.
2840 if (C == -1) {
2841 RHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
2842 CC = ISD::SETGE;
2843 return;
2844 }
2845 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
2846 C != INT64_MAX && isInt<5>(x: C + 1)) {
2847 // We have a conditional move instruction for SETGE but not SETGT.
2848 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
2849 RHS = DAG.getSignedConstant(Val: C + 1, DL, VT: RHS.getValueType());
2850 CC = ISD::SETGE;
2851 return;
2852 }
2853 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isInt<16>(x: C + 1)) {
2854 // We have a branch immediate instruction for SETGE but not SETGT.
2855 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit signed immediate.
2856 RHS = DAG.getSignedConstant(Val: C + 1, DL, VT: RHS.getValueType());
2857 CC = ISD::SETGE;
2858 return;
2859 }
2860 break;
2861 case ISD::SETLT:
2862 // Convert X < 1 to 0 >= X.
2863 if (C == 1) {
2864 RHS = LHS;
2865 LHS = DAG.getConstant(Val: 0, DL, VT: RHS.getValueType());
2866 CC = ISD::SETGE;
2867 return;
2868 }
2869 break;
2870 case ISD::SETUGT:
2871 if ((Subtarget.hasVendorXqcicm() || Subtarget.hasVendorXqcicli()) &&
2872 C != INT64_MAX && isUInt<5>(x: C + 1)) {
2873 // We have a conditional move instruction for SETUGE but not SETUGT.
2874 // Convert X > C to X >= C + 1, if (C + 1) is a 5-bit signed immediate.
2875 RHS = DAG.getConstant(Val: C + 1, DL, VT: RHS.getValueType());
2876 CC = ISD::SETUGE;
2877 return;
2878 }
2879 if (Subtarget.hasVendorXqcibi() && C != INT64_MAX && isUInt<16>(x: C + 1)) {
2880 // We have a branch immediate instruction for SETUGE but not SETUGT.
2881 // Convert X > C to X >= C + 1, if (C + 1) is a 16-bit unsigned
2882 // immediate.
2883 RHS = DAG.getConstant(Val: C + 1, DL, VT: RHS.getValueType());
2884 CC = ISD::SETUGE;
2885 return;
2886 }
2887 break;
2888 }
2889 }
2890
2891 switch (CC) {
2892 default:
2893 break;
2894 case ISD::SETGT:
2895 case ISD::SETLE:
2896 case ISD::SETUGT:
2897 case ISD::SETULE:
2898 CC = ISD::getSetCCSwappedOperands(Operation: CC);
2899 std::swap(a&: LHS, b&: RHS);
2900 break;
2901 }
2902}
2903
2904RISCVVType::VLMUL RISCVTargetLowering::getLMUL(MVT VT) {
2905 if (VT.isRISCVVectorTuple()) {
2906 if (VT.SimpleTy >= MVT::riscv_nxv1i8x2 &&
2907 VT.SimpleTy <= MVT::riscv_nxv1i8x8)
2908 return RISCVVType::LMUL_F8;
2909 if (VT.SimpleTy >= MVT::riscv_nxv2i8x2 &&
2910 VT.SimpleTy <= MVT::riscv_nxv2i8x8)
2911 return RISCVVType::LMUL_F4;
2912 if (VT.SimpleTy >= MVT::riscv_nxv4i8x2 &&
2913 VT.SimpleTy <= MVT::riscv_nxv4i8x8)
2914 return RISCVVType::LMUL_F2;
2915 if (VT.SimpleTy >= MVT::riscv_nxv8i8x2 &&
2916 VT.SimpleTy <= MVT::riscv_nxv8i8x8)
2917 return RISCVVType::LMUL_1;
2918 if (VT.SimpleTy >= MVT::riscv_nxv16i8x2 &&
2919 VT.SimpleTy <= MVT::riscv_nxv16i8x4)
2920 return RISCVVType::LMUL_2;
2921 if (VT.SimpleTy == MVT::riscv_nxv32i8x2)
2922 return RISCVVType::LMUL_4;
2923 llvm_unreachable("Invalid vector tuple type LMUL.");
2924 }
2925
2926 assert(VT.isScalableVector() && "Expecting a scalable vector type");
2927 unsigned KnownSize = VT.getSizeInBits().getKnownMinValue();
2928 if (VT.getVectorElementType() == MVT::i1)
2929 KnownSize *= 8;
2930
2931 switch (KnownSize) {
2932 default:
2933 llvm_unreachable("Invalid LMUL.");
2934 case 8:
2935 return RISCVVType::LMUL_F8;
2936 case 16:
2937 return RISCVVType::LMUL_F4;
2938 case 32:
2939 return RISCVVType::LMUL_F2;
2940 case 64:
2941 return RISCVVType::LMUL_1;
2942 case 128:
2943 return RISCVVType::LMUL_2;
2944 case 256:
2945 return RISCVVType::LMUL_4;
2946 case 512:
2947 return RISCVVType::LMUL_8;
2948 }
2949}
2950
2951unsigned RISCVTargetLowering::getRegClassIDForLMUL(RISCVVType::VLMUL LMul) {
2952 switch (LMul) {
2953 default:
2954 llvm_unreachable("Invalid LMUL.");
2955 case RISCVVType::LMUL_F8:
2956 case RISCVVType::LMUL_F4:
2957 case RISCVVType::LMUL_F2:
2958 case RISCVVType::LMUL_1:
2959 return RISCV::VRRegClassID;
2960 case RISCVVType::LMUL_2:
2961 return RISCV::VRM2RegClassID;
2962 case RISCVVType::LMUL_4:
2963 return RISCV::VRM4RegClassID;
2964 case RISCVVType::LMUL_8:
2965 return RISCV::VRM8RegClassID;
2966 }
2967}
2968
2969unsigned RISCVTargetLowering::getSubregIndexByMVT(MVT VT, unsigned Index) {
2970 RISCVVType::VLMUL LMUL = getLMUL(VT);
2971 if (LMUL == RISCVVType::LMUL_F8 || LMUL == RISCVVType::LMUL_F4 ||
2972 LMUL == RISCVVType::LMUL_F2 || LMUL == RISCVVType::LMUL_1) {
2973 static_assert(RISCV::sub_vrm1_7 == RISCV::sub_vrm1_0 + 7,
2974 "Unexpected subreg numbering");
2975 return RISCV::sub_vrm1_0 + Index;
2976 }
2977 if (LMUL == RISCVVType::LMUL_2) {
2978 static_assert(RISCV::sub_vrm2_3 == RISCV::sub_vrm2_0 + 3,
2979 "Unexpected subreg numbering");
2980 return RISCV::sub_vrm2_0 + Index;
2981 }
2982 if (LMUL == RISCVVType::LMUL_4) {
2983 static_assert(RISCV::sub_vrm4_1 == RISCV::sub_vrm4_0 + 1,
2984 "Unexpected subreg numbering");
2985 return RISCV::sub_vrm4_0 + Index;
2986 }
2987 llvm_unreachable("Invalid vector type.");
2988}
2989
2990unsigned RISCVTargetLowering::getRegClassIDForVecVT(MVT VT) {
2991 if (VT.isRISCVVectorTuple()) {
2992 unsigned NF = VT.getRISCVVectorTupleNumFields();
2993 unsigned RegsPerField =
2994 std::max(a: 1U, b: (unsigned)VT.getSizeInBits().getKnownMinValue() /
2995 (NF * RISCV::RVVBitsPerBlock));
2996 switch (RegsPerField) {
2997 case 1:
2998 if (NF == 2)
2999 return RISCV::VRN2M1RegClassID;
3000 if (NF == 3)
3001 return RISCV::VRN3M1RegClassID;
3002 if (NF == 4)
3003 return RISCV::VRN4M1RegClassID;
3004 if (NF == 5)
3005 return RISCV::VRN5M1RegClassID;
3006 if (NF == 6)
3007 return RISCV::VRN6M1RegClassID;
3008 if (NF == 7)
3009 return RISCV::VRN7M1RegClassID;
3010 if (NF == 8)
3011 return RISCV::VRN8M1RegClassID;
3012 break;
3013 case 2:
3014 if (NF == 2)
3015 return RISCV::VRN2M2RegClassID;
3016 if (NF == 3)
3017 return RISCV::VRN3M2RegClassID;
3018 if (NF == 4)
3019 return RISCV::VRN4M2RegClassID;
3020 break;
3021 case 4:
3022 assert(NF == 2);
3023 return RISCV::VRN2M4RegClassID;
3024 default:
3025 break;
3026 }
3027 llvm_unreachable("Invalid vector tuple type RegClass.");
3028 }
3029
3030 if (VT.getVectorElementType() == MVT::i1)
3031 return RISCV::VRRegClassID;
3032 return getRegClassIDForLMUL(LMul: getLMUL(VT));
3033}
3034
3035// Attempt to decompose a subvector insert/extract between VecVT and
3036// SubVecVT via subregister indices. Returns the subregister index that
3037// can perform the subvector insert/extract with the given element index, as
3038// well as the index corresponding to any leftover subvectors that must be
3039// further inserted/extracted within the register class for SubVecVT.
3040std::pair<unsigned, unsigned>
3041RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
3042 MVT VecVT, MVT SubVecVT, unsigned InsertExtractIdx,
3043 const RISCVRegisterInfo *TRI) {
3044 static_assert((RISCV::VRM8RegClassID > RISCV::VRM4RegClassID &&
3045 RISCV::VRM4RegClassID > RISCV::VRM2RegClassID &&
3046 RISCV::VRM2RegClassID > RISCV::VRRegClassID),
3047 "Register classes not ordered");
3048 unsigned VecRegClassID = getRegClassIDForVecVT(VT: VecVT);
3049 unsigned SubRegClassID = getRegClassIDForVecVT(VT: SubVecVT);
3050
3051 // If VecVT is a vector tuple type, either it's the tuple type with same
3052 // RegClass with SubVecVT or SubVecVT is a actually a subvector of the VecVT.
3053 if (VecVT.isRISCVVectorTuple()) {
3054 if (VecRegClassID == SubRegClassID)
3055 return {RISCV::NoSubRegister, 0};
3056
3057 assert(SubVecVT.isScalableVector() &&
3058 "Only allow scalable vector subvector.");
3059 assert(getLMUL(VecVT) == getLMUL(SubVecVT) &&
3060 "Invalid vector tuple insert/extract for vector and subvector with "
3061 "different LMUL.");
3062 return {getSubregIndexByMVT(VT: VecVT, Index: InsertExtractIdx), 0};
3063 }
3064
3065 // Try to compose a subregister index that takes us from the incoming
3066 // LMUL>1 register class down to the outgoing one. At each step we half
3067 // the LMUL:
3068 // nxv16i32@12 -> nxv2i32: sub_vrm4_1_then_sub_vrm2_1_then_sub_vrm1_0
3069 // Note that this is not guaranteed to find a subregister index, such as
3070 // when we are extracting from one VR type to another.
3071 unsigned SubRegIdx = RISCV::NoSubRegister;
3072 for (const unsigned RCID :
3073 {RISCV::VRM4RegClassID, RISCV::VRM2RegClassID, RISCV::VRRegClassID})
3074 if (VecRegClassID > RCID && SubRegClassID <= RCID) {
3075 VecVT = VecVT.getHalfNumVectorElementsVT();
3076 bool IsHi =
3077 InsertExtractIdx >= VecVT.getVectorElementCount().getKnownMinValue();
3078 SubRegIdx = TRI->composeSubRegIndices(a: SubRegIdx,
3079 b: getSubregIndexByMVT(VT: VecVT, Index: IsHi));
3080 if (IsHi)
3081 InsertExtractIdx -= VecVT.getVectorElementCount().getKnownMinValue();
3082 }
3083 return {SubRegIdx, InsertExtractIdx};
3084}
3085
3086// Permit combining of mask vectors as BUILD_VECTOR never expands to scalar
3087// stores for those types.
3088bool RISCVTargetLowering::mergeStoresAfterLegalization(EVT VT) const {
3089 return !Subtarget.useRVVForFixedLengthVectors() ||
3090 (VT.isFixedLengthVector() && VT.getVectorElementType() == MVT::i1);
3091}
3092
3093bool RISCVTargetLowering::isLegalElementTypeForRVV(EVT ScalarTy) const {
3094 if (!ScalarTy.isSimple())
3095 return false;
3096 switch (ScalarTy.getSimpleVT().SimpleTy) {
3097 case MVT::iPTR:
3098 return Subtarget.is64Bit() ? Subtarget.hasVInstructionsI64() : true;
3099 case MVT::i8:
3100 case MVT::i16:
3101 case MVT::i32:
3102 return Subtarget.hasVInstructions();
3103 case MVT::i64:
3104 return Subtarget.hasVInstructionsI64();
3105 case MVT::f16:
3106 return Subtarget.hasVInstructionsF16Minimal();
3107 case MVT::bf16:
3108 return Subtarget.hasVInstructionsBF16Minimal();
3109 case MVT::f32:
3110 return Subtarget.hasVInstructionsF32();
3111 case MVT::f64:
3112 return Subtarget.hasVInstructionsF64();
3113 default:
3114 return false;
3115 }
3116}
3117
3118
3119unsigned RISCVTargetLowering::combineRepeatedFPDivisors() const {
3120 return NumRepeatedDivisors;
3121}
3122
3123static SDValue getVLOperand(SDValue Op) {
3124 assert((Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
3125 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
3126 "Unexpected opcode");
3127 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
3128 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
3129 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
3130 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
3131 if (!II)
3132 return SDValue();
3133 return Op.getOperand(i: II->VLOperand + 1 + HasChain);
3134}
3135
3136static bool useRVVForFixedLengthVectorVT(MVT VT,
3137 const RISCVSubtarget &Subtarget) {
3138 assert(VT.isFixedLengthVector() && "Expected a fixed length vector type!");
3139 if (!Subtarget.useRVVForFixedLengthVectors())
3140 return false;
3141
3142 // We only support a set of vector types with a consistent maximum fixed size
3143 // across all supported vector element types to avoid legalization issues.
3144 // Therefore -- since the largest is v1024i8/v512i16/etc -- the largest
3145 // fixed-length vector type we support is 1024 bytes.
3146 if (VT.getVectorNumElements() > 1024 || VT.getFixedSizeInBits() > 1024 * 8)
3147 return false;
3148
3149 unsigned MinVLen = Subtarget.getRealMinVLen();
3150
3151 MVT EltVT = VT.getVectorElementType();
3152
3153 // Don't use RVV for vectors we cannot scalarize if required.
3154 switch (EltVT.SimpleTy) {
3155 // i1 is supported but has different rules.
3156 default:
3157 return false;
3158 case MVT::i1:
3159 // Masks can only use a single register.
3160 if (VT.getVectorNumElements() > MinVLen)
3161 return false;
3162 MinVLen /= 8;
3163 break;
3164 case MVT::i8:
3165 case MVT::i16:
3166 case MVT::i32:
3167 break;
3168 case MVT::i64:
3169 if (!Subtarget.hasVInstructionsI64())
3170 return false;
3171 break;
3172 case MVT::f16:
3173 if (!Subtarget.hasVInstructionsF16Minimal())
3174 return false;
3175 break;
3176 case MVT::bf16:
3177 if (!Subtarget.hasVInstructionsBF16Minimal())
3178 return false;
3179 break;
3180 case MVT::f32:
3181 if (!Subtarget.hasVInstructionsF32())
3182 return false;
3183 break;
3184 case MVT::f64:
3185 if (!Subtarget.hasVInstructionsF64())
3186 return false;
3187 break;
3188 }
3189
3190 // Reject elements larger than ELEN.
3191 if (EltVT.getSizeInBits() > Subtarget.getELen())
3192 return false;
3193
3194 unsigned LMul = divideCeil(Numerator: VT.getSizeInBits(), Denominator: MinVLen);
3195 // Don't use RVV for types that don't fit.
3196 if (LMul > Subtarget.getMaxLMULForFixedLengthVectors())
3197 return false;
3198
3199 // TODO: Perhaps an artificial restriction, but worth having whilst getting
3200 // the base fixed length RVV support in place.
3201 if (!VT.isPow2VectorType())
3202 return false;
3203
3204 return true;
3205}
3206
3207bool RISCVTargetLowering::useRVVForFixedLengthVectorVT(MVT VT) const {
3208 return ::useRVVForFixedLengthVectorVT(VT, Subtarget);
3209}
3210
3211// Return the largest legal scalable vector type that matches VT's element type.
3212static MVT getContainerForFixedLengthVector(const TargetLowering &TLI, MVT VT,
3213 const RISCVSubtarget &Subtarget) {
3214 // This may be called before legal types are setup.
3215 assert(((VT.isFixedLengthVector() && TLI.isTypeLegal(VT)) ||
3216 useRVVForFixedLengthVectorVT(VT, Subtarget)) &&
3217 "Expected legal fixed length vector!");
3218
3219 unsigned MinVLen = Subtarget.getRealMinVLen();
3220 unsigned MaxELen = Subtarget.getELen();
3221
3222 MVT EltVT = VT.getVectorElementType();
3223 switch (EltVT.SimpleTy) {
3224 default:
3225 llvm_unreachable("unexpected element type for RVV container");
3226 case MVT::i1:
3227 case MVT::i8:
3228 case MVT::i16:
3229 case MVT::i32:
3230 case MVT::i64:
3231 case MVT::bf16:
3232 case MVT::f16:
3233 case MVT::f32:
3234 case MVT::f64: {
3235 // We prefer to use LMUL=1 for VLEN sized types. Use fractional lmuls for
3236 // narrower types. The smallest fractional LMUL we support is 8/ELEN. Within
3237 // each fractional LMUL we support SEW between 8 and LMUL*ELEN.
3238 unsigned NumElts =
3239 (VT.getVectorNumElements() * RISCV::RVVBitsPerBlock) / MinVLen;
3240 NumElts = std::max(a: NumElts, b: RISCV::RVVBitsPerBlock / MaxELen);
3241 assert(isPowerOf2_32(NumElts) && "Expected power of 2 NumElts");
3242 return MVT::getScalableVectorVT(VT: EltVT, NumElements: NumElts);
3243 }
3244 }
3245}
3246
3247static MVT getContainerForFixedLengthVector(SelectionDAG &DAG, MVT VT,
3248 const RISCVSubtarget &Subtarget) {
3249 return getContainerForFixedLengthVector(TLI: DAG.getTargetLoweringInfo(), VT,
3250 Subtarget);
3251}
3252
3253MVT RISCVTargetLowering::getContainerForFixedLengthVector(MVT VT) const {
3254 return ::getContainerForFixedLengthVector(TLI: *this, VT, Subtarget: getSubtarget());
3255}
3256
3257// Grow V to consume an entire RVV register.
3258static SDValue convertToScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3259 const RISCVSubtarget &Subtarget) {
3260 assert(VT.isScalableVector() &&
3261 "Expected to convert into a scalable vector!");
3262 assert(V.getValueType().isFixedLengthVector() &&
3263 "Expected a fixed length vector operand!");
3264 SDLoc DL(V);
3265 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: V, Idx: 0);
3266}
3267
3268// Shrink V so it's just big enough to maintain a VT's worth of data.
3269static SDValue convertFromScalableVector(EVT VT, SDValue V, SelectionDAG &DAG,
3270 const RISCVSubtarget &Subtarget) {
3271 assert(VT.isFixedLengthVector() &&
3272 "Expected to convert into a fixed length vector!");
3273 assert(V.getValueType().isScalableVector() &&
3274 "Expected a scalable vector operand!");
3275 SDLoc DL(V);
3276 return DAG.getExtractSubvector(DL, VT, Vec: V, Idx: 0);
3277}
3278
3279/// Return the type of the mask type suitable for masking the provided
3280/// vector type. This is simply an i1 element type vector of the same
3281/// (possibly scalable) length.
3282static MVT getMaskTypeFor(MVT VecVT) {
3283 assert(VecVT.isVector());
3284 ElementCount EC = VecVT.getVectorElementCount();
3285 return MVT::getVectorVT(VT: MVT::i1, EC);
3286}
3287
3288/// Creates an all ones mask suitable for masking a vector of type VecTy with
3289/// vector length VL. .
3290static SDValue getAllOnesMask(MVT VecVT, SDValue VL, const SDLoc &DL,
3291 SelectionDAG &DAG) {
3292 MVT MaskVT = getMaskTypeFor(VecVT);
3293 return DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: MaskVT, Operand: VL);
3294}
3295
3296static std::pair<SDValue, SDValue>
3297getDefaultScalableVLOps(MVT VecVT, const SDLoc &DL, SelectionDAG &DAG,
3298 const RISCVSubtarget &Subtarget) {
3299 assert(VecVT.isScalableVector() && "Expecting a scalable vector");
3300 SDValue VL = DAG.getRegister(Reg: RISCV::X0, VT: Subtarget.getXLenVT());
3301 SDValue Mask = getAllOnesMask(VecVT, VL, DL, DAG);
3302 return {Mask, VL};
3303}
3304
3305static std::pair<SDValue, SDValue>
3306getDefaultVLOps(uint64_t NumElts, MVT ContainerVT, const SDLoc &DL,
3307 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
3308 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3309 SDValue VL = DAG.getConstant(Val: NumElts, DL, VT: Subtarget.getXLenVT());
3310 SDValue Mask = getAllOnesMask(VecVT: ContainerVT, VL, DL, DAG);
3311 return {Mask, VL};
3312}
3313
3314// Gets the two common "VL" operands: an all-ones mask and the vector length.
3315// VecVT is a vector type, either fixed-length or scalable, and ContainerVT is
3316// the vector type that the fixed-length vector is contained in. Otherwise if
3317// VecVT is scalable, then ContainerVT should be the same as VecVT.
3318static std::pair<SDValue, SDValue>
3319getDefaultVLOps(MVT VecVT, MVT ContainerVT, const SDLoc &DL, SelectionDAG &DAG,
3320 const RISCVSubtarget &Subtarget) {
3321 if (VecVT.isFixedLengthVector())
3322 return getDefaultVLOps(NumElts: VecVT.getVectorNumElements(), ContainerVT, DL, DAG,
3323 Subtarget);
3324 assert(ContainerVT.isScalableVector() && "Expecting scalable container type");
3325 return getDefaultScalableVLOps(VecVT: ContainerVT, DL, DAG, Subtarget);
3326}
3327
3328SDValue RISCVTargetLowering::computeVLMax(MVT VecVT, const SDLoc &DL,
3329 SelectionDAG &DAG) const {
3330 assert(VecVT.isScalableVector() && "Expected scalable vector");
3331 return DAG.getElementCount(DL, VT: Subtarget.getXLenVT(),
3332 EC: VecVT.getVectorElementCount());
3333}
3334
3335std::pair<unsigned, unsigned>
3336RISCVTargetLowering::computeVLMAXBounds(MVT VecVT,
3337 const RISCVSubtarget &Subtarget) {
3338 assert(VecVT.isScalableVector() && "Expected scalable vector");
3339
3340 unsigned EltSize = VecVT.getScalarSizeInBits();
3341 unsigned MinSize = VecVT.getSizeInBits().getKnownMinValue();
3342
3343 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
3344 unsigned MaxVLMAX =
3345 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
3346
3347 unsigned VectorBitsMin = Subtarget.getRealMinVLen();
3348 unsigned MinVLMAX =
3349 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMin, EltSize, MinSize);
3350
3351 return std::make_pair(x&: MinVLMAX, y&: MaxVLMAX);
3352}
3353
3354// The state of RVV BUILD_VECTOR and VECTOR_SHUFFLE lowering is that very few
3355// of either is (currently) supported. This can get us into an infinite loop
3356// where we try to lower a BUILD_VECTOR as a VECTOR_SHUFFLE as a BUILD_VECTOR
3357// as a ..., etc.
3358// Until either (or both) of these can reliably lower any node, reporting that
3359// we don't want to expand BUILD_VECTORs via VECTOR_SHUFFLEs at least breaks
3360// the infinite loop. Note that this lowers BUILD_VECTOR through the stack,
3361// which is not desirable.
3362bool RISCVTargetLowering::shouldExpandBuildVectorWithShuffles(
3363 EVT VT, unsigned DefinedValues) const {
3364 return false;
3365}
3366
3367InstructionCost RISCVTargetLowering::getLMULCost(MVT VT) const {
3368 // TODO: Here assume reciprocal throughput is 1 for LMUL_1, it is
3369 // implementation-defined.
3370 if (!VT.isVector())
3371 return InstructionCost::getInvalid();
3372 unsigned DLenFactor = Subtarget.getDLenFactor();
3373 unsigned Cost;
3374 if (VT.isScalableVector()) {
3375 unsigned LMul;
3376 bool Fractional;
3377 std::tie(args&: LMul, args&: Fractional) =
3378 RISCVVType::decodeVLMUL(VLMul: RISCVTargetLowering::getLMUL(VT));
3379 if (Fractional)
3380 Cost = LMul <= DLenFactor ? (DLenFactor / LMul) : 1;
3381 else
3382 Cost = (LMul * DLenFactor);
3383 } else {
3384 Cost = divideCeil(Numerator: VT.getSizeInBits(), Denominator: Subtarget.getRealMinVLen() / DLenFactor);
3385 }
3386 return Cost;
3387}
3388
3389
3390/// Return the cost of a vrgather.vv instruction for the type VT. vrgather.vv
3391/// may be quadratic in the number of vreg implied by LMUL, and is assumed to
3392/// be by default. VRGatherCostModel reflects available options. Note that
3393/// operand (index and possibly mask) are handled separately.
3394InstructionCost RISCVTargetLowering::getVRGatherVVCost(MVT VT) const {
3395 auto LMULCost = getLMULCost(VT);
3396 bool Log2CostModel =
3397 Subtarget.getVRGatherCostModel() == llvm::RISCVSubtarget::NLog2N;
3398 if (Log2CostModel && LMULCost.isValid()) {
3399 unsigned Log = Log2_64(Value: LMULCost.getValue());
3400 if (Log > 0)
3401 return LMULCost * Log;
3402 }
3403 return LMULCost * LMULCost;
3404}
3405
3406/// Return the cost of a vrgather.vi (or vx) instruction for the type VT.
3407/// vrgather.vi/vx may be linear in the number of vregs implied by LMUL,
3408/// or may track the vrgather.vv cost. It is implementation-dependent.
3409InstructionCost RISCVTargetLowering::getVRGatherVICost(MVT VT) const {
3410 return getLMULCost(VT);
3411}
3412
3413/// Return the cost of a vslidedown.vx or vslideup.vx instruction
3414/// for the type VT. (This does not cover the vslide1up or vslide1down
3415/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3416/// or may track the vrgather.vv cost. It is implementation-dependent.
3417InstructionCost RISCVTargetLowering::getVSlideVXCost(MVT VT) const {
3418 return getLMULCost(VT);
3419}
3420
3421/// Return the cost of a vslidedown.vi or vslideup.vi instruction
3422/// for the type VT. (This does not cover the vslide1up or vslide1down
3423/// variants.) Slides may be linear in the number of vregs implied by LMUL,
3424/// or may track the vrgather.vv cost. It is implementation-dependent.
3425InstructionCost RISCVTargetLowering::getVSlideVICost(MVT VT) const {
3426 return getLMULCost(VT);
3427}
3428
3429static SDValue lowerINT_TO_FP(SDValue Op, SelectionDAG &DAG,
3430 const RISCVSubtarget &Subtarget) {
3431 // f16 conversions are promoted to f32 when Zfh/Zhinx are not supported.
3432 // bf16 conversions are always promoted to f32.
3433 if ((Op.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3434 Op.getValueType() == MVT::bf16) {
3435 bool IsStrict = Op->isStrictFPOpcode();
3436
3437 SDLoc DL(Op);
3438 if (IsStrict) {
3439 SDValue Val = DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {MVT::f32, MVT::Other},
3440 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
3441 return DAG.getNode(Opcode: ISD::STRICT_FP_ROUND, DL,
3442 ResultTys: {Op.getValueType(), MVT::Other},
3443 Ops: {Val.getValue(R: 1), Val.getValue(R: 0),
3444 DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true)});
3445 }
3446 return DAG.getNode(
3447 Opcode: ISD::FP_ROUND, DL, VT: Op.getValueType(),
3448 N1: DAG.getNode(Opcode: Op.getOpcode(), DL, VT: MVT::f32, Operand: Op.getOperand(i: 0)),
3449 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
3450 }
3451
3452 // Other operations are legal.
3453 return Op;
3454}
3455
3456static SDValue lowerFP_TO_INT_SAT(SDValue Op, SelectionDAG &DAG,
3457 const RISCVSubtarget &Subtarget) {
3458 // RISC-V FP-to-int conversions saturate to the destination register size, but
3459 // don't produce 0 for nan. We can use a conversion instruction and fix the
3460 // nan case with a compare and a select.
3461 SDValue Src = Op.getOperand(i: 0);
3462
3463 MVT DstVT = Op.getSimpleValueType();
3464 EVT SatVT = cast<VTSDNode>(Val: Op.getOperand(i: 1))->getVT();
3465
3466 bool IsSigned = Op.getOpcode() == ISD::FP_TO_SINT_SAT;
3467
3468 if (!DstVT.isVector()) {
3469 // For bf16 or for f16 in absence of Zfh, promote to f32, then saturate
3470 // the result.
3471 if ((Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3472 Src.getValueType() == MVT::bf16) {
3473 Src = DAG.getNode(Opcode: ISD::FP_EXTEND, DL: SDLoc(Op), VT: MVT::f32, Operand: Src);
3474 }
3475
3476 unsigned Opc;
3477 if (SatVT == DstVT)
3478 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
3479 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
3480 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
3481 else
3482 return SDValue();
3483 // FIXME: Support other SatVTs by clamping before or after the conversion.
3484
3485 SDLoc DL(Op);
3486 SDValue FpToInt = DAG.getNode(
3487 Opcode: Opc, DL, VT: DstVT, N1: Src,
3488 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: Subtarget.getXLenVT()));
3489
3490 if (Opc == RISCVISD::FCVT_WU_RV64)
3491 FpToInt = DAG.getZeroExtendInReg(Op: FpToInt, DL, VT: MVT::i32);
3492
3493 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: DstVT);
3494 return DAG.getSelectCC(DL, LHS: Src, RHS: Src, True: ZeroInt, False: FpToInt,
3495 Cond: ISD::CondCode::SETUO);
3496 }
3497
3498 // Vectors.
3499
3500 MVT DstEltVT = DstVT.getVectorElementType();
3501 MVT SrcVT = Src.getSimpleValueType();
3502 MVT SrcEltVT = SrcVT.getVectorElementType();
3503 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
3504 unsigned DstEltSize = DstEltVT.getSizeInBits();
3505
3506 // Only handle saturating to the destination type.
3507 if (SatVT != DstEltVT)
3508 return SDValue();
3509
3510 MVT DstContainerVT = DstVT;
3511 MVT SrcContainerVT = SrcVT;
3512 if (DstVT.isFixedLengthVector()) {
3513 DstContainerVT = getContainerForFixedLengthVector(DAG, VT: DstVT, Subtarget);
3514 SrcContainerVT = getContainerForFixedLengthVector(DAG, VT: SrcVT, Subtarget);
3515 assert(DstContainerVT.getVectorElementCount() ==
3516 SrcContainerVT.getVectorElementCount() &&
3517 "Expected same element count");
3518 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
3519 }
3520
3521 SDLoc DL(Op);
3522
3523 auto [Mask, VL] = getDefaultVLOps(VecVT: DstVT, ContainerVT: DstContainerVT, DL, DAG, Subtarget);
3524
3525 SDValue IsNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
3526 Ops: {Src, Src, DAG.getCondCode(Cond: ISD::SETNE),
3527 DAG.getUNDEF(VT: Mask.getValueType()), Mask, VL});
3528
3529 // Need to widen by more than 1 step, promote the FP type, then do a widening
3530 // convert.
3531 if (DstEltSize > (2 * SrcEltSize)) {
3532 assert(SrcContainerVT.getVectorElementType() == MVT::f16 && "Unexpected VT!");
3533 MVT InterVT = SrcContainerVT.changeVectorElementType(EltVT: MVT::f32);
3534 Src = DAG.getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT: InterVT, N1: Src, N2: Mask, N3: VL);
3535 }
3536
3537 MVT CvtContainerVT = DstContainerVT;
3538 MVT CvtEltVT = DstEltVT;
3539 if (SrcEltSize > (2 * DstEltSize)) {
3540 CvtEltVT = MVT::getIntegerVT(BitWidth: SrcEltVT.getSizeInBits() / 2);
3541 CvtContainerVT = CvtContainerVT.changeVectorElementType(EltVT: CvtEltVT);
3542 }
3543
3544 unsigned RVVOpc =
3545 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
3546 SDValue Res = DAG.getNode(Opcode: RVVOpc, DL, VT: CvtContainerVT, N1: Src, N2: Mask, N3: VL);
3547
3548 while (CvtContainerVT != DstContainerVT) {
3549 CvtEltVT = MVT::getIntegerVT(BitWidth: CvtEltVT.getSizeInBits() / 2);
3550 CvtContainerVT = CvtContainerVT.changeVectorElementType(EltVT: CvtEltVT);
3551 // Rounding mode here is arbitrary since we aren't shifting out any bits.
3552 unsigned ClipOpc = IsSigned ? RISCVISD::TRUNCATE_VECTOR_VL_SSAT
3553 : RISCVISD::TRUNCATE_VECTOR_VL_USAT;
3554 Res = DAG.getNode(Opcode: ClipOpc, DL, VT: CvtContainerVT, N1: Res, N2: Mask, N3: VL);
3555 }
3556
3557 SDValue SplatZero = DAG.getNode(
3558 Opcode: RISCVISD::VMV_V_X_VL, DL, VT: DstContainerVT, N1: DAG.getUNDEF(VT: DstContainerVT),
3559 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()), N3: VL);
3560 Res = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: DstContainerVT, N1: IsNan, N2: SplatZero,
3561 N3: Res, N4: DAG.getUNDEF(VT: DstContainerVT), N5: VL);
3562
3563 if (DstVT.isFixedLengthVector())
3564 Res = convertFromScalableVector(VT: DstVT, V: Res, DAG, Subtarget);
3565
3566 return Res;
3567}
3568
3569static SDValue lowerFP_TO_INT(SDValue Op, SelectionDAG &DAG,
3570 const RISCVSubtarget &Subtarget) {
3571 bool IsStrict = Op->isStrictFPOpcode();
3572 SDValue SrcVal = Op.getOperand(i: IsStrict ? 1 : 0);
3573
3574 // f16 conversions are promoted to f32 when Zfh/Zhinx is not enabled.
3575 // bf16 conversions are always promoted to f32.
3576 if ((SrcVal.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx()) ||
3577 SrcVal.getValueType() == MVT::bf16) {
3578 SDLoc DL(Op);
3579 if (IsStrict) {
3580 SDValue Ext =
3581 DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
3582 Ops: {Op.getOperand(i: 0), SrcVal});
3583 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
3584 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
3585 }
3586 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
3587 Operand: DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: SrcVal));
3588 }
3589
3590 // Other operations are legal.
3591 return Op;
3592}
3593
3594static RISCVFPRndMode::RoundingMode matchRoundingOp(unsigned Opc) {
3595 switch (Opc) {
3596 case ISD::FROUNDEVEN:
3597 case ISD::STRICT_FROUNDEVEN:
3598 case ISD::VP_FROUNDEVEN:
3599 return RISCVFPRndMode::RNE;
3600 case ISD::FTRUNC:
3601 case ISD::STRICT_FTRUNC:
3602 case ISD::VP_FROUNDTOZERO:
3603 return RISCVFPRndMode::RTZ;
3604 case ISD::FFLOOR:
3605 case ISD::STRICT_FFLOOR:
3606 case ISD::VP_FFLOOR:
3607 return RISCVFPRndMode::RDN;
3608 case ISD::FCEIL:
3609 case ISD::STRICT_FCEIL:
3610 case ISD::VP_FCEIL:
3611 return RISCVFPRndMode::RUP;
3612 case ISD::FROUND:
3613 case ISD::LROUND:
3614 case ISD::LLROUND:
3615 case ISD::STRICT_FROUND:
3616 case ISD::STRICT_LROUND:
3617 case ISD::STRICT_LLROUND:
3618 case ISD::VP_FROUND:
3619 return RISCVFPRndMode::RMM;
3620 case ISD::FRINT:
3621 case ISD::LRINT:
3622 case ISD::LLRINT:
3623 case ISD::STRICT_FRINT:
3624 case ISD::STRICT_LRINT:
3625 case ISD::STRICT_LLRINT:
3626 case ISD::VP_FRINT:
3627 case ISD::VP_LRINT:
3628 case ISD::VP_LLRINT:
3629 return RISCVFPRndMode::DYN;
3630 }
3631
3632 return RISCVFPRndMode::Invalid;
3633}
3634
3635// Expand vector FTRUNC, FCEIL, FFLOOR, FROUND, VP_FCEIL, VP_FFLOOR, VP_FROUND
3636// VP_FROUNDEVEN, VP_FROUNDTOZERO, VP_FRINT and VP_FNEARBYINT by converting to
3637// the integer domain and back. Taking care to avoid converting values that are
3638// nan or already correct.
3639static SDValue
3640lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3641 const RISCVSubtarget &Subtarget) {
3642 MVT VT = Op.getSimpleValueType();
3643 assert(VT.isVector() && "Unexpected type");
3644
3645 SDLoc DL(Op);
3646
3647 SDValue Src = Op.getOperand(i: 0);
3648
3649 // Freeze the source since we are increasing the number of uses.
3650 Src = DAG.getFreeze(V: Src);
3651
3652 MVT ContainerVT = VT;
3653 if (VT.isFixedLengthVector()) {
3654 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3655 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
3656 }
3657
3658 SDValue Mask, VL;
3659 if (Op->isVPOpcode()) {
3660 Mask = Op.getOperand(i: 1);
3661 if (VT.isFixedLengthVector())
3662 Mask = convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Mask, DAG,
3663 Subtarget);
3664 VL = Op.getOperand(i: 2);
3665 } else {
3666 std::tie(args&: Mask, args&: VL) = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
3667 }
3668
3669 // We do the conversion on the absolute value and fix the sign at the end.
3670 SDValue Abs = DAG.getNode(Opcode: RISCVISD::FABS_VL, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
3671
3672 // Determine the largest integer that can be represented exactly. This and
3673 // values larger than it don't have any fractional bits so don't need to
3674 // be converted.
3675 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3676 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3677 APFloat MaxVal = APFloat(FltSem);
3678 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3679 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3680 SDValue MaxValNode =
3681 DAG.getConstantFP(Val: MaxVal, DL, VT: ContainerVT.getVectorElementType());
3682 SDValue MaxValSplat = DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: ContainerVT,
3683 N1: DAG.getUNDEF(VT: ContainerVT), N2: MaxValNode, N3: VL);
3684
3685 // If abs(Src) was larger than MaxVal or nan, keep it.
3686 MVT SetccVT = MVT::getVectorVT(VT: MVT::i1, EC: ContainerVT.getVectorElementCount());
3687 Mask =
3688 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: SetccVT,
3689 Ops: {Abs, MaxValSplat, DAG.getCondCode(Cond: ISD::SETOLT),
3690 Mask, Mask, VL});
3691
3692 // Truncate to integer and convert back to FP.
3693 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3694 MVT XLenVT = Subtarget.getXLenVT();
3695 SDValue Truncated;
3696
3697 switch (Op.getOpcode()) {
3698 default:
3699 llvm_unreachable("Unexpected opcode");
3700 case ISD::FRINT:
3701 case ISD::VP_FRINT:
3702 case ISD::FCEIL:
3703 case ISD::VP_FCEIL:
3704 case ISD::FFLOOR:
3705 case ISD::VP_FFLOOR:
3706 case ISD::FROUND:
3707 case ISD::FROUNDEVEN:
3708 case ISD::VP_FROUND:
3709 case ISD::VP_FROUNDEVEN:
3710 case ISD::VP_FROUNDTOZERO: {
3711 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3712 assert(FRM != RISCVFPRndMode::Invalid);
3713 Truncated = DAG.getNode(Opcode: RISCVISD::VFCVT_RM_X_F_VL, DL, VT: IntVT, N1: Src, N2: Mask,
3714 N3: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), N4: VL);
3715 break;
3716 }
3717 case ISD::FTRUNC:
3718 Truncated = DAG.getNode(Opcode: RISCVISD::VFCVT_RTZ_X_F_VL, DL, VT: IntVT, N1: Src,
3719 N2: Mask, N3: VL);
3720 break;
3721 case ISD::FNEARBYINT:
3722 case ISD::VP_FNEARBYINT:
3723 Truncated = DAG.getNode(Opcode: RISCVISD::VFROUND_NOEXCEPT_VL, DL, VT: ContainerVT, N1: Src,
3724 N2: Mask, N3: VL);
3725 break;
3726 }
3727
3728 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3729 if (Truncated.getOpcode() != RISCVISD::VFROUND_NOEXCEPT_VL)
3730 Truncated = DAG.getNode(Opcode: RISCVISD::SINT_TO_FP_VL, DL, VT: ContainerVT, N1: Truncated,
3731 N2: Mask, N3: VL);
3732
3733 // Restore the original sign so that -0.0 is preserved.
3734 Truncated = DAG.getNode(Opcode: RISCVISD::FCOPYSIGN_VL, DL, VT: ContainerVT, N1: Truncated,
3735 N2: Src, N3: Src, N4: Mask, N5: VL);
3736
3737 if (!VT.isFixedLengthVector())
3738 return Truncated;
3739
3740 return convertFromScalableVector(VT, V: Truncated, DAG, Subtarget);
3741}
3742
3743// Expand vector STRICT_FTRUNC, STRICT_FCEIL, STRICT_FFLOOR, STRICT_FROUND
3744// STRICT_FROUNDEVEN and STRICT_FNEARBYINT by converting sNan of the source to
3745// qNan and converting the new source to integer and back to FP.
3746static SDValue
3747lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3748 const RISCVSubtarget &Subtarget) {
3749 SDLoc DL(Op);
3750 MVT VT = Op.getSimpleValueType();
3751 SDValue Chain = Op.getOperand(i: 0);
3752 SDValue Src = Op.getOperand(i: 1);
3753
3754 MVT ContainerVT = VT;
3755 if (VT.isFixedLengthVector()) {
3756 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
3757 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
3758 }
3759
3760 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
3761
3762 // Freeze the source since we are increasing the number of uses.
3763 Src = DAG.getFreeze(V: Src);
3764
3765 // Convert sNan to qNan by executing x + x for all unordered element x in Src.
3766 MVT MaskVT = Mask.getSimpleValueType();
3767 SDValue Unorder = DAG.getNode(Opcode: RISCVISD::STRICT_FSETCC_VL, DL,
3768 VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
3769 Ops: {Chain, Src, Src, DAG.getCondCode(Cond: ISD::SETUNE),
3770 DAG.getUNDEF(VT: MaskVT), Mask, VL});
3771 Chain = Unorder.getValue(R: 1);
3772 Src = DAG.getNode(Opcode: RISCVISD::STRICT_FADD_VL, DL,
3773 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
3774 Ops: {Chain, Src, Src, Src, Unorder, VL});
3775 Chain = Src.getValue(R: 1);
3776
3777 // We do the conversion on the absolute value and fix the sign at the end.
3778 SDValue Abs = DAG.getNode(Opcode: RISCVISD::FABS_VL, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
3779
3780 // Determine the largest integer that can be represented exactly. This and
3781 // values larger than it don't have any fractional bits so don't need to
3782 // be converted.
3783 const fltSemantics &FltSem = ContainerVT.getFltSemantics();
3784 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3785 APFloat MaxVal = APFloat(FltSem);
3786 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3787 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3788 SDValue MaxValNode =
3789 DAG.getConstantFP(Val: MaxVal, DL, VT: ContainerVT.getVectorElementType());
3790 SDValue MaxValSplat = DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: ContainerVT,
3791 N1: DAG.getUNDEF(VT: ContainerVT), N2: MaxValNode, N3: VL);
3792
3793 // If abs(Src) was larger than MaxVal or nan, keep it.
3794 Mask = DAG.getNode(
3795 Opcode: RISCVISD::SETCC_VL, DL, VT: MaskVT,
3796 Ops: {Abs, MaxValSplat, DAG.getCondCode(Cond: ISD::SETOLT), Mask, Mask, VL});
3797
3798 // Truncate to integer and convert back to FP.
3799 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
3800 MVT XLenVT = Subtarget.getXLenVT();
3801 SDValue Truncated;
3802
3803 switch (Op.getOpcode()) {
3804 default:
3805 llvm_unreachable("Unexpected opcode");
3806 case ISD::STRICT_FCEIL:
3807 case ISD::STRICT_FFLOOR:
3808 case ISD::STRICT_FROUND:
3809 case ISD::STRICT_FROUNDEVEN: {
3810 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3811 assert(FRM != RISCVFPRndMode::Invalid);
3812 Truncated = DAG.getNode(
3813 Opcode: RISCVISD::STRICT_VFCVT_RM_X_F_VL, DL, VTList: DAG.getVTList(VT1: IntVT, VT2: MVT::Other),
3814 Ops: {Chain, Src, Mask, DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), VL});
3815 break;
3816 }
3817 case ISD::STRICT_FTRUNC:
3818 Truncated =
3819 DAG.getNode(Opcode: RISCVISD::STRICT_VFCVT_RTZ_X_F_VL, DL,
3820 VTList: DAG.getVTList(VT1: IntVT, VT2: MVT::Other), N1: Chain, N2: Src, N3: Mask, N4: VL);
3821 break;
3822 case ISD::STRICT_FNEARBYINT:
3823 Truncated = DAG.getNode(Opcode: RISCVISD::STRICT_VFROUND_NOEXCEPT_VL, DL,
3824 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), N1: Chain, N2: Src,
3825 N3: Mask, N4: VL);
3826 break;
3827 }
3828 Chain = Truncated.getValue(R: 1);
3829
3830 // VFROUND_NOEXCEPT_VL includes SINT_TO_FP_VL.
3831 if (Op.getOpcode() != ISD::STRICT_FNEARBYINT) {
3832 Truncated = DAG.getNode(Opcode: RISCVISD::STRICT_SINT_TO_FP_VL, DL,
3833 VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), N1: Chain,
3834 N2: Truncated, N3: Mask, N4: VL);
3835 Chain = Truncated.getValue(R: 1);
3836 }
3837
3838 // Restore the original sign so that -0.0 is preserved.
3839 Truncated = DAG.getNode(Opcode: RISCVISD::FCOPYSIGN_VL, DL, VT: ContainerVT, N1: Truncated,
3840 N2: Src, N3: Src, N4: Mask, N5: VL);
3841
3842 if (VT.isFixedLengthVector())
3843 Truncated = convertFromScalableVector(VT, V: Truncated, DAG, Subtarget);
3844 return DAG.getMergeValues(Ops: {Truncated, Chain}, dl: DL);
3845}
3846
3847static SDValue
3848lowerFTRUNC_FCEIL_FFLOOR_FROUND(SDValue Op, SelectionDAG &DAG,
3849 const RISCVSubtarget &Subtarget) {
3850 MVT VT = Op.getSimpleValueType();
3851 if (VT.isVector())
3852 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
3853
3854 if (DAG.shouldOptForSize())
3855 return SDValue();
3856
3857 SDLoc DL(Op);
3858 SDValue Src = Op.getOperand(i: 0);
3859
3860 // Create an integer the size of the mantissa with the MSB set. This and all
3861 // values larger than it don't have any fractional bits so don't need to be
3862 // converted.
3863 const fltSemantics &FltSem = VT.getFltSemantics();
3864 unsigned Precision = APFloat::semanticsPrecision(FltSem);
3865 APFloat MaxVal = APFloat(FltSem);
3866 MaxVal.convertFromAPInt(Input: APInt::getOneBitSet(numBits: Precision, BitNo: Precision - 1),
3867 /*IsSigned*/ false, RM: APFloat::rmNearestTiesToEven);
3868 SDValue MaxValNode = DAG.getConstantFP(Val: MaxVal, DL, VT);
3869
3870 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Op.getOpcode());
3871 return DAG.getNode(Opcode: RISCVISD::FROUND, DL, VT, N1: Src, N2: MaxValNode,
3872 N3: DAG.getTargetConstant(Val: FRM, DL, VT: Subtarget.getXLenVT()));
3873}
3874
3875// Expand vector [L]LRINT and [L]LROUND by converting to the integer domain.
3876static SDValue lowerVectorXRINT_XROUND(SDValue Op, SelectionDAG &DAG,
3877 const RISCVSubtarget &Subtarget) {
3878 SDLoc DL(Op);
3879 MVT DstVT = Op.getSimpleValueType();
3880 SDValue Src = Op.getOperand(i: 0);
3881 MVT SrcVT = Src.getSimpleValueType();
3882 assert(SrcVT.isVector() && DstVT.isVector() &&
3883 !(SrcVT.isFixedLengthVector() ^ DstVT.isFixedLengthVector()) &&
3884 "Unexpected type");
3885
3886 MVT DstContainerVT = DstVT;
3887 MVT SrcContainerVT = SrcVT;
3888
3889 if (DstVT.isFixedLengthVector()) {
3890 DstContainerVT = getContainerForFixedLengthVector(DAG, VT: DstVT, Subtarget);
3891 SrcContainerVT = getContainerForFixedLengthVector(DAG, VT: SrcVT, Subtarget);
3892 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
3893 }
3894
3895 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT: SrcContainerVT, DL, DAG, Subtarget);
3896
3897 // [b]f16 -> f32
3898 MVT SrcElemType = SrcVT.getVectorElementType();
3899 if (SrcElemType == MVT::f16 || SrcElemType == MVT::bf16) {
3900 MVT F32VT = SrcContainerVT.changeVectorElementType(EltVT: MVT::f32);
3901 Src = DAG.getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT: F32VT, N1: Src, N2: Mask, N3: VL);
3902 }
3903
3904 SDValue Res =
3905 DAG.getNode(Opcode: RISCVISD::VFCVT_RM_X_F_VL, DL, VT: DstContainerVT, N1: Src, N2: Mask,
3906 N3: DAG.getTargetConstant(Val: matchRoundingOp(Opc: Op.getOpcode()), DL,
3907 VT: Subtarget.getXLenVT()),
3908 N4: VL);
3909
3910 if (!DstVT.isFixedLengthVector())
3911 return Res;
3912
3913 return convertFromScalableVector(VT: DstVT, V: Res, DAG, Subtarget);
3914}
3915
3916static SDValue
3917getVSlidedown(SelectionDAG &DAG, const RISCVSubtarget &Subtarget,
3918 const SDLoc &DL, EVT VT, SDValue Passthru, SDValue Op,
3919 SDValue Offset, SDValue Mask, SDValue VL,
3920 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
3921 if (Passthru.isUndef())
3922 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
3923 SDValue PolicyOp = DAG.getTargetConstant(Val: Policy, DL, VT: Subtarget.getXLenVT());
3924 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
3925 return DAG.getNode(Opcode: RISCVISD::VSLIDEDOWN_VL, DL, VT, Ops);
3926}
3927
3928static SDValue
3929getVSlideup(SelectionDAG &DAG, const RISCVSubtarget &Subtarget, const SDLoc &DL,
3930 EVT VT, SDValue Passthru, SDValue Op, SDValue Offset, SDValue Mask,
3931 SDValue VL,
3932 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED) {
3933 if (Passthru.isUndef())
3934 Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
3935 SDValue PolicyOp = DAG.getTargetConstant(Val: Policy, DL, VT: Subtarget.getXLenVT());
3936 SDValue Ops[] = {Passthru, Op, Offset, Mask, VL, PolicyOp};
3937 return DAG.getNode(Opcode: RISCVISD::VSLIDEUP_VL, DL, VT, Ops);
3938}
3939
3940struct VIDSequence {
3941 int64_t StepNumerator;
3942 unsigned StepDenominator;
3943 int64_t Addend;
3944};
3945
3946static std::optional<APInt> getExactInteger(const APFloat &APF,
3947 uint32_t BitWidth) {
3948 // We will use a SINT_TO_FP to materialize this constant so we should use a
3949 // signed APSInt here.
3950 APSInt ValInt(BitWidth, /*IsUnsigned*/ false);
3951 // We use an arbitrary rounding mode here. If a floating-point is an exact
3952 // integer (e.g., 1.0), the rounding mode does not affect the output value. If
3953 // the rounding mode changes the output value, then it is not an exact
3954 // integer.
3955 RoundingMode ArbitraryRM = RoundingMode::TowardZero;
3956 bool IsExact;
3957 // If it is out of signed integer range, it will return an invalid operation.
3958 // If it is not an exact integer, IsExact is false.
3959 if ((APF.convertToInteger(Result&: ValInt, RM: ArbitraryRM, IsExact: &IsExact) ==
3960 APFloatBase::opInvalidOp) ||
3961 !IsExact)
3962 return std::nullopt;
3963 return ValInt.extractBits(numBits: BitWidth, bitPosition: 0);
3964}
3965
3966// Try to match an arithmetic-sequence BUILD_VECTOR [X,X+S,X+2*S,...,X+(N-1)*S]
3967// to the (non-zero) step S and start value X. This can be then lowered as the
3968// RVV sequence (VID * S) + X, for example.
3969// The step S is represented as an integer numerator divided by a positive
3970// denominator. Note that the implementation currently only identifies
3971// sequences in which either the numerator is +/- 1 or the denominator is 1. It
3972// cannot detect 2/3, for example.
3973// Note that this method will also match potentially unappealing index
3974// sequences, like <i32 0, i32 50939494>, however it is left to the caller to
3975// determine whether this is worth generating code for.
3976//
3977// EltSizeInBits is the size of the type that the sequence will be calculated
3978// in, i.e. SEW for build_vectors or XLEN for address calculations.
3979static std::optional<VIDSequence> isSimpleVIDSequence(SDValue Op,
3980 unsigned EltSizeInBits) {
3981 assert(Op.getOpcode() == ISD::BUILD_VECTOR && "Unexpected BUILD_VECTOR");
3982 if (!cast<BuildVectorSDNode>(Val&: Op)->isConstant())
3983 return std::nullopt;
3984 bool IsInteger = Op.getValueType().isInteger();
3985
3986 std::optional<unsigned> SeqStepDenom;
3987 std::optional<APInt> SeqStepNum;
3988 std::optional<APInt> SeqAddend;
3989 std::optional<std::pair<APInt, unsigned>> PrevElt;
3990 assert(EltSizeInBits >= Op.getValueType().getScalarSizeInBits());
3991
3992 // First extract the ops into a list of constant integer values. This may not
3993 // be possible for floats if they're not all representable as integers.
3994 SmallVector<std::optional<APInt>> Elts(Op.getNumOperands());
3995 const unsigned OpSize = Op.getScalarValueSizeInBits();
3996 for (auto [Idx, Elt] : enumerate(First: Op->op_values())) {
3997 if (Elt.isUndef()) {
3998 Elts[Idx] = std::nullopt;
3999 continue;
4000 }
4001 if (IsInteger) {
4002 Elts[Idx] = Elt->getAsAPIntVal().trunc(width: OpSize).zext(width: EltSizeInBits);
4003 } else {
4004 auto ExactInteger =
4005 getExactInteger(APF: cast<ConstantFPSDNode>(Val: Elt)->getValueAPF(), BitWidth: OpSize);
4006 if (!ExactInteger)
4007 return std::nullopt;
4008 Elts[Idx] = *ExactInteger;
4009 }
4010 }
4011
4012 for (auto [Idx, Elt] : enumerate(First&: Elts)) {
4013 // Assume undef elements match the sequence; we just have to be careful
4014 // when interpolating across them.
4015 if (!Elt)
4016 continue;
4017
4018 if (PrevElt) {
4019 // Calculate the step since the last non-undef element, and ensure
4020 // it's consistent across the entire sequence.
4021 unsigned IdxDiff = Idx - PrevElt->second;
4022 APInt ValDiff = *Elt - PrevElt->first;
4023
4024 // A zero-value value difference means that we're somewhere in the middle
4025 // of a fractional step, e.g. <0,0,0*,0,1,1,1,1>. Wait until we notice a
4026 // step change before evaluating the sequence.
4027 if (ValDiff == 0)
4028 continue;
4029
4030 int64_t Remainder = ValDiff.srem(RHS: IdxDiff);
4031 // Normalize the step if it's greater than 1.
4032 if (Remainder != ValDiff.getSExtValue()) {
4033 // The difference must cleanly divide the element span.
4034 if (Remainder != 0)
4035 return std::nullopt;
4036 ValDiff = ValDiff.sdiv(RHS: IdxDiff);
4037 IdxDiff = 1;
4038 }
4039
4040 if (!SeqStepNum)
4041 SeqStepNum = ValDiff;
4042 else if (ValDiff != SeqStepNum)
4043 return std::nullopt;
4044
4045 if (!SeqStepDenom)
4046 SeqStepDenom = IdxDiff;
4047 else if (IdxDiff != *SeqStepDenom)
4048 return std::nullopt;
4049 }
4050
4051 // Record this non-undef element for later.
4052 if (!PrevElt || PrevElt->first != *Elt)
4053 PrevElt = std::make_pair(x&: *Elt, y&: Idx);
4054 }
4055
4056 // We need to have logged a step for this to count as a legal index sequence.
4057 if (!SeqStepNum || !SeqStepDenom)
4058 return std::nullopt;
4059
4060 // Loop back through the sequence and validate elements we might have skipped
4061 // while waiting for a valid step. While doing this, log any sequence addend.
4062 for (auto [Idx, Elt] : enumerate(First&: Elts)) {
4063 if (!Elt)
4064 continue;
4065 APInt ExpectedVal =
4066 (APInt(EltSizeInBits, Idx, /*isSigned=*/false, /*implicitTrunc=*/true) *
4067 *SeqStepNum)
4068 .sdiv(RHS: *SeqStepDenom);
4069
4070 APInt Addend = *Elt - ExpectedVal;
4071 if (!SeqAddend)
4072 SeqAddend = Addend;
4073 else if (Addend != SeqAddend)
4074 return std::nullopt;
4075 }
4076
4077 assert(SeqAddend && "Must have an addend if we have a step");
4078
4079 return VIDSequence{.StepNumerator: SeqStepNum->getSExtValue(), .StepDenominator: *SeqStepDenom,
4080 .Addend: SeqAddend->getSExtValue()};
4081}
4082
4083// Match a splatted value (SPLAT_VECTOR/BUILD_VECTOR) of an EXTRACT_VECTOR_ELT
4084// and lower it as a VRGATHER_VX_VL from the source vector.
4085static SDValue matchSplatAsGather(SDValue SplatVal, MVT VT, const SDLoc &DL,
4086 SelectionDAG &DAG,
4087 const RISCVSubtarget &Subtarget) {
4088 if (SplatVal.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
4089 return SDValue();
4090 SDValue Src = SplatVal.getOperand(i: 0);
4091 // Don't perform this optimization for i1 vectors, or if the element types are
4092 // different
4093 // FIXME: Support i1 vectors, maybe by promoting to i8?
4094 MVT EltTy = VT.getVectorElementType();
4095 if (EltTy == MVT::i1 ||
4096 !DAG.getTargetLoweringInfo().isTypeLegal(VT: Src.getValueType()))
4097 return SDValue();
4098 MVT SrcVT = Src.getSimpleValueType();
4099 if (EltTy != SrcVT.getVectorElementType())
4100 return SDValue();
4101 SDValue Idx = SplatVal.getOperand(i: 1);
4102 // The index must be a legal type.
4103 if (Idx.getValueType() != Subtarget.getXLenVT())
4104 return SDValue();
4105
4106 // Check that we know Idx lies within VT
4107 if (!TypeSize::isKnownLE(LHS: SrcVT.getSizeInBits(), RHS: VT.getSizeInBits())) {
4108 auto *CIdx = dyn_cast<ConstantSDNode>(Val&: Idx);
4109 if (!CIdx || CIdx->getZExtValue() >= VT.getVectorMinNumElements())
4110 return SDValue();
4111 }
4112
4113 // Convert fixed length vectors to scalable
4114 MVT ContainerVT = VT;
4115 if (VT.isFixedLengthVector())
4116 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4117
4118 MVT SrcContainerVT = SrcVT;
4119 if (SrcVT.isFixedLengthVector()) {
4120 SrcContainerVT = getContainerForFixedLengthVector(DAG, VT: SrcVT, Subtarget);
4121 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
4122 }
4123
4124 // Put Vec in a VT sized vector
4125 if (SrcContainerVT.getVectorMinNumElements() <
4126 ContainerVT.getVectorMinNumElements())
4127 Src = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec: Src, Idx: 0);
4128 else
4129 Src = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec: Src, Idx: 0);
4130
4131 // We checked that Idx fits inside VT earlier
4132 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4133 SDValue Gather = DAG.getNode(Opcode: RISCVISD::VRGATHER_VX_VL, DL, VT: ContainerVT, N1: Src,
4134 N2: Idx, N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
4135 if (VT.isFixedLengthVector())
4136 Gather = convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
4137 return Gather;
4138}
4139
4140static SDValue lowerBuildVectorViaVID(SDValue Op, SelectionDAG &DAG,
4141 const RISCVSubtarget &Subtarget) {
4142 MVT VT = Op.getSimpleValueType();
4143 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4144
4145 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4146
4147 SDLoc DL(Op);
4148 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4149
4150 if (auto SimpleVID = isSimpleVIDSequence(Op, EltSizeInBits: Op.getScalarValueSizeInBits())) {
4151 int64_t StepNumerator = SimpleVID->StepNumerator;
4152 unsigned StepDenominator = SimpleVID->StepDenominator;
4153 int64_t Addend = SimpleVID->Addend;
4154
4155 assert(StepNumerator != 0 && "Invalid step");
4156 bool Negate = false;
4157 int64_t SplatStepVal = StepNumerator;
4158 unsigned StepOpcode = ISD::MUL;
4159 // Exclude INT64_MIN to avoid passing it to std::abs. We won't optimize it
4160 // anyway as the shift of 63 won't fit in uimm5.
4161 if (StepNumerator != 1 && StepNumerator != INT64_MIN &&
4162 isPowerOf2_64(Value: std::abs(i: StepNumerator))) {
4163 Negate = StepNumerator < 0;
4164 StepOpcode = ISD::SHL;
4165 SplatStepVal = Log2_64(Value: std::abs(i: StepNumerator));
4166 }
4167
4168 // Only emit VIDs with suitably-small steps. We use imm5 as a threshold
4169 // since it's the immediate value many RVV instructions accept. There is
4170 // no vmul.vi instruction so ensure multiply constant can fit in a
4171 // single addi instruction. For the addend, we allow up to 32 bits..
4172 if (((StepOpcode == ISD::MUL && isInt<12>(x: SplatStepVal)) ||
4173 (StepOpcode == ISD::SHL && isUInt<5>(x: SplatStepVal))) &&
4174 isPowerOf2_32(Value: StepDenominator) &&
4175 (SplatStepVal >= 0 || StepDenominator == 1) && isInt<32>(x: Addend)) {
4176 MVT VIDVT =
4177 VT.isFloatingPoint() ? VT.changeVectorElementTypeToInteger() : VT;
4178 MVT VIDContainerVT =
4179 getContainerForFixedLengthVector(DAG, VT: VIDVT, Subtarget);
4180 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: VIDContainerVT, N1: Mask, N2: VL);
4181 // Convert right out of the scalable type so we can use standard ISD
4182 // nodes for the rest of the computation. If we used scalable types with
4183 // these, we'd lose the fixed-length vector info and generate worse
4184 // vsetvli code.
4185 VID = convertFromScalableVector(VT: VIDVT, V: VID, DAG, Subtarget);
4186 if ((StepOpcode == ISD::MUL && SplatStepVal != 1) ||
4187 (StepOpcode == ISD::SHL && SplatStepVal != 0)) {
4188 SDValue SplatStep = DAG.getSignedConstant(Val: SplatStepVal, DL, VT: VIDVT);
4189 VID = DAG.getNode(Opcode: StepOpcode, DL, VT: VIDVT, N1: VID, N2: SplatStep);
4190 }
4191 if (StepDenominator != 1) {
4192 SDValue SplatStep =
4193 DAG.getConstant(Val: Log2_64(Value: StepDenominator), DL, VT: VIDVT);
4194 VID = DAG.getNode(Opcode: ISD::SRL, DL, VT: VIDVT, N1: VID, N2: SplatStep);
4195 }
4196 if (Addend != 0 || Negate) {
4197 SDValue SplatAddend = DAG.getSignedConstant(Val: Addend, DL, VT: VIDVT);
4198 VID = DAG.getNode(Opcode: Negate ? ISD::SUB : ISD::ADD, DL, VT: VIDVT, N1: SplatAddend,
4199 N2: VID);
4200 }
4201 if (VT.isFloatingPoint()) {
4202 // TODO: Use vfwcvt to reduce register pressure.
4203 VID = DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: VID);
4204 }
4205 return VID;
4206 }
4207 }
4208
4209 return SDValue();
4210}
4211
4212/// Try and optimize BUILD_VECTORs with "dominant values" - these are values
4213/// which constitute a large proportion of the elements. In such cases we can
4214/// splat a vector with the dominant element and make up the shortfall with
4215/// INSERT_VECTOR_ELTs. Returns SDValue if not profitable.
4216/// Note that this includes vectors of 2 elements by association. The
4217/// upper-most element is the "dominant" one, allowing us to use a splat to
4218/// "insert" the upper element, and an insert of the lower element at position
4219/// 0, which improves codegen.
4220static SDValue lowerBuildVectorViaDominantValues(SDValue Op, SelectionDAG &DAG,
4221 const RISCVSubtarget &Subtarget) {
4222 MVT VT = Op.getSimpleValueType();
4223 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4224
4225 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4226
4227 SDLoc DL(Op);
4228 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4229
4230 MVT XLenVT = Subtarget.getXLenVT();
4231 unsigned NumElts = Op.getNumOperands();
4232
4233 SDValue DominantValue;
4234 unsigned MostCommonCount = 0;
4235 DenseMap<SDValue, unsigned> ValueCounts;
4236 unsigned NumUndefElts =
4237 count_if(Range: Op->op_values(), P: [](const SDValue &V) { return V.isUndef(); });
4238
4239 // Track the number of scalar loads we know we'd be inserting, estimated as
4240 // any non-zero floating-point constant. Other kinds of element are either
4241 // already in registers or are materialized on demand. The threshold at which
4242 // a vector load is more desirable than several scalar materializion and
4243 // vector-insertion instructions is not known.
4244 unsigned NumScalarLoads = 0;
4245
4246 for (SDValue V : Op->op_values()) {
4247 if (V.isUndef())
4248 continue;
4249
4250 unsigned &Count = ValueCounts[V];
4251 if (0 == Count)
4252 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: V))
4253 NumScalarLoads += !CFP->isExactlyValue(V: +0.0);
4254
4255 // Is this value dominant? In case of a tie, prefer the highest element as
4256 // it's cheaper to insert near the beginning of a vector than it is at the
4257 // end.
4258 if (++Count >= MostCommonCount) {
4259 DominantValue = V;
4260 MostCommonCount = Count;
4261 }
4262 }
4263
4264 assert(DominantValue && "Not expecting an all-undef BUILD_VECTOR");
4265 unsigned NumDefElts = NumElts - NumUndefElts;
4266 unsigned DominantValueCountThreshold = NumDefElts <= 2 ? 0 : NumDefElts - 2;
4267
4268 // Don't perform this optimization when optimizing for size, since
4269 // materializing elements and inserting them tends to cause code bloat.
4270 if (!DAG.shouldOptForSize() && NumScalarLoads < NumElts &&
4271 (NumElts != 2 || ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode())) &&
4272 ((MostCommonCount > DominantValueCountThreshold) ||
4273 (ValueCounts.size() <= Log2_32(Value: NumDefElts)))) {
4274 // Start by splatting the most common element.
4275 SDValue Vec = DAG.getSplatBuildVector(VT, DL, Op: DominantValue);
4276
4277 DenseSet<SDValue> Processed{DominantValue};
4278
4279 // We can handle an insert into the last element (of a splat) via
4280 // v(f)slide1down. This is slightly better than the vslideup insert
4281 // lowering as it avoids the need for a vector group temporary. It
4282 // is also better than using vmerge.vx as it avoids the need to
4283 // materialize the mask in a vector register.
4284 if (SDValue LastOp = Op->getOperand(Num: Op->getNumOperands() - 1);
4285 !LastOp.isUndef() && ValueCounts[LastOp] == 1 &&
4286 LastOp != DominantValue) {
4287 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
4288 auto OpCode =
4289 VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL;
4290 if (!VT.isFloatingPoint())
4291 LastOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: LastOp);
4292 Vec = DAG.getNode(Opcode: OpCode, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Vec,
4293 N3: LastOp, N4: Mask, N5: VL);
4294 Vec = convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
4295 Processed.insert(V: LastOp);
4296 }
4297
4298 MVT SelMaskTy = VT.changeVectorElementType(EltVT: MVT::i1);
4299 for (const auto &OpIdx : enumerate(First: Op->ops())) {
4300 const SDValue &V = OpIdx.value();
4301 if (V.isUndef() || !Processed.insert(V).second)
4302 continue;
4303 if (ValueCounts[V] == 1) {
4304 Vec = DAG.getInsertVectorElt(DL, Vec, Elt: V, Idx: OpIdx.index());
4305 } else {
4306 // Blend in all instances of this value using a VSELECT, using a
4307 // mask where each bit signals whether that element is the one
4308 // we're after.
4309 SmallVector<SDValue> Ops;
4310 transform(Range: Op->op_values(), d_first: std::back_inserter(x&: Ops), F: [&](SDValue V1) {
4311 return DAG.getConstant(Val: V == V1, DL, VT: XLenVT);
4312 });
4313 Vec = DAG.getNode(Opcode: ISD::VSELECT, DL, VT,
4314 N1: DAG.getBuildVector(VT: SelMaskTy, DL, Ops),
4315 N2: DAG.getSplatBuildVector(VT, DL, Op: V), N3: Vec);
4316 }
4317 }
4318
4319 return Vec;
4320 }
4321
4322 return SDValue();
4323}
4324
4325static SDValue lowerBuildVectorOfConstants(SDValue Op, SelectionDAG &DAG,
4326 const RISCVSubtarget &Subtarget) {
4327 MVT VT = Op.getSimpleValueType();
4328 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4329
4330 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4331
4332 SDLoc DL(Op);
4333 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4334
4335 MVT XLenVT = Subtarget.getXLenVT();
4336 unsigned NumElts = Op.getNumOperands();
4337
4338 if (VT.getVectorElementType() == MVT::i1) {
4339 if (ISD::isBuildVectorAllZeros(N: Op.getNode())) {
4340 SDValue VMClr = DAG.getNode(Opcode: RISCVISD::VMCLR_VL, DL, VT: ContainerVT, Operand: VL);
4341 return convertFromScalableVector(VT, V: VMClr, DAG, Subtarget);
4342 }
4343
4344 if (ISD::isBuildVectorAllOnes(N: Op.getNode())) {
4345 SDValue VMSet = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
4346 return convertFromScalableVector(VT, V: VMSet, DAG, Subtarget);
4347 }
4348
4349 // Lower constant mask BUILD_VECTORs via an integer vector type, in
4350 // scalar integer chunks whose bit-width depends on the number of mask
4351 // bits and XLEN.
4352 // First, determine the most appropriate scalar integer type to use. This
4353 // is at most XLenVT, but may be shrunk to a smaller vector element type
4354 // according to the size of the final vector - use i8 chunks rather than
4355 // XLenVT if we're producing a v8i1. This results in more consistent
4356 // codegen across RV32 and RV64.
4357 unsigned NumViaIntegerBits = std::clamp(val: NumElts, lo: 8u, hi: Subtarget.getXLen());
4358 NumViaIntegerBits = std::min(a: NumViaIntegerBits, b: Subtarget.getELen());
4359 // If we have to use more than one INSERT_VECTOR_ELT then this
4360 // optimization is likely to increase code size; avoid performing it in
4361 // such a case. We can use a load from a constant pool in this case.
4362 if (DAG.shouldOptForSize() && NumElts > NumViaIntegerBits)
4363 return SDValue();
4364 // Now we can create our integer vector type. Note that it may be larger
4365 // than the resulting mask type: v4i1 would use v1i8 as its integer type.
4366 unsigned IntegerViaVecElts = divideCeil(Numerator: NumElts, Denominator: NumViaIntegerBits);
4367 MVT IntegerViaVecVT =
4368 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: NumViaIntegerBits),
4369 NumElements: IntegerViaVecElts);
4370
4371 uint64_t Bits = 0;
4372 unsigned BitPos = 0, IntegerEltIdx = 0;
4373 SmallVector<SDValue, 8> Elts(IntegerViaVecElts);
4374
4375 for (unsigned I = 0; I < NumElts;) {
4376 SDValue V = Op.getOperand(i: I);
4377 bool BitValue = !V.isUndef() && V->getAsZExtVal();
4378 Bits |= ((uint64_t)BitValue << BitPos);
4379 ++BitPos;
4380 ++I;
4381
4382 // Once we accumulate enough bits to fill our scalar type or process the
4383 // last element, insert into our vector and clear our accumulated data.
4384 if (I % NumViaIntegerBits == 0 || I == NumElts) {
4385 if (NumViaIntegerBits <= 32)
4386 Bits = SignExtend64<32>(x: Bits);
4387 SDValue Elt = DAG.getSignedConstant(Val: Bits, DL, VT: XLenVT);
4388 Elts[IntegerEltIdx] = Elt;
4389 Bits = 0;
4390 BitPos = 0;
4391 IntegerEltIdx++;
4392 }
4393 }
4394
4395 SDValue Vec = DAG.getBuildVector(VT: IntegerViaVecVT, DL, Ops: Elts);
4396
4397 if (NumElts < NumViaIntegerBits) {
4398 // If we're producing a smaller vector than our minimum legal integer
4399 // type, bitcast to the equivalent (known-legal) mask type, and extract
4400 // our final mask.
4401 assert(IntegerViaVecVT == MVT::v1i8 && "Unexpected mask vector type");
4402 Vec = DAG.getBitcast(VT: MVT::v8i1, V: Vec);
4403 Vec = DAG.getExtractSubvector(DL, VT, Vec, Idx: 0);
4404 } else {
4405 // Else we must have produced an integer type with the same size as the
4406 // mask type; bitcast for the final result.
4407 assert(VT.getSizeInBits() == IntegerViaVecVT.getSizeInBits());
4408 Vec = DAG.getBitcast(VT, V: Vec);
4409 }
4410
4411 return Vec;
4412 }
4413
4414 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4415 unsigned Opc = VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
4416 : RISCVISD::VMV_V_X_VL;
4417 if (!VT.isFloatingPoint())
4418 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Splat);
4419 Splat =
4420 DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Splat, N3: VL);
4421 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
4422 }
4423
4424 // Try and match index sequences, which we can lower to the vid instruction
4425 // with optional modifications. An all-undef vector is matched by
4426 // getSplatValue, above.
4427 if (SDValue Res = lowerBuildVectorViaVID(Op, DAG, Subtarget))
4428 return Res;
4429
4430 // For very small build_vectors, use a single scalar insert of a constant.
4431 // TODO: Base this on constant rematerialization cost, not size.
4432 const unsigned EltBitSize = VT.getScalarSizeInBits();
4433 if (VT.getSizeInBits() <= 32 &&
4434 ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode())) {
4435 MVT ViaIntVT = MVT::getIntegerVT(BitWidth: VT.getSizeInBits());
4436 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32) &&
4437 "Unexpected sequence type");
4438 // If we can use the original VL with the modified element type, this
4439 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4440 // be moved into InsertVSETVLI?
4441 unsigned ViaVecLen =
4442 (Subtarget.getRealMinVLen() >= VT.getSizeInBits() * NumElts) ? NumElts : 1;
4443 MVT ViaVecVT = MVT::getVectorVT(VT: ViaIntVT, NumElements: ViaVecLen);
4444
4445 uint64_t EltMask = maskTrailingOnes<uint64_t>(N: EltBitSize);
4446 uint64_t SplatValue = 0;
4447 // Construct the amalgamated value at this larger vector type.
4448 for (const auto &OpIdx : enumerate(First: Op->op_values())) {
4449 const auto &SeqV = OpIdx.value();
4450 if (!SeqV.isUndef())
4451 SplatValue |=
4452 ((SeqV->getAsZExtVal() & EltMask) << (OpIdx.index() * EltBitSize));
4453 }
4454
4455 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4456 // achieve better constant materializion.
4457 // On RV32, we need to sign-extend to use getSignedConstant.
4458 if (ViaIntVT == MVT::i32)
4459 SplatValue = SignExtend64<32>(x: SplatValue);
4460
4461 SDValue Vec = DAG.getInsertVectorElt(
4462 DL, Vec: DAG.getUNDEF(VT: ViaVecVT),
4463 Elt: DAG.getSignedConstant(Val: SplatValue, DL, VT: XLenVT), Idx: 0);
4464 if (ViaVecLen != 1)
4465 Vec = DAG.getExtractSubvector(DL, VT: MVT::getVectorVT(VT: ViaIntVT, NumElements: 1), Vec, Idx: 0);
4466 return DAG.getBitcast(VT, V: Vec);
4467 }
4468
4469
4470 // Attempt to detect "hidden" splats, which only reveal themselves as splats
4471 // when re-interpreted as a vector with a larger element type. For example,
4472 // v4i16 = build_vector i16 0, i16 1, i16 0, i16 1
4473 // could be instead splat as
4474 // v2i32 = build_vector i32 0x00010000, i32 0x00010000
4475 // TODO: This optimization could also work on non-constant splats, but it
4476 // would require bit-manipulation instructions to construct the splat value.
4477 SmallVector<SDValue> Sequence;
4478 const auto *BV = cast<BuildVectorSDNode>(Val&: Op);
4479 if (VT.isInteger() && EltBitSize < Subtarget.getELen() &&
4480 ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode()) &&
4481 BV->getRepeatedSequence(Sequence) &&
4482 (Sequence.size() * EltBitSize) <= Subtarget.getELen()) {
4483 unsigned SeqLen = Sequence.size();
4484 MVT ViaIntVT = MVT::getIntegerVT(BitWidth: EltBitSize * SeqLen);
4485 assert((ViaIntVT == MVT::i16 || ViaIntVT == MVT::i32 ||
4486 ViaIntVT == MVT::i64) &&
4487 "Unexpected sequence type");
4488
4489 // If we can use the original VL with the modified element type, this
4490 // means we only have a VTYPE toggle, not a VL toggle. TODO: Should this
4491 // be moved into InsertVSETVLI?
4492 const unsigned RequiredVL = NumElts / SeqLen;
4493 const unsigned ViaVecLen =
4494 (Subtarget.getRealMinVLen() >= ViaIntVT.getSizeInBits() * NumElts) ?
4495 NumElts : RequiredVL;
4496 MVT ViaVecVT = MVT::getVectorVT(VT: ViaIntVT, NumElements: ViaVecLen);
4497
4498 unsigned EltIdx = 0;
4499 uint64_t EltMask = maskTrailingOnes<uint64_t>(N: EltBitSize);
4500 uint64_t SplatValue = 0;
4501 // Construct the amalgamated value which can be splatted as this larger
4502 // vector type.
4503 for (const auto &SeqV : Sequence) {
4504 if (!SeqV.isUndef())
4505 SplatValue |=
4506 ((SeqV->getAsZExtVal() & EltMask) << (EltIdx * EltBitSize));
4507 EltIdx++;
4508 }
4509
4510 // On RV64, sign-extend from 32 to 64 bits where possible in order to
4511 // achieve better constant materializion.
4512 // On RV32, we need to sign-extend to use getSignedConstant.
4513 if (ViaIntVT == MVT::i32)
4514 SplatValue = SignExtend64<32>(x: SplatValue);
4515
4516 // Since we can't introduce illegal i64 types at this stage, we can only
4517 // perform an i64 splat on RV32 if it is its own sign-extended value. That
4518 // way we can use RVV instructions to splat.
4519 assert((ViaIntVT.bitsLE(XLenVT) ||
4520 (!Subtarget.is64Bit() && ViaIntVT == MVT::i64)) &&
4521 "Unexpected bitcast sequence");
4522 if (ViaIntVT.bitsLE(VT: XLenVT) || isInt<32>(x: SplatValue)) {
4523 SDValue ViaVL =
4524 DAG.getConstant(Val: ViaVecVT.getVectorNumElements(), DL, VT: XLenVT);
4525 MVT ViaContainerVT =
4526 getContainerForFixedLengthVector(DAG, VT: ViaVecVT, Subtarget);
4527 SDValue Splat =
4528 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ViaContainerVT,
4529 N1: DAG.getUNDEF(VT: ViaContainerVT),
4530 N2: DAG.getSignedConstant(Val: SplatValue, DL, VT: XLenVT), N3: ViaVL);
4531 Splat = convertFromScalableVector(VT: ViaVecVT, V: Splat, DAG, Subtarget);
4532 if (ViaVecLen != RequiredVL)
4533 Splat = DAG.getExtractSubvector(
4534 DL, VT: MVT::getVectorVT(VT: ViaIntVT, NumElements: RequiredVL), Vec: Splat, Idx: 0);
4535 return DAG.getBitcast(VT, V: Splat);
4536 }
4537 }
4538
4539 // If the number of signbits allows, see if we can lower as a <N x i8>.
4540 // Our main goal here is to reduce LMUL (and thus work) required to
4541 // build the constant, but we will also narrow if the resulting
4542 // narrow vector is known to materialize cheaply.
4543 // TODO: We really should be costing the smaller vector. There are
4544 // profitable cases this misses.
4545 if (EltBitSize > 8 && VT.isInteger() &&
4546 (NumElts <= 4 || VT.getSizeInBits() > Subtarget.getRealMinVLen()) &&
4547 DAG.ComputeMaxSignificantBits(Op) <= 8) {
4548 SDValue Source = DAG.getBuildVector(VT: VT.changeVectorElementType(EltVT: MVT::i8),
4549 DL, Ops: Op->ops());
4550 Source = convertToScalableVector(VT: ContainerVT.changeVectorElementType(EltVT: MVT::i8),
4551 V: Source, DAG, Subtarget);
4552 SDValue Res = DAG.getNode(Opcode: RISCVISD::VSEXT_VL, DL, VT: ContainerVT, N1: Source, N2: Mask, N3: VL);
4553 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
4554 }
4555
4556 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4557 return Res;
4558
4559 // For constant vectors, use generic constant pool lowering. Otherwise,
4560 // we'd have to materialize constants in GPRs just to move them into the
4561 // vector.
4562 return SDValue();
4563}
4564
4565static unsigned getPACKOpcode(unsigned DestBW,
4566 const RISCVSubtarget &Subtarget) {
4567 switch (DestBW) {
4568 default:
4569 llvm_unreachable("Unsupported pack size");
4570 case 16:
4571 return RISCV::PACKH;
4572 case 32:
4573 return Subtarget.is64Bit() ? RISCV::PACKW : RISCV::PACK;
4574 case 64:
4575 assert(Subtarget.is64Bit());
4576 return RISCV::PACK;
4577 }
4578}
4579
4580/// Double the element size of the build vector to reduce the number
4581/// of vslide1down in the build vector chain. In the worst case, this
4582/// trades three scalar operations for 1 vector operation. Scalar
4583/// operations are generally lower latency, and for out-of-order cores
4584/// we also benefit from additional parallelism.
4585static SDValue lowerBuildVectorViaPacking(SDValue Op, SelectionDAG &DAG,
4586 const RISCVSubtarget &Subtarget) {
4587 SDLoc DL(Op);
4588 MVT VT = Op.getSimpleValueType();
4589 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4590 MVT ElemVT = VT.getVectorElementType();
4591 if (!ElemVT.isInteger())
4592 return SDValue();
4593
4594 // TODO: Relax these architectural restrictions, possibly with costing
4595 // of the actual instructions required.
4596 if (!Subtarget.hasStdExtZbb() || !Subtarget.hasStdExtZba())
4597 return SDValue();
4598
4599 unsigned NumElts = VT.getVectorNumElements();
4600 unsigned ElemSizeInBits = ElemVT.getSizeInBits();
4601 if (ElemSizeInBits >= std::min(a: Subtarget.getELen(), b: Subtarget.getXLen()) ||
4602 NumElts % 2 != 0)
4603 return SDValue();
4604
4605 // Produce [B,A] packed into a type twice as wide. Note that all
4606 // scalars are XLenVT, possibly masked (see below).
4607 MVT XLenVT = Subtarget.getXLenVT();
4608 SDValue Mask = DAG.getConstant(
4609 Val: APInt::getLowBitsSet(numBits: XLenVT.getSizeInBits(), loBitsSet: ElemSizeInBits), DL, VT: XLenVT);
4610 auto pack = [&](SDValue A, SDValue B) {
4611 // Bias the scheduling of the inserted operations to near the
4612 // definition of the element - this tends to reduce register
4613 // pressure overall.
4614 SDLoc ElemDL(B);
4615 if (Subtarget.hasStdExtZbkb())
4616 // Note that we're relying on the high bits of the result being
4617 // don't care. For PACKW, the result is *sign* extended.
4618 return SDValue(
4619 DAG.getMachineNode(Opcode: getPACKOpcode(DestBW: ElemSizeInBits * 2, Subtarget),
4620 dl: ElemDL, VT: XLenVT, Op1: A, Op2: B),
4621 0);
4622
4623 A = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(A), VT: XLenVT, N1: A, N2: Mask);
4624 B = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(B), VT: XLenVT, N1: B, N2: Mask);
4625 SDValue ShtAmt = DAG.getConstant(Val: ElemSizeInBits, DL: ElemDL, VT: XLenVT);
4626 return DAG.getNode(Opcode: ISD::OR, DL: ElemDL, VT: XLenVT, N1: A,
4627 N2: DAG.getNode(Opcode: ISD::SHL, DL: ElemDL, VT: XLenVT, N1: B, N2: ShtAmt),
4628 Flags: SDNodeFlags::Disjoint);
4629 };
4630
4631 SmallVector<SDValue> NewOperands;
4632 NewOperands.reserve(N: NumElts / 2);
4633 for (unsigned i = 0; i < VT.getVectorNumElements(); i += 2)
4634 NewOperands.push_back(Elt: pack(Op.getOperand(i), Op.getOperand(i: i + 1)));
4635 assert(NumElts == NewOperands.size() * 2);
4636 MVT WideVT = MVT::getIntegerVT(BitWidth: ElemSizeInBits * 2);
4637 MVT WideVecVT = MVT::getVectorVT(VT: WideVT, NumElements: NumElts / 2);
4638 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
4639 Operand: DAG.getBuildVector(VT: WideVecVT, DL, Ops: NewOperands));
4640}
4641
4642static SDValue lowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG,
4643 const RISCVSubtarget &Subtarget) {
4644 MVT VT = Op.getSimpleValueType();
4645 assert(VT.isFixedLengthVector() && "Unexpected vector!");
4646
4647 MVT EltVT = VT.getVectorElementType();
4648 MVT XLenVT = Subtarget.getXLenVT();
4649
4650 SDLoc DL(Op);
4651
4652 if (Subtarget.isRV32() && Subtarget.hasStdExtP()) {
4653 if (VT != MVT::v4i8)
4654 return SDValue();
4655
4656 // <4 x i8> BUILD_VECTOR a, b, c, d -> PACK(PPACK.DH pair(a, c), pair(b, d))
4657 SDValue Val0 =
4658 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 0));
4659 SDValue Val1 =
4660 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 1));
4661 SDValue Val2 =
4662 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 2));
4663 SDValue Val3 =
4664 DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v4i8, Operand: Op->getOperand(Num: 3));
4665 SDValue PPairDB =
4666 DAG.getNode(Opcode: RISCVISD::PPAIRE_DB, DL, ResultTys: {MVT::v4i8, MVT::v4i8},
4667 Ops: {Val0, Val2, Val1, Val3});
4668
4669 return DAG.getNode(
4670 Opcode: ISD::BITCAST, DL, VT: MVT::v4i8,
4671 Operand: SDValue(
4672 DAG.getMachineNode(
4673 Opcode: RISCV::PACK, dl: DL, VT: MVT::i32,
4674 Ops: {DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: PPairDB.getValue(R: 0)),
4675 DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: PPairDB.getValue(R: 1))}),
4676 0));
4677 }
4678
4679 // Proper support for f16 requires Zvfh. bf16 always requires special
4680 // handling. We need to cast the scalar to integer and create an integer
4681 // build_vector.
4682 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
4683 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
4684 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
4685 SmallVector<SDValue, 16> NewOps(Op.getNumOperands());
4686 for (const auto &[I, U] : enumerate(First: Op->ops())) {
4687 SDValue Elem = U.get();
4688 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
4689 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin())) {
4690 // Called by LegalizeDAG, we need to use XLenVT operations since we
4691 // can't create illegal types.
4692 if (auto *C = dyn_cast<ConstantFPSDNode>(Val&: Elem)) {
4693 // Manually constant fold so the integer build_vector can be lowered
4694 // better. Waiting for DAGCombine will be too late.
4695 APInt V =
4696 C->getValueAPF().bitcastToAPInt().sext(width: XLenVT.getSizeInBits());
4697 NewOps[I] = DAG.getConstant(Val: V, DL, VT: XLenVT);
4698 } else {
4699 NewOps[I] = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Elem);
4700 }
4701 } else {
4702 // Called by scalar type legalizer, we can use i16.
4703 NewOps[I] = DAG.getBitcast(VT: MVT::i16, V: Op.getOperand(i: I));
4704 }
4705 }
4706 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: IVT, Ops: NewOps);
4707 return DAG.getBitcast(VT, V: Res);
4708 }
4709
4710 if (ISD::isBuildVectorOfConstantSDNodes(N: Op.getNode()) ||
4711 ISD::isBuildVectorOfConstantFPSDNodes(N: Op.getNode()))
4712 return lowerBuildVectorOfConstants(Op, DAG, Subtarget);
4713
4714 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4715
4716 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
4717
4718 if (VT.getVectorElementType() == MVT::i1) {
4719 // A BUILD_VECTOR can be lowered as a SETCC. For each fixed-length mask
4720 // vector type, we have a legal equivalently-sized i8 type, so we can use
4721 // that.
4722 MVT WideVecVT = VT.changeVectorElementType(EltVT: MVT::i8);
4723 SDValue VecZero = DAG.getConstant(Val: 0, DL, VT: WideVecVT);
4724
4725 SDValue WideVec;
4726 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4727 // For a splat, perform a scalar truncate before creating the wider
4728 // vector.
4729 Splat = DAG.getNode(Opcode: ISD::AND, DL, VT: Splat.getValueType(), N1: Splat,
4730 N2: DAG.getConstant(Val: 1, DL, VT: Splat.getValueType()));
4731 WideVec = DAG.getSplatBuildVector(VT: WideVecVT, DL, Op: Splat);
4732 } else {
4733 SmallVector<SDValue, 8> Ops(Op->op_values());
4734 WideVec = DAG.getBuildVector(VT: WideVecVT, DL, Ops);
4735 SDValue VecOne = DAG.getConstant(Val: 1, DL, VT: WideVecVT);
4736 WideVec = DAG.getNode(Opcode: ISD::AND, DL, VT: WideVecVT, N1: WideVec, N2: VecOne);
4737 }
4738
4739 return DAG.getSetCC(DL, VT, LHS: WideVec, RHS: VecZero, Cond: ISD::SETNE);
4740 }
4741
4742 if (SDValue Splat = cast<BuildVectorSDNode>(Val&: Op)->getSplatValue()) {
4743 if (auto Gather = matchSplatAsGather(SplatVal: Splat, VT, DL, DAG, Subtarget))
4744 return Gather;
4745
4746 if (!VT.isFloatingPoint())
4747 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Splat);
4748
4749 // Prefer vmv.s.x/vfmv.s.f if legal to reduce work and register
4750 // pressure at high LMUL.
4751 bool IsScalar = all_of(Range: Op->ops().drop_front(),
4752 P: [](const SDUse &U) { return U.get().isUndef(); });
4753 unsigned Opc =
4754 VT.isFloatingPoint()
4755 ? (IsScalar ? RISCVISD::VFMV_S_F_VL : RISCVISD::VFMV_V_F_VL)
4756 : (IsScalar ? RISCVISD::VMV_S_X_VL : RISCVISD::VMV_V_X_VL);
4757 Splat =
4758 DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Splat, N3: VL);
4759 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
4760 }
4761
4762 if (SDValue Res = lowerBuildVectorViaDominantValues(Op, DAG, Subtarget))
4763 return Res;
4764
4765 // If we're compiling for an exact VLEN value, we can split our work per
4766 // register in the register group.
4767 if (const auto VLen = Subtarget.getRealVLen();
4768 VLen && VT.getSizeInBits().getKnownMinValue() > *VLen) {
4769 MVT ElemVT = VT.getVectorElementType();
4770 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
4771 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
4772 MVT OneRegVT = MVT::getVectorVT(VT: ElemVT, NumElements: ElemsPerVReg);
4773 MVT M1VT = getContainerForFixedLengthVector(DAG, VT: OneRegVT, Subtarget);
4774 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
4775
4776 // The following semantically builds up a fixed length concat_vector
4777 // of the component build_vectors. We eagerly lower to scalable and
4778 // insert_subvector here to avoid DAG combining it back to a large
4779 // build_vector.
4780 SmallVector<SDValue> BuildVectorOps(Op->ops());
4781 unsigned NumOpElts = M1VT.getVectorMinNumElements();
4782 SDValue Vec = DAG.getUNDEF(VT: ContainerVT);
4783 for (unsigned i = 0; i < VT.getVectorNumElements(); i += ElemsPerVReg) {
4784 auto OneVRegOfOps = ArrayRef(BuildVectorOps).slice(N: i, M: ElemsPerVReg);
4785 SDValue SubBV =
4786 DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: OneRegVT, Ops: OneVRegOfOps);
4787 SubBV = convertToScalableVector(VT: M1VT, V: SubBV, DAG, Subtarget);
4788 unsigned InsertIdx = (i / ElemsPerVReg) * NumOpElts;
4789 Vec = DAG.getInsertSubvector(DL, Vec, SubVec: SubBV, Idx: InsertIdx);
4790 }
4791 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
4792 }
4793
4794 // If we're about to resort to vslide1down (or stack usage), pack our
4795 // elements into the widest scalar type we can. This will force a VL/VTYPE
4796 // toggle, but reduces the critical path, the number of vslide1down ops
4797 // required, and possibly enables scalar folds of the values.
4798 if (SDValue Res = lowerBuildVectorViaPacking(Op, DAG, Subtarget))
4799 return Res;
4800
4801 // For m1 vectors, if we have non-undef values in both halves of our vector,
4802 // split the vector into low and high halves, build them separately, then
4803 // use a vselect to combine them. For long vectors, this cuts the critical
4804 // path of the vslide1down sequence in half, and gives us an opportunity
4805 // to special case each half independently. Note that we don't change the
4806 // length of the sub-vectors here, so if both fallback to the generic
4807 // vslide1down path, we should be able to fold the vselect into the final
4808 // vslidedown (for the undef tail) for the first half w/ masking.
4809 unsigned NumElts = VT.getVectorNumElements();
4810 unsigned NumUndefElts =
4811 count_if(Range: Op->op_values(), P: [](const SDValue &V) { return V.isUndef(); });
4812 unsigned NumDefElts = NumElts - NumUndefElts;
4813 if (NumDefElts >= 8 && NumDefElts > NumElts / 2 &&
4814 ContainerVT.bitsLE(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT))) {
4815 SmallVector<SDValue> SubVecAOps, SubVecBOps;
4816 SmallVector<SDValue> MaskVals;
4817 SDValue UndefElem = DAG.getUNDEF(VT: Op->getOperand(Num: 0)->getValueType(ResNo: 0));
4818 SubVecAOps.reserve(N: NumElts);
4819 SubVecBOps.reserve(N: NumElts);
4820 for (const auto &[Idx, U] : enumerate(First: Op->ops())) {
4821 SDValue Elem = U.get();
4822 if (Idx < NumElts / 2) {
4823 SubVecAOps.push_back(Elt: Elem);
4824 SubVecBOps.push_back(Elt: UndefElem);
4825 } else {
4826 SubVecAOps.push_back(Elt: UndefElem);
4827 SubVecBOps.push_back(Elt: Elem);
4828 }
4829 bool SelectMaskVal = (Idx < NumElts / 2);
4830 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
4831 }
4832 assert(SubVecAOps.size() == NumElts && SubVecBOps.size() == NumElts &&
4833 MaskVals.size() == NumElts);
4834
4835 SDValue SubVecA = DAG.getBuildVector(VT, DL, Ops: SubVecAOps);
4836 SDValue SubVecB = DAG.getBuildVector(VT, DL, Ops: SubVecBOps);
4837 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
4838 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
4839 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: SubVecA, N3: SubVecB);
4840 }
4841
4842 // Cap the cost at a value linear to the number of elements in the vector.
4843 // The default lowering is to use the stack. The vector store + scalar loads
4844 // is linear in VL. However, at high lmuls vslide1down and vslidedown end up
4845 // being (at least) linear in LMUL. As a result, using the vslidedown
4846 // lowering for every element ends up being VL*LMUL..
4847 // TODO: Should we be directly costing the stack alternative? Doing so might
4848 // give us a more accurate upper bound.
4849 InstructionCost LinearBudget = VT.getVectorNumElements() * 2;
4850
4851 // TODO: unify with TTI getSlideCost.
4852 InstructionCost PerSlideCost = 1;
4853 switch (RISCVTargetLowering::getLMUL(VT: ContainerVT)) {
4854 default: break;
4855 case RISCVVType::LMUL_2:
4856 PerSlideCost = 2;
4857 break;
4858 case RISCVVType::LMUL_4:
4859 PerSlideCost = 4;
4860 break;
4861 case RISCVVType::LMUL_8:
4862 PerSlideCost = 8;
4863 break;
4864 }
4865
4866 // TODO: Should we be using the build instseq then cost + evaluate scheme
4867 // we use for integer constants here?
4868 unsigned UndefCount = 0;
4869 for (const SDValue &V : Op->ops()) {
4870 if (V.isUndef()) {
4871 UndefCount++;
4872 continue;
4873 }
4874 if (UndefCount) {
4875 LinearBudget -= PerSlideCost;
4876 UndefCount = 0;
4877 }
4878 LinearBudget -= PerSlideCost;
4879 }
4880 if (UndefCount) {
4881 LinearBudget -= PerSlideCost;
4882 }
4883
4884 if (LinearBudget < 0)
4885 return SDValue();
4886
4887 assert((!VT.isFloatingPoint() ||
4888 VT.getVectorElementType().getSizeInBits() <= Subtarget.getFLen()) &&
4889 "Illegal type which will result in reserved encoding");
4890
4891 const unsigned Policy = RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC;
4892
4893 // General case: splat the first operand and slide other operands down one
4894 // by one to form a vector. Alternatively, if every operand is an
4895 // extraction from element 0 of a vector, we use that vector from the last
4896 // extraction as the start value and slide up instead of slide down. Such that
4897 // (1) we can avoid the initial splat (2) we can turn those vslide1up into
4898 // vslideup of 1 later and eliminate the vector to scalar movement, which is
4899 // something we cannot do with vslide1down/vslidedown.
4900 // Of course, using vslide1up/vslideup might increase the register pressure,
4901 // and that's why we conservatively limit to cases where every operand is an
4902 // extraction from the first element.
4903 SmallVector<SDValue> Operands(Op->op_begin(), Op->op_end());
4904 SDValue EVec;
4905 bool SlideUp = false;
4906 auto getVSlide = [&](EVT ContainerVT, SDValue Passthru, SDValue Vec,
4907 SDValue Offset, SDValue Mask, SDValue VL) -> SDValue {
4908 if (SlideUp)
4909 return getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: Vec, Offset,
4910 Mask, VL, Policy);
4911 return getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: Vec, Offset,
4912 Mask, VL, Policy);
4913 };
4914
4915 // The reason we don't use all_of here is because we're also capturing EVec
4916 // from the last non-undef operand. If the std::execution_policy of the
4917 // underlying std::all_of is anything but std::sequenced_policy we might
4918 // capture the wrong EVec.
4919 for (SDValue V : Operands) {
4920 using namespace SDPatternMatch;
4921 SlideUp = V.isUndef() || sd_match(N: V, P: m_ExtractElt(Vec: m_Value(N&: EVec), Idx: m_Zero()));
4922 if (!SlideUp)
4923 break;
4924 }
4925
4926 // Do not slideup if the element type of EVec is different.
4927 if (SlideUp) {
4928 MVT EVecEltVT = EVec.getSimpleValueType().getVectorElementType();
4929 MVT ContainerEltVT = ContainerVT.getVectorElementType();
4930 if (EVecEltVT != ContainerEltVT)
4931 SlideUp = false;
4932 }
4933
4934 if (SlideUp) {
4935 MVT EVecContainerVT = EVec.getSimpleValueType();
4936 // Make sure the original vector has scalable vector type.
4937 if (EVecContainerVT.isFixedLengthVector()) {
4938 EVecContainerVT =
4939 getContainerForFixedLengthVector(DAG, VT: EVecContainerVT, Subtarget);
4940 EVec = convertToScalableVector(VT: EVecContainerVT, V: EVec, DAG, Subtarget);
4941 }
4942
4943 // Adapt EVec's type into ContainerVT.
4944 if (EVecContainerVT.getVectorMinNumElements() <
4945 ContainerVT.getVectorMinNumElements())
4946 EVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec: EVec, Idx: 0);
4947 else
4948 EVec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec: EVec, Idx: 0);
4949
4950 // Reverse the elements as we're going to slide up from the last element.
4951 std::reverse(first: Operands.begin(), last: Operands.end());
4952 }
4953
4954 SDValue Vec;
4955 UndefCount = 0;
4956 for (SDValue V : Operands) {
4957 if (V.isUndef()) {
4958 UndefCount++;
4959 continue;
4960 }
4961
4962 // Start our sequence with either a TA splat or extract source in the
4963 // hopes that hardware is able to recognize there's no dependency on the
4964 // prior value of our temporary register.
4965 if (!Vec) {
4966 if (SlideUp) {
4967 Vec = EVec;
4968 } else {
4969 Vec = DAG.getSplatVector(VT, DL, Op: V);
4970 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
4971 }
4972
4973 UndefCount = 0;
4974 continue;
4975 }
4976
4977 if (UndefCount) {
4978 const SDValue Offset = DAG.getConstant(Val: UndefCount, DL, VT: Subtarget.getXLenVT());
4979 Vec = getVSlide(ContainerVT, DAG.getUNDEF(VT: ContainerVT), Vec, Offset, Mask,
4980 VL);
4981 UndefCount = 0;
4982 }
4983
4984 unsigned Opcode;
4985 if (VT.isFloatingPoint())
4986 Opcode = SlideUp ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VFSLIDE1DOWN_VL;
4987 else
4988 Opcode = SlideUp ? RISCVISD::VSLIDE1UP_VL : RISCVISD::VSLIDE1DOWN_VL;
4989
4990 if (!VT.isFloatingPoint())
4991 V = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: V);
4992 Vec = DAG.getNode(Opcode, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Vec,
4993 N3: V, N4: Mask, N5: VL);
4994 }
4995 if (UndefCount) {
4996 const SDValue Offset = DAG.getConstant(Val: UndefCount, DL, VT: Subtarget.getXLenVT());
4997 Vec = getVSlide(ContainerVT, DAG.getUNDEF(VT: ContainerVT), Vec, Offset, Mask,
4998 VL);
4999 }
5000 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5001}
5002
5003static SDValue splatPartsI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5004 SDValue Lo, SDValue Hi, SDValue VL,
5005 SelectionDAG &DAG) {
5006 if (!Passthru)
5007 Passthru = DAG.getUNDEF(VT);
5008 if (isa<ConstantSDNode>(Val: Lo) && isa<ConstantSDNode>(Val: Hi)) {
5009 int32_t LoC = cast<ConstantSDNode>(Val&: Lo)->getSExtValue();
5010 int32_t HiC = cast<ConstantSDNode>(Val&: Hi)->getSExtValue();
5011 // If Hi constant is all the same sign bit as Lo, lower this as a custom
5012 // node in order to try and match RVV vector/scalar instructions.
5013 if ((LoC >> 31) == HiC)
5014 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5015
5016 // Use vmv.v.x with EEW=32. Use either a vsetivli or vsetvli to change
5017 // VL. This can temporarily increase VL if VL less than VLMAX.
5018 if (LoC == HiC) {
5019 SDValue NewVL;
5020 if (isa<ConstantSDNode>(Val: VL) && isUInt<4>(x: VL->getAsZExtVal()))
5021 NewVL = DAG.getNode(Opcode: ISD::ADD, DL, VT: VL.getValueType(), N1: VL, N2: VL);
5022 else
5023 NewVL = DAG.getRegister(Reg: RISCV::X0, VT: MVT::i32);
5024 MVT InterVT =
5025 MVT::getVectorVT(VT: MVT::i32, EC: VT.getVectorElementCount() * 2);
5026 auto InterVec = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: InterVT,
5027 N1: DAG.getUNDEF(VT: InterVT), N2: Lo, N3: NewVL);
5028 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: InterVec);
5029 }
5030 }
5031
5032 // Detect cases where Hi is (SRA Lo, 31) which means Hi is Lo sign extended.
5033 if (Hi.getOpcode() == ISD::SRA && Hi.getOperand(i: 0) == Lo &&
5034 isa<ConstantSDNode>(Val: Hi.getOperand(i: 1)) &&
5035 Hi.getConstantOperandVal(i: 1) == 31)
5036 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5037
5038 // If the hi bits of the splat are undefined, then it's fine to just splat Lo
5039 // even if it might be sign extended.
5040 if (Hi.isUndef())
5041 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Lo, N3: VL);
5042
5043 // Fall back to a stack store and stride x0 vector load.
5044 return DAG.getNode(Opcode: RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL, DL, VT, N1: Passthru, N2: Lo,
5045 N3: Hi, N4: VL);
5046}
5047
5048// Called by type legalization to handle splat of i64 on RV32.
5049// FIXME: We can optimize this when the type has sign or zero bits in one
5050// of the halves.
5051static SDValue splatSplitI64WithVL(const SDLoc &DL, MVT VT, SDValue Passthru,
5052 SDValue Scalar, SDValue VL,
5053 SelectionDAG &DAG) {
5054 assert(Scalar.getValueType() == MVT::i64 && "Unexpected VT!");
5055 SDValue Lo, Hi;
5056 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Scalar, DL, LoVT: MVT::i32, HiVT: MVT::i32);
5057 return splatPartsI64WithVL(DL, VT, Passthru, Lo, Hi, VL, DAG);
5058}
5059
5060// This function lowers a splat of a scalar operand Splat with the vector
5061// length VL. It ensures the final sequence is type legal, which is useful when
5062// lowering a splat after type legalization.
5063static SDValue lowerScalarSplat(SDValue Passthru, SDValue Scalar, SDValue VL,
5064 MVT VT, const SDLoc &DL, SelectionDAG &DAG,
5065 const RISCVSubtarget &Subtarget) {
5066 bool HasPassthru = Passthru && !Passthru.isUndef();
5067 if (!HasPassthru && !Passthru)
5068 Passthru = DAG.getUNDEF(VT);
5069
5070 MVT EltVT = VT.getVectorElementType();
5071 MVT XLenVT = Subtarget.getXLenVT();
5072
5073 if (VT.isFloatingPoint()) {
5074 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
5075 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
5076 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
5077 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
5078 Scalar = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Scalar);
5079 else
5080 Scalar = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Scalar);
5081 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
5082 Passthru = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: IVT, Operand: Passthru);
5083 SDValue Splat =
5084 lowerScalarSplat(Passthru, Scalar, VL, VT: IVT, DL, DAG, Subtarget);
5085 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Splat);
5086 }
5087 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
5088 }
5089
5090 // Simplest case is that the operand needs to be promoted to XLenVT.
5091 if (Scalar.getValueType().bitsLE(VT: XLenVT)) {
5092 // If the operand is a constant, sign extend to increase our chances
5093 // of being able to use a .vi instruction. ANY_EXTEND would become a
5094 // a zero extend and the simm5 check in isel would fail.
5095 // FIXME: Should we ignore the upper bits in isel instead?
5096 unsigned ExtOpc =
5097 isa<ConstantSDNode>(Val: Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5098 Scalar = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: Scalar);
5099 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
5100 }
5101
5102 assert(XLenVT == MVT::i32 && Scalar.getValueType() == MVT::i64 &&
5103 "Unexpected scalar for splat lowering!");
5104
5105 if (isOneConstant(V: VL) && isNullConstant(V: Scalar))
5106 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: Passthru,
5107 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: VL);
5108
5109 // Otherwise use the more complicated splatting algorithm.
5110 return splatSplitI64WithVL(DL, VT, Passthru, Scalar, VL, DAG);
5111}
5112
5113// This function lowers an insert of a scalar operand Scalar into lane
5114// 0 of the vector regardless of the value of VL. The contents of the
5115// remaining lanes of the result vector are unspecified. VL is assumed
5116// to be non-zero.
5117static SDValue lowerScalarInsert(SDValue Scalar, SDValue VL, MVT VT,
5118 const SDLoc &DL, SelectionDAG &DAG,
5119 const RISCVSubtarget &Subtarget) {
5120 assert(VT.isScalableVector() && "Expect VT is scalable vector type.");
5121
5122 const MVT XLenVT = Subtarget.getXLenVT();
5123 SDValue Passthru = DAG.getUNDEF(VT);
5124
5125 if (Scalar.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
5126 isNullConstant(V: Scalar.getOperand(i: 1))) {
5127 SDValue ExtractedVal = Scalar.getOperand(i: 0);
5128 // The element types must be the same.
5129 if (ExtractedVal.getValueType().getVectorElementType() ==
5130 VT.getVectorElementType()) {
5131 MVT ExtractedVT = ExtractedVal.getSimpleValueType();
5132 MVT ExtractedContainerVT = ExtractedVT;
5133 if (ExtractedContainerVT.isFixedLengthVector()) {
5134 ExtractedContainerVT = getContainerForFixedLengthVector(
5135 DAG, VT: ExtractedContainerVT, Subtarget);
5136 ExtractedVal = convertToScalableVector(VT: ExtractedContainerVT,
5137 V: ExtractedVal, DAG, Subtarget);
5138 }
5139 if (ExtractedContainerVT.bitsLE(VT))
5140 return DAG.getInsertSubvector(DL, Vec: Passthru, SubVec: ExtractedVal, Idx: 0);
5141 return DAG.getExtractSubvector(DL, VT, Vec: ExtractedVal, Idx: 0);
5142 }
5143 }
5144
5145 if (VT.isFloatingPoint())
5146 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT, N1: DAG.getUNDEF(VT), N2: Scalar,
5147 N3: VL);
5148
5149 // Avoid the tricky legalization cases by falling back to using the
5150 // splat code which already handles it gracefully.
5151 if (!Scalar.getValueType().bitsLE(VT: XLenVT))
5152 return lowerScalarSplat(Passthru: DAG.getUNDEF(VT), Scalar,
5153 VL: DAG.getConstant(Val: 1, DL, VT: XLenVT),
5154 VT, DL, DAG, Subtarget);
5155
5156 // If the operand is a constant, sign extend to increase our chances
5157 // of being able to use a .vi instruction. ANY_EXTEND would become a
5158 // a zero extend and the simm5 check in isel would fail.
5159 // FIXME: Should we ignore the upper bits in isel instead?
5160 unsigned ExtOpc =
5161 isa<ConstantSDNode>(Val: Scalar) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
5162 Scalar = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: Scalar);
5163 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: DAG.getUNDEF(VT), N2: Scalar,
5164 N3: VL);
5165}
5166
5167/// If concat_vector(V1,V2) could be folded away to some existing
5168/// vector source, return it. Note that the source may be larger
5169/// than the requested concat_vector (i.e. a extract_subvector
5170/// might be required.)
5171static SDValue foldConcatVector(SDValue V1, SDValue V2) {
5172 EVT VT = V1.getValueType();
5173 assert(VT == V2.getValueType() && "argument types must match");
5174 // Both input must be extracts.
5175 if (V1.getOpcode() != ISD::EXTRACT_SUBVECTOR ||
5176 V2.getOpcode() != ISD::EXTRACT_SUBVECTOR)
5177 return SDValue();
5178
5179 // Extracting from the same source.
5180 SDValue Src = V1.getOperand(i: 0);
5181 if (Src != V2.getOperand(i: 0) ||
5182 VT.isScalableVector() != Src.getValueType().isScalableVector())
5183 return SDValue();
5184
5185 // The extracts must extract the two halves of the source.
5186 if (V1.getConstantOperandVal(i: 1) != 0 ||
5187 V2.getConstantOperandVal(i: 1) != VT.getVectorMinNumElements())
5188 return SDValue();
5189
5190 return Src;
5191}
5192
5193// Can this shuffle be performed on exactly one (possibly larger) input?
5194static SDValue getSingleShuffleSrc(MVT VT, SDValue V1, SDValue V2) {
5195
5196 if (V2.isUndef())
5197 return V1;
5198
5199 unsigned NumElts = VT.getVectorNumElements();
5200 // Src needs to have twice the number of elements.
5201 // TODO: Update shuffle lowering to add the extract subvector
5202 if (SDValue Src = foldConcatVector(V1, V2);
5203 Src && Src.getValueType().getVectorNumElements() == (NumElts * 2))
5204 return Src;
5205
5206 return SDValue();
5207}
5208
5209/// Is this shuffle interleaving contiguous elements from one vector into the
5210/// even elements and contiguous elements from another vector into the odd
5211/// elements. \p EvenSrc will contain the element that should be in the first
5212/// even element. \p OddSrc will contain the element that should be in the first
5213/// odd element. These can be the first element in a source or the element half
5214/// way through the source.
5215static bool isInterleaveShuffle(ArrayRef<int> Mask, MVT VT, int &EvenSrc,
5216 int &OddSrc, const RISCVSubtarget &Subtarget) {
5217 // We need to be able to widen elements to the next larger integer type or
5218 // use the zip2a instruction at e64.
5219 if (VT.getScalarSizeInBits() >= Subtarget.getELen() &&
5220 !Subtarget.hasVendorXRivosVizip())
5221 return false;
5222
5223 int Size = Mask.size();
5224 int NumElts = VT.getVectorNumElements();
5225 assert(Size == (int)NumElts && "Unexpected mask size");
5226
5227 SmallVector<unsigned, 2> StartIndexes;
5228 if (!ShuffleVectorInst::isInterleaveMask(Mask, Factor: 2, NumInputElts: Size * 2, StartIndexes))
5229 return false;
5230
5231 EvenSrc = StartIndexes[0];
5232 OddSrc = StartIndexes[1];
5233
5234 // One source should be low half of first vector.
5235 if (EvenSrc != 0 && OddSrc != 0)
5236 return false;
5237
5238 // Subvectors will be subtracted from either at the start of the two input
5239 // vectors, or at the start and middle of the first vector if it's an unary
5240 // interleave.
5241 // In both cases, HalfNumElts will be extracted.
5242 // We need to ensure that the extract indices are 0 or HalfNumElts otherwise
5243 // we'll create an illegal extract_subvector.
5244 // FIXME: We could support other values using a slidedown first.
5245 int HalfNumElts = NumElts / 2;
5246 return ((EvenSrc % HalfNumElts) == 0) && ((OddSrc % HalfNumElts) == 0);
5247}
5248
5249/// Is this mask representing a masked combination of two slides?
5250static bool isMaskedSlidePair(ArrayRef<int> Mask,
5251 std::array<std::pair<int, int>, 2> &SrcInfo) {
5252 if (!llvm::isMaskedSlidePair(Mask, NumElts: Mask.size(), SrcInfo))
5253 return false;
5254
5255 // Avoid matching vselect idioms
5256 if (SrcInfo[0].second == 0 && SrcInfo[1].second == 0)
5257 return false;
5258 // Prefer vslideup as the second instruction, and identity
5259 // only as the initial instruction.
5260 if ((SrcInfo[0].second > 0 && SrcInfo[1].second < 0) ||
5261 SrcInfo[1].second == 0)
5262 std::swap(x&: SrcInfo[0], y&: SrcInfo[1]);
5263 assert(SrcInfo[0].first != -1 && "Must find one slide");
5264 return true;
5265}
5266
5267// Exactly matches the semantics of a previously existing custom matcher
5268// to allow migration to new matcher without changing output.
5269static bool isElementRotate(const std::array<std::pair<int, int>, 2> &SrcInfo,
5270 unsigned NumElts) {
5271 if (SrcInfo[1].first == -1)
5272 return true;
5273 return SrcInfo[0].second < 0 && SrcInfo[1].second > 0 &&
5274 SrcInfo[1].second - SrcInfo[0].second == (int)NumElts;
5275}
5276
5277static bool isAlternating(const std::array<std::pair<int, int>, 2> &SrcInfo,
5278 ArrayRef<int> Mask, unsigned Factor,
5279 bool RequiredPolarity) {
5280 int NumElts = Mask.size();
5281 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
5282 if (M < 0)
5283 continue;
5284 int Src = M >= NumElts;
5285 int Diff = (int)Idx - (M % NumElts);
5286 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
5287 assert(C != (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
5288 "Must match exactly one of the two slides");
5289 if (RequiredPolarity != (C == (Idx / Factor) % 2))
5290 return false;
5291 }
5292 return true;
5293}
5294
5295/// Given a shuffle which can be represented as a pair of two slides,
5296/// see if it is a zipeven idiom. Zipeven is:
5297/// vs2: a0 a1 a2 a3
5298/// vs1: b0 b1 b2 b3
5299/// vd: a0 b0 a2 b2
5300static bool isZipEven(const std::array<std::pair<int, int>, 2> &SrcInfo,
5301 ArrayRef<int> Mask, unsigned &Factor) {
5302 Factor = SrcInfo[1].second;
5303 return SrcInfo[0].second == 0 && isPowerOf2_32(Value: Factor) &&
5304 Mask.size() % Factor == 0 &&
5305 isAlternating(SrcInfo, Mask, Factor, RequiredPolarity: true);
5306}
5307
5308/// Given a shuffle which can be represented as a pair of two slides,
5309/// see if it is a zipodd idiom. Zipodd is:
5310/// vs2: a0 a1 a2 a3
5311/// vs1: b0 b1 b2 b3
5312/// vd: a1 b1 a3 b3
5313/// Note that the operand order is swapped due to the way we canonicalize
5314/// the slides, so SrCInfo[0] is vs1, and SrcInfo[1] is vs2.
5315static bool isZipOdd(const std::array<std::pair<int, int>, 2> &SrcInfo,
5316 ArrayRef<int> Mask, unsigned &Factor) {
5317 Factor = -SrcInfo[1].second;
5318 return SrcInfo[0].second == 0 && isPowerOf2_32(Value: Factor) &&
5319 Mask.size() % Factor == 0 &&
5320 isAlternating(SrcInfo, Mask, Factor, RequiredPolarity: false);
5321}
5322
5323// Lower a deinterleave shuffle to SRL and TRUNC. Factor must be
5324// 2, 4, 8 and the integer type Factor-times larger than VT's
5325// element type must be a legal element type.
5326// [a, p, b, q, c, r, d, s] -> [a, b, c, d] (Factor=2, Index=0)
5327// -> [p, q, r, s] (Factor=2, Index=1)
5328static SDValue getDeinterleaveShiftAndTrunc(const SDLoc &DL, MVT VT,
5329 SDValue Src, unsigned Factor,
5330 unsigned Index, SelectionDAG &DAG) {
5331 unsigned EltBits = VT.getScalarSizeInBits();
5332 ElementCount SrcEC = Src.getValueType().getVectorElementCount();
5333 MVT WideSrcVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits * Factor),
5334 EC: SrcEC.divideCoefficientBy(RHS: Factor));
5335 MVT ResVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits),
5336 EC: SrcEC.divideCoefficientBy(RHS: Factor));
5337 Src = DAG.getBitcast(VT: WideSrcVT, V: Src);
5338
5339 unsigned Shift = Index * EltBits;
5340 SDValue Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: WideSrcVT, N1: Src,
5341 N2: DAG.getConstant(Val: Shift, DL, VT: WideSrcVT));
5342 Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: ResVT, Operand: Res);
5343 MVT CastVT = ResVT.changeVectorElementType(EltVT: VT.getVectorElementType());
5344 Res = DAG.getBitcast(VT: CastVT, V: Res);
5345 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: Res, Idx: 0);
5346}
5347
5348/// Match a single source shuffle which is an identity except that some
5349/// particular element is repeated. This can be lowered as a masked
5350/// vrgather.vi/vx. Note that the two source form of this is handled
5351/// by the recursive splitting logic and doesn't need special handling.
5352static SDValue lowerVECTOR_SHUFFLEAsVRGatherVX(ShuffleVectorSDNode *SVN,
5353 const RISCVSubtarget &Subtarget,
5354 SelectionDAG &DAG) {
5355
5356 SDLoc DL(SVN);
5357 MVT VT = SVN->getSimpleValueType(ResNo: 0);
5358 SDValue V1 = SVN->getOperand(Num: 0);
5359 assert(SVN->getOperand(1).isUndef());
5360 ArrayRef<int> Mask = SVN->getMask();
5361 const unsigned NumElts = VT.getVectorNumElements();
5362 MVT XLenVT = Subtarget.getXLenVT();
5363
5364 std::optional<int> SplatIdx;
5365 for (auto [I, M] : enumerate(First&: Mask)) {
5366 if (M == -1 || I == (unsigned)M)
5367 continue;
5368 if (SplatIdx && *SplatIdx != M)
5369 return SDValue();
5370 SplatIdx = M;
5371 }
5372
5373 if (!SplatIdx)
5374 return SDValue();
5375
5376 SmallVector<SDValue> MaskVals;
5377 for (int MaskIndex : Mask) {
5378 bool SelectMaskVal = MaskIndex == *SplatIdx;
5379 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
5380 }
5381 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
5382 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
5383 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
5384 SDValue Splat = DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT),
5385 Mask: SmallVector<int>(NumElts, *SplatIdx));
5386 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: Splat, N3: V1);
5387}
5388
5389// Lower the following shuffle to vslidedown.
5390// a)
5391// t49: v8i8 = extract_subvector t13, Constant:i64<0>
5392// t109: v8i8 = extract_subvector t13, Constant:i64<8>
5393// t108: v8i8 = vector_shuffle<1,2,3,4,5,6,7,8> t49, t106
5394// b)
5395// t69: v16i16 = extract_subvector t68, Constant:i64<0>
5396// t23: v8i16 = extract_subvector t69, Constant:i64<0>
5397// t29: v4i16 = extract_subvector t23, Constant:i64<4>
5398// t26: v8i16 = extract_subvector t69, Constant:i64<8>
5399// t30: v4i16 = extract_subvector t26, Constant:i64<0>
5400// t54: v4i16 = vector_shuffle<1,2,3,4> t29, t30
5401static SDValue lowerVECTOR_SHUFFLEAsVSlidedown(const SDLoc &DL, MVT VT,
5402 SDValue V1, SDValue V2,
5403 ArrayRef<int> Mask,
5404 const RISCVSubtarget &Subtarget,
5405 SelectionDAG &DAG) {
5406 auto findNonEXTRACT_SUBVECTORParent =
5407 [](SDValue Parent) -> std::pair<SDValue, uint64_t> {
5408 uint64_t Offset = 0;
5409 while (Parent.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
5410 // EXTRACT_SUBVECTOR can be used to extract a fixed-width vector from
5411 // a scalable vector. But we don't want to match the case.
5412 Parent.getOperand(i: 0).getSimpleValueType().isFixedLengthVector()) {
5413 Offset += Parent.getConstantOperandVal(i: 1);
5414 Parent = Parent.getOperand(i: 0);
5415 }
5416 return std::make_pair(x&: Parent, y&: Offset);
5417 };
5418
5419 auto [V1Src, V1IndexOffset] = findNonEXTRACT_SUBVECTORParent(V1);
5420 auto [V2Src, V2IndexOffset] = findNonEXTRACT_SUBVECTORParent(V2);
5421
5422 // Extracting from the same source.
5423 SDValue Src = V1Src;
5424 if (Src != V2Src)
5425 return SDValue();
5426
5427 // Rebuild mask because Src may be from multiple EXTRACT_SUBVECTORs.
5428 SmallVector<int, 16> NewMask(Mask);
5429 for (size_t i = 0; i != NewMask.size(); ++i) {
5430 if (NewMask[i] == -1)
5431 continue;
5432
5433 if (static_cast<size_t>(NewMask[i]) < NewMask.size()) {
5434 NewMask[i] = NewMask[i] + V1IndexOffset;
5435 } else {
5436 // Minus NewMask.size() is needed. Otherwise, the b case would be
5437 // <5,6,7,12> instead of <5,6,7,8>.
5438 NewMask[i] = NewMask[i] - NewMask.size() + V2IndexOffset;
5439 }
5440 }
5441
5442 // First index must be known and non-zero. It will be used as the slidedown
5443 // amount.
5444 if (NewMask[0] <= 0)
5445 return SDValue();
5446
5447 // NewMask is also continuous.
5448 for (unsigned i = 1; i != NewMask.size(); ++i)
5449 if (NewMask[i - 1] + 1 != NewMask[i])
5450 return SDValue();
5451
5452 MVT XLenVT = Subtarget.getXLenVT();
5453 MVT SrcVT = Src.getSimpleValueType();
5454 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT: SrcVT, Subtarget);
5455 auto [TrueMask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
5456 SDValue Slidedown =
5457 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru: DAG.getUNDEF(VT: ContainerVT),
5458 Op: convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget),
5459 Offset: DAG.getConstant(Val: NewMask[0], DL, VT: XLenVT), Mask: TrueMask, VL);
5460 return DAG.getExtractSubvector(
5461 DL, VT, Vec: convertFromScalableVector(VT: SrcVT, V: Slidedown, DAG, Subtarget), Idx: 0);
5462}
5463
5464// Because vslideup leaves the destination elements at the start intact, we can
5465// use it to perform shuffles that insert subvectors:
5466//
5467// vector_shuffle v8:v8i8, v9:v8i8, <0, 1, 2, 3, 8, 9, 10, 11>
5468// ->
5469// vsetvli zero, 8, e8, mf2, ta, ma
5470// vslideup.vi v8, v9, 4
5471//
5472// vector_shuffle v8:v8i8, v9:v8i8 <0, 1, 8, 9, 10, 5, 6, 7>
5473// ->
5474// vsetvli zero, 5, e8, mf2, tu, ma
5475// vslideup.v1 v8, v9, 2
5476static SDValue lowerVECTOR_SHUFFLEAsVSlideup(const SDLoc &DL, MVT VT,
5477 SDValue V1, SDValue V2,
5478 ArrayRef<int> Mask,
5479 const RISCVSubtarget &Subtarget,
5480 SelectionDAG &DAG) {
5481 unsigned NumElts = VT.getVectorNumElements();
5482 int NumSubElts, Index;
5483 if (!ShuffleVectorInst::isInsertSubvectorMask(Mask, NumSrcElts: NumElts, NumSubElts,
5484 Index))
5485 return SDValue();
5486
5487 bool OpsSwapped = Mask[Index] < (int)NumElts;
5488 SDValue InPlace = OpsSwapped ? V2 : V1;
5489 SDValue ToInsert = OpsSwapped ? V1 : V2;
5490
5491 MVT XLenVT = Subtarget.getXLenVT();
5492 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5493 auto TrueMask = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).first;
5494 // We slide up by the index that the subvector is being inserted at, and set
5495 // VL to the index + the number of elements being inserted.
5496 unsigned Policy =
5497 RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED | RISCVVType::MASK_AGNOSTIC;
5498 // If the we're adding a suffix to the in place vector, i.e. inserting right
5499 // up to the very end of it, then we don't actually care about the tail.
5500 if (NumSubElts + Index >= (int)NumElts)
5501 Policy |= RISCVVType::TAIL_AGNOSTIC;
5502
5503 InPlace = convertToScalableVector(VT: ContainerVT, V: InPlace, DAG, Subtarget);
5504 ToInsert = convertToScalableVector(VT: ContainerVT, V: ToInsert, DAG, Subtarget);
5505 SDValue VL = DAG.getConstant(Val: NumSubElts + Index, DL, VT: XLenVT);
5506
5507 SDValue Res;
5508 // If we're inserting into the lowest elements, use a tail undisturbed
5509 // vmv.v.v.
5510 if (Index == 0)
5511 Res = DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: ContainerVT, N1: InPlace, N2: ToInsert,
5512 N3: VL);
5513 else
5514 Res = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: InPlace, Op: ToInsert,
5515 Offset: DAG.getConstant(Val: Index, DL, VT: XLenVT), Mask: TrueMask, VL, Policy);
5516 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
5517}
5518
5519// A shuffle of shuffles where the final data only is drawn from 2 input ops
5520// can be compressed into a single shuffle
5521static SDValue compressShuffleOfShuffles(ShuffleVectorSDNode *SVN,
5522 const RISCVSubtarget &Subtarget,
5523 SelectionDAG &DAG) {
5524 SDValue V1 = SVN->getOperand(Num: 0);
5525 SDValue V2 = SVN->getOperand(Num: 1);
5526
5527 if (V1.getOpcode() != ISD::VECTOR_SHUFFLE ||
5528 V2.getOpcode() != ISD::VECTOR_SHUFFLE)
5529 return SDValue();
5530
5531 if (!V1.hasOneUse() || !V2.hasOneUse())
5532 return SDValue();
5533
5534 ArrayRef<int> Mask = SVN->getMask();
5535 ArrayRef<int> V1Mask = cast<ShuffleVectorSDNode>(Val: V1.getNode())->getMask();
5536 ArrayRef<int> V2Mask = cast<ShuffleVectorSDNode>(Val: V2.getNode())->getMask();
5537 unsigned NumElts = Mask.size();
5538 SmallVector<int> NewMask(NumElts, -1);
5539 for (unsigned Idx : seq<unsigned>(Size: NumElts)) {
5540 int Lane = Mask[Idx];
5541 // Don't assign if poison
5542 if (Lane == -1)
5543 continue;
5544 int OrigLane;
5545 bool SecondOp = false;
5546 if ((unsigned)Lane < NumElts) {
5547 OrigLane = V1Mask[Lane];
5548 } else {
5549 OrigLane = V2Mask[Lane - NumElts];
5550 SecondOp = true;
5551 }
5552 if (OrigLane == -1)
5553 continue;
5554 // Don't handle if shuffling from a second operand
5555 if ((unsigned)OrigLane >= NumElts)
5556 return SDValue();
5557 if (SecondOp)
5558 OrigLane += NumElts;
5559 NewMask[Idx] = OrigLane;
5560 }
5561
5562 MVT VT = SVN->getSimpleValueType(ResNo: 0);
5563 SDLoc DL(SVN);
5564
5565 return DAG.getVectorShuffle(VT, dl: DL, N1: V1->getOperand(Num: 0), N2: V2->getOperand(Num: 0),
5566 Mask: NewMask);
5567}
5568
5569/// Match v(f)slide1up/down idioms. These operations involve sliding
5570/// N-1 elements to make room for an inserted scalar at one end.
5571static SDValue lowerVECTOR_SHUFFLEAsVSlide1(const SDLoc &DL, MVT VT,
5572 SDValue V1, SDValue V2,
5573 ArrayRef<int> Mask,
5574 const RISCVSubtarget &Subtarget,
5575 SelectionDAG &DAG) {
5576 bool OpsSwapped = false;
5577 if (!isa<BuildVectorSDNode>(Val: V1)) {
5578 if (!isa<BuildVectorSDNode>(Val: V2))
5579 return SDValue();
5580 std::swap(a&: V1, b&: V2);
5581 OpsSwapped = true;
5582 }
5583 SDValue Splat = cast<BuildVectorSDNode>(Val&: V1)->getSplatValue();
5584 if (!Splat)
5585 return SDValue();
5586
5587 // Return true if the mask could describe a slide of Mask.size() - 1
5588 // elements from concat_vector(V1, V2)[Base:] to [Offset:].
5589 auto isSlideMask = [](ArrayRef<int> Mask, unsigned Base, int Offset) {
5590 const unsigned S = (Offset > 0) ? 0 : -Offset;
5591 const unsigned E = Mask.size() - ((Offset > 0) ? Offset : 0);
5592 for (unsigned i = S; i != E; ++i)
5593 if (Mask[i] >= 0 && (unsigned)Mask[i] != Base + i + Offset)
5594 return false;
5595 return true;
5596 };
5597
5598 const unsigned NumElts = VT.getVectorNumElements();
5599 bool IsVSlidedown = isSlideMask(Mask, OpsSwapped ? 0 : NumElts, 1);
5600 if (!IsVSlidedown && !isSlideMask(Mask, OpsSwapped ? 0 : NumElts, -1))
5601 return SDValue();
5602
5603 const int InsertIdx = Mask[IsVSlidedown ? (NumElts - 1) : 0];
5604 // Inserted lane must come from splat, undef scalar is legal but not profitable.
5605 if (InsertIdx < 0 || InsertIdx / NumElts != (unsigned)OpsSwapped)
5606 return SDValue();
5607
5608 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5609 auto [TrueMask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
5610
5611 // zvfhmin and zvfbfmin don't have vfslide1{down,up}.vf so use fmv.x.h +
5612 // vslide1{down,up}.vx instead.
5613 if (VT.getVectorElementType() == MVT::bf16 ||
5614 (VT.getVectorElementType() == MVT::f16 &&
5615 !Subtarget.hasVInstructionsF16())) {
5616 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
5617 Splat =
5618 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: Subtarget.getXLenVT(), Operand: Splat);
5619 V2 = DAG.getBitcast(
5620 VT: IntVT, V: convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget));
5621 SDValue Vec = DAG.getNode(
5622 Opcode: IsVSlidedown ? RISCVISD::VSLIDE1DOWN_VL : RISCVISD::VSLIDE1UP_VL, DL,
5623 VT: IntVT, N1: DAG.getUNDEF(VT: IntVT), N2: V2, N3: Splat, N4: TrueMask, N5: VL);
5624 Vec = DAG.getBitcast(VT: ContainerVT, V: Vec);
5625 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5626 }
5627
5628 auto OpCode = IsVSlidedown ?
5629 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1DOWN_VL : RISCVISD::VSLIDE1DOWN_VL) :
5630 (VT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL : RISCVISD::VSLIDE1UP_VL);
5631 if (!VT.isFloatingPoint())
5632 Splat = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: Splat);
5633 auto Vec = DAG.getNode(Opcode: OpCode, DL, VT: ContainerVT,
5634 N1: DAG.getUNDEF(VT: ContainerVT),
5635 N2: convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget),
5636 N3: Splat, N4: TrueMask, N5: VL);
5637 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
5638}
5639
5640/// Match a mask which "spreads" the leading elements of a vector evenly
5641/// across the result. Factor is the spread amount, and Index is the
5642/// offset applied. (on success, Index < Factor) This is the inverse
5643/// of a deinterleave with the same Factor and Index. This is analogous
5644/// to an interleave, except that all but one lane is undef.
5645bool RISCVTargetLowering::isSpreadMask(ArrayRef<int> Mask, unsigned Factor,
5646 unsigned &Index) {
5647 SmallVector<bool> LaneIsUndef(Factor, true);
5648 for (unsigned i = 0; i < Mask.size(); i++)
5649 LaneIsUndef[i % Factor] &= (Mask[i] == -1);
5650
5651 bool Found = false;
5652 for (unsigned i = 0; i < Factor; i++) {
5653 if (LaneIsUndef[i])
5654 continue;
5655 if (Found)
5656 return false;
5657 Index = i;
5658 Found = true;
5659 }
5660 if (!Found)
5661 return false;
5662
5663 for (unsigned i = 0; i < Mask.size() / Factor; i++) {
5664 unsigned j = i * Factor + Index;
5665 if (Mask[j] != -1 && (unsigned)Mask[j] != i)
5666 return false;
5667 }
5668 return true;
5669}
5670
5671static SDValue lowerVZIP(unsigned Opc, SDValue Op0, SDValue Op1,
5672 const SDLoc &DL, SelectionDAG &DAG,
5673 const RISCVSubtarget &Subtarget) {
5674 assert(RISCVISD::RI_VZIPEVEN_VL == Opc || RISCVISD::RI_VZIPODD_VL == Opc ||
5675 RISCVISD::RI_VZIP2A_VL == Opc || RISCVISD::RI_VZIP2B_VL == Opc ||
5676 RISCVISD::RI_VUNZIP2A_VL == Opc || RISCVISD::RI_VUNZIP2B_VL == Opc);
5677 assert(Op0.getSimpleValueType() == Op1.getSimpleValueType());
5678
5679 MVT VT = Op0.getSimpleValueType();
5680 MVT IntVT = VT.changeVectorElementTypeToInteger();
5681 Op0 = DAG.getBitcast(VT: IntVT, V: Op0);
5682 Op1 = DAG.getBitcast(VT: IntVT, V: Op1);
5683
5684 MVT ContainerVT = IntVT;
5685 if (VT.isFixedLengthVector()) {
5686 ContainerVT = getContainerForFixedLengthVector(DAG, VT: IntVT, Subtarget);
5687 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
5688 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
5689 }
5690
5691 MVT InnerVT = ContainerVT;
5692 auto [Mask, VL] = getDefaultVLOps(VecVT: IntVT, ContainerVT: InnerVT, DL, DAG, Subtarget);
5693 if (Op1.isUndef() &&
5694 ContainerVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT)) &&
5695 (RISCVISD::RI_VUNZIP2A_VL == Opc || RISCVISD::RI_VUNZIP2B_VL == Opc)) {
5696 InnerVT = ContainerVT.getHalfNumVectorElementsVT();
5697 VL = DAG.getConstant(Val: VT.getVectorNumElements() / 2, DL,
5698 VT: Subtarget.getXLenVT());
5699 Mask = getAllOnesMask(VecVT: InnerVT, VL, DL, DAG);
5700 unsigned HighIdx = InnerVT.getVectorElementCount().getKnownMinValue();
5701 Op1 = DAG.getExtractSubvector(DL, VT: InnerVT, Vec: Op0, Idx: HighIdx);
5702 Op0 = DAG.getExtractSubvector(DL, VT: InnerVT, Vec: Op0, Idx: 0);
5703 }
5704
5705 SDValue Passthru = DAG.getUNDEF(VT: InnerVT);
5706 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: InnerVT, N1: Op0, N2: Op1, N3: Passthru, N4: Mask, N5: VL);
5707 if (InnerVT.bitsLT(VT: ContainerVT))
5708 Res = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec: Res, Idx: 0);
5709 if (IntVT.isFixedLengthVector())
5710 Res = convertFromScalableVector(VT: IntVT, V: Res, DAG, Subtarget);
5711 Res = DAG.getBitcast(VT, V: Res);
5712 return Res;
5713}
5714
5715// Given a vector a, b, c, d return a vector Factor times longer
5716// with Factor-1 undef's between elements. Ex:
5717// a, undef, b, undef, c, undef, d, undef (Factor=2, Index=0)
5718// undef, a, undef, b, undef, c, undef, d (Factor=2, Index=1)
5719static SDValue getWideningSpread(SDValue V, unsigned Factor, unsigned Index,
5720 const SDLoc &DL, SelectionDAG &DAG) {
5721
5722 MVT VT = V.getSimpleValueType();
5723 unsigned EltBits = VT.getScalarSizeInBits();
5724 ElementCount EC = VT.getVectorElementCount();
5725 V = DAG.getBitcast(VT: VT.changeTypeToInteger(), V);
5726
5727 MVT WideVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltBits * Factor), EC);
5728
5729 SDValue Result = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: V);
5730 // TODO: On rv32, the constant becomes a splat_vector_parts which does not
5731 // allow the SHL to fold away if Index is 0.
5732 if (Index != 0)
5733 Result = DAG.getNode(Opcode: ISD::SHL, DL, VT: WideVT, N1: Result,
5734 N2: DAG.getConstant(Val: EltBits * Index, DL, VT: WideVT));
5735 // Make sure to use original element type
5736 MVT ResultVT = MVT::getVectorVT(VT: VT.getVectorElementType(),
5737 EC: EC.multiplyCoefficientBy(RHS: Factor));
5738 return DAG.getBitcast(VT: ResultVT, V: Result);
5739}
5740
5741// Given two input vectors of <[vscale x ]n x ty>, use vwaddu.vv and vwmaccu.vx
5742// to create an interleaved vector of <[vscale x] n*2 x ty>.
5743// This requires that the size of ty is less than the subtarget's maximum ELEN.
5744static SDValue getWideningInterleave(SDValue EvenV, SDValue OddV,
5745 const SDLoc &DL, SelectionDAG &DAG,
5746 const RISCVSubtarget &Subtarget) {
5747
5748 // FIXME: Not only does this optimize the code, it fixes some correctness
5749 // issues because MIR does not have freeze.
5750 if (EvenV.isUndef())
5751 return getWideningSpread(V: OddV, Factor: 2, Index: 1, DL, DAG);
5752 if (OddV.isUndef())
5753 return getWideningSpread(V: EvenV, Factor: 2, Index: 0, DL, DAG);
5754
5755 MVT VecVT = EvenV.getSimpleValueType();
5756 MVT VecContainerVT = VecVT; // <vscale x n x ty>
5757 // Convert fixed vectors to scalable if needed
5758 if (VecContainerVT.isFixedLengthVector()) {
5759 VecContainerVT = getContainerForFixedLengthVector(DAG, VT: VecVT, Subtarget);
5760 EvenV = convertToScalableVector(VT: VecContainerVT, V: EvenV, DAG, Subtarget);
5761 OddV = convertToScalableVector(VT: VecContainerVT, V: OddV, DAG, Subtarget);
5762 }
5763
5764 assert(VecVT.getScalarSizeInBits() < Subtarget.getELen());
5765
5766 // We're working with a vector of the same size as the resulting
5767 // interleaved vector, but with half the number of elements and
5768 // twice the SEW (Hence the restriction on not using the maximum
5769 // ELEN)
5770 MVT WideVT =
5771 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VecVT.getScalarSizeInBits() * 2),
5772 EC: VecVT.getVectorElementCount());
5773 MVT WideContainerVT = WideVT; // <vscale x n x ty*2>
5774 if (WideContainerVT.isFixedLengthVector())
5775 WideContainerVT = getContainerForFixedLengthVector(DAG, VT: WideVT, Subtarget);
5776
5777 // Bitcast the input vectors to integers in case they are FP
5778 VecContainerVT = VecContainerVT.changeTypeToInteger();
5779 EvenV = DAG.getBitcast(VT: VecContainerVT, V: EvenV);
5780 OddV = DAG.getBitcast(VT: VecContainerVT, V: OddV);
5781
5782 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT: VecContainerVT, DL, DAG, Subtarget);
5783 SDValue Passthru = DAG.getUNDEF(VT: WideContainerVT);
5784
5785 SDValue Interleaved;
5786 if (Subtarget.hasStdExtZvbb()) {
5787 // Interleaved = (OddV << VecVT.getScalarSizeInBits()) + EvenV.
5788 SDValue OffsetVec =
5789 DAG.getConstant(Val: VecVT.getScalarSizeInBits(), DL, VT: VecContainerVT);
5790 Interleaved = DAG.getNode(Opcode: RISCVISD::VWSLL_VL, DL, VT: WideContainerVT, N1: OddV,
5791 N2: OffsetVec, N3: Passthru, N4: Mask, N5: VL);
5792 Interleaved = DAG.getNode(Opcode: RISCVISD::VWADDU_W_VL, DL, VT: WideContainerVT,
5793 N1: Interleaved, N2: EvenV, N3: Passthru, N4: Mask, N5: VL);
5794 } else {
5795 // FIXME: We should freeze the odd vector here. We already handled the case
5796 // of provably undef/poison above.
5797
5798 // Widen EvenV and OddV with 0s and add one copy of OddV to EvenV with
5799 // vwaddu.vv
5800 Interleaved = DAG.getNode(Opcode: RISCVISD::VWADDU_VL, DL, VT: WideContainerVT, N1: EvenV,
5801 N2: OddV, N3: Passthru, N4: Mask, N5: VL);
5802
5803 // Then get OddV * by 2^(VecVT.getScalarSizeInBits() - 1)
5804 SDValue AllOnesVec = DAG.getSplatVector(
5805 VT: VecContainerVT, DL, Op: DAG.getAllOnesConstant(DL, VT: Subtarget.getXLenVT()));
5806 SDValue OddsMul = DAG.getNode(Opcode: RISCVISD::VWMULU_VL, DL, VT: WideContainerVT,
5807 N1: OddV, N2: AllOnesVec, N3: Passthru, N4: Mask, N5: VL);
5808
5809 // Add the two together so we get
5810 // (OddV * 0xff...ff) + (OddV + EvenV)
5811 // = (OddV * 0x100...00) + EvenV
5812 // = (OddV << VecVT.getScalarSizeInBits()) + EvenV
5813 // Note the ADD_VL and VLMULU_VL should get selected as vwmaccu.vx
5814 Interleaved = DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT: WideContainerVT,
5815 N1: Interleaved, N2: OddsMul, N3: Passthru, N4: Mask, N5: VL);
5816 }
5817
5818 // Bitcast from <vscale x n * ty*2> to <vscale x 2*n x ty>
5819 MVT ResultContainerVT = MVT::getVectorVT(
5820 VT: VecVT.getVectorElementType(), // Make sure to use original type
5821 EC: VecContainerVT.getVectorElementCount().multiplyCoefficientBy(RHS: 2));
5822 Interleaved = DAG.getBitcast(VT: ResultContainerVT, V: Interleaved);
5823
5824 // Convert back to a fixed vector if needed
5825 MVT ResultVT =
5826 MVT::getVectorVT(VT: VecVT.getVectorElementType(),
5827 EC: VecVT.getVectorElementCount().multiplyCoefficientBy(RHS: 2));
5828 if (ResultVT.isFixedLengthVector())
5829 Interleaved =
5830 convertFromScalableVector(VT: ResultVT, V: Interleaved, DAG, Subtarget);
5831
5832 return Interleaved;
5833}
5834
5835// If we have a vector of bits that we want to reverse, we can use a vbrev on a
5836// larger element type, e.g. v32i1 can be reversed with a v1i32 bitreverse.
5837static SDValue lowerBitreverseShuffle(ShuffleVectorSDNode *SVN,
5838 SelectionDAG &DAG,
5839 const RISCVSubtarget &Subtarget) {
5840 SDLoc DL(SVN);
5841 MVT VT = SVN->getSimpleValueType(ResNo: 0);
5842 SDValue V = SVN->getOperand(Num: 0);
5843 unsigned NumElts = VT.getVectorNumElements();
5844
5845 assert(VT.getVectorElementType() == MVT::i1);
5846
5847 if (!ShuffleVectorInst::isReverseMask(Mask: SVN->getMask(),
5848 NumSrcElts: SVN->getMask().size()) ||
5849 !SVN->getOperand(Num: 1).isUndef())
5850 return SDValue();
5851
5852 unsigned ViaEltSize = std::max(a: (uint64_t)8, b: PowerOf2Ceil(A: NumElts));
5853 EVT ViaVT = EVT::getVectorVT(
5854 Context&: *DAG.getContext(), VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ViaEltSize), NumElements: 1);
5855 EVT ViaBitVT =
5856 EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: ViaVT.getScalarSizeInBits());
5857
5858 // If we don't have zvbb or the larger element type > ELEN, the operation will
5859 // be illegal.
5860 if (!Subtarget.getTargetLowering()->isOperationLegalOrCustom(Op: ISD::BITREVERSE,
5861 VT: ViaVT) ||
5862 !Subtarget.getTargetLowering()->isTypeLegal(VT: ViaBitVT))
5863 return SDValue();
5864
5865 // If the bit vector doesn't fit exactly into the larger element type, we need
5866 // to insert it into the larger vector and then shift up the reversed bits
5867 // afterwards to get rid of the gap introduced.
5868 if (ViaEltSize > NumElts)
5869 V = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ViaBitVT), SubVec: V, Idx: 0);
5870
5871 SDValue Res =
5872 DAG.getNode(Opcode: ISD::BITREVERSE, DL, VT: ViaVT, Operand: DAG.getBitcast(VT: ViaVT, V));
5873
5874 // Shift up the reversed bits if the vector didn't exactly fit into the larger
5875 // element type.
5876 if (ViaEltSize > NumElts)
5877 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: ViaVT, N1: Res,
5878 N2: DAG.getConstant(Val: ViaEltSize - NumElts, DL, VT: ViaVT));
5879
5880 Res = DAG.getBitcast(VT: ViaBitVT, V: Res);
5881
5882 if (ViaEltSize > NumElts)
5883 Res = DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0);
5884 return Res;
5885}
5886
5887static bool isLegalBitRotate(ArrayRef<int> Mask, EVT VT,
5888 const RISCVSubtarget &Subtarget,
5889 MVT &RotateVT, unsigned &RotateAmt) {
5890 unsigned NumElts = VT.getVectorNumElements();
5891 unsigned EltSizeInBits = VT.getScalarSizeInBits();
5892 unsigned NumSubElts;
5893 if (!ShuffleVectorInst::isBitRotateMask(Mask, EltSizeInBits, MinSubElts: 2,
5894 MaxSubElts: NumElts, NumSubElts, RotateAmt))
5895 return false;
5896 RotateVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSizeInBits * NumSubElts),
5897 NumElements: NumElts / NumSubElts);
5898
5899 // We might have a RotateVT that isn't legal, e.g. v4i64 on zve32x.
5900 return Subtarget.getTargetLowering()->isTypeLegal(VT: RotateVT);
5901}
5902
5903// Given a shuffle mask like <3, 0, 1, 2, 7, 4, 5, 6> for v8i8, we can
5904// reinterpret it as a v2i32 and rotate it right by 8 instead. We can lower this
5905// as a vror.vi if we have Zvkb, or otherwise as a vsll, vsrl and vor.
5906static SDValue lowerVECTOR_SHUFFLEAsRotate(ShuffleVectorSDNode *SVN,
5907 SelectionDAG &DAG,
5908 const RISCVSubtarget &Subtarget) {
5909 SDLoc DL(SVN);
5910
5911 EVT VT = SVN->getValueType(ResNo: 0);
5912 unsigned RotateAmt;
5913 MVT RotateVT;
5914 if (!isLegalBitRotate(Mask: SVN->getMask(), VT, Subtarget, RotateVT, RotateAmt))
5915 return SDValue();
5916
5917 SDValue Op = DAG.getBitcast(VT: RotateVT, V: SVN->getOperand(Num: 0));
5918
5919 SDValue Rotate;
5920 // A rotate of an i16 by 8 bits either direction is equivalent to a byteswap,
5921 // so canonicalize to vrev8.
5922 if (RotateVT.getScalarType() == MVT::i16 && RotateAmt == 8)
5923 Rotate = DAG.getNode(Opcode: ISD::BSWAP, DL, VT: RotateVT, Operand: Op);
5924 else
5925 Rotate = DAG.getNode(Opcode: ISD::ROTL, DL, VT: RotateVT, N1: Op,
5926 N2: DAG.getConstant(Val: RotateAmt, DL, VT: RotateVT));
5927
5928 return DAG.getBitcast(VT, V: Rotate);
5929}
5930
5931// If compiling with an exactly known VLEN, see if we can split a
5932// shuffle on m2 or larger into a small number of m1 sized shuffles
5933// which write each destination registers exactly once.
5934static SDValue lowerShuffleViaVRegSplitting(ShuffleVectorSDNode *SVN,
5935 SelectionDAG &DAG,
5936 const RISCVSubtarget &Subtarget) {
5937 SDLoc DL(SVN);
5938 MVT VT = SVN->getSimpleValueType(ResNo: 0);
5939 SDValue V1 = SVN->getOperand(Num: 0);
5940 SDValue V2 = SVN->getOperand(Num: 1);
5941 ArrayRef<int> Mask = SVN->getMask();
5942
5943 // If we don't know exact data layout, not much we can do. If this
5944 // is already m1 or smaller, no point in splitting further.
5945 const auto VLen = Subtarget.getRealVLen();
5946 if (!VLen || VT.getSizeInBits().getFixedValue() <= *VLen)
5947 return SDValue();
5948
5949 // Avoid picking up bitrotate patterns which we have a linear-in-lmul
5950 // expansion for.
5951 unsigned RotateAmt;
5952 MVT RotateVT;
5953 if (isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))
5954 return SDValue();
5955
5956 MVT ElemVT = VT.getVectorElementType();
5957 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
5958
5959 EVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
5960 MVT OneRegVT = MVT::getVectorVT(VT: ElemVT, NumElements: ElemsPerVReg);
5961 MVT M1VT = getContainerForFixedLengthVector(DAG, VT: OneRegVT, Subtarget);
5962 assert(M1VT == RISCVTargetLowering::getM1VT(M1VT));
5963 unsigned NumOpElts = M1VT.getVectorMinNumElements();
5964 unsigned NumElts = ContainerVT.getVectorMinNumElements();
5965 unsigned NumOfSrcRegs = NumElts / NumOpElts;
5966 unsigned NumOfDestRegs = NumElts / NumOpElts;
5967 // The following semantically builds up a fixed length concat_vector
5968 // of the component shuffle_vectors. We eagerly lower to scalable here
5969 // to avoid DAG combining it back to a large shuffle_vector again.
5970 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
5971 V2 = convertToScalableVector(VT: ContainerVT, V: V2, DAG, Subtarget);
5972 SmallVector<SmallVector<std::tuple<unsigned, unsigned, SmallVector<int>>>>
5973 Operands;
5974 processShuffleMasks(
5975 Mask, NumOfSrcRegs, NumOfDestRegs, NumOfUsedRegs: NumOfDestRegs,
5976 NoInputAction: [&]() { Operands.emplace_back(); },
5977 SingleInputAction: [&](ArrayRef<int> SrcSubMask, unsigned SrcVecIdx, unsigned DstVecIdx) {
5978 Operands.emplace_back().emplace_back(Args&: SrcVecIdx, UINT_MAX,
5979 Args: SmallVector<int>(SrcSubMask));
5980 },
5981 ManyInputsAction: [&](ArrayRef<int> SrcSubMask, unsigned Idx1, unsigned Idx2, bool NewReg) {
5982 if (NewReg)
5983 Operands.emplace_back();
5984 Operands.back().emplace_back(Args&: Idx1, Args&: Idx2, Args: SmallVector<int>(SrcSubMask));
5985 });
5986 assert(Operands.size() == NumOfDestRegs && "Whole vector must be processed");
5987 // Note: check that we do not emit too many shuffles here to prevent code
5988 // size explosion.
5989 // TODO: investigate, if it can be improved by extra analysis of the masks to
5990 // check if the code is more profitable.
5991 unsigned NumShuffles = std::accumulate(
5992 first: Operands.begin(), last: Operands.end(), init: 0u,
5993 binary_op: [&](unsigned N,
5994 ArrayRef<std::tuple<unsigned, unsigned, SmallVector<int>>> Data) {
5995 if (Data.empty())
5996 return N;
5997 N += Data.size();
5998 for (const auto &P : Data) {
5999 unsigned Idx2 = std::get<1>(t: P);
6000 ArrayRef<int> Mask = std::get<2>(t: P);
6001 if (Idx2 != UINT_MAX)
6002 ++N;
6003 else if (ShuffleVectorInst::isIdentityMask(Mask, NumSrcElts: Mask.size()))
6004 --N;
6005 }
6006 return N;
6007 });
6008 if ((NumOfDestRegs > 2 && NumShuffles > NumOfDestRegs) ||
6009 (NumOfDestRegs <= 2 && NumShuffles >= 4))
6010 return SDValue();
6011 auto ExtractValue = [&, &DAG = DAG](SDValue SrcVec, unsigned ExtractIdx) {
6012 SDValue SubVec = DAG.getExtractSubvector(DL, VT: M1VT, Vec: SrcVec, Idx: ExtractIdx);
6013 SubVec = convertFromScalableVector(VT: OneRegVT, V: SubVec, DAG, Subtarget);
6014 return SubVec;
6015 };
6016 auto PerformShuffle = [&, &DAG = DAG](SDValue SubVec1, SDValue SubVec2,
6017 ArrayRef<int> Mask) {
6018 SDValue SubVec = DAG.getVectorShuffle(VT: OneRegVT, dl: DL, N1: SubVec1, N2: SubVec2, Mask);
6019 return SubVec;
6020 };
6021 SDValue Vec = DAG.getUNDEF(VT: ContainerVT);
6022 for (auto [I, Data] : enumerate(First&: Operands)) {
6023 if (Data.empty())
6024 continue;
6025 SmallDenseMap<unsigned, SDValue, 4> Values;
6026 for (unsigned I : seq<unsigned>(Size: Data.size())) {
6027 const auto &[Idx1, Idx2, _] = Data[I];
6028 // If the shuffle contains permutation of odd number of elements,
6029 // Idx1 might be used already in the first iteration.
6030 //
6031 // Idx1 = shuffle Idx1, Idx2
6032 // Idx1 = shuffle Idx1, Idx3
6033 SDValue &V = Values.try_emplace(Key: Idx1).first->getSecond();
6034 if (!V)
6035 V = ExtractValue(Idx1 >= NumOfSrcRegs ? V2 : V1,
6036 (Idx1 % NumOfSrcRegs) * NumOpElts);
6037 if (Idx2 != UINT_MAX) {
6038 SDValue &V = Values.try_emplace(Key: Idx2).first->getSecond();
6039 if (!V)
6040 V = ExtractValue(Idx2 >= NumOfSrcRegs ? V2 : V1,
6041 (Idx2 % NumOfSrcRegs) * NumOpElts);
6042 }
6043 }
6044 SDValue V;
6045 for (const auto &[Idx1, Idx2, Mask] : Data) {
6046 SDValue V1 = Values.at(Val: Idx1);
6047 SDValue V2 = Idx2 == UINT_MAX ? V1 : Values.at(Val: Idx2);
6048 V = PerformShuffle(V1, V2, Mask);
6049 Values[Idx1] = V;
6050 }
6051
6052 unsigned InsertIdx = I * NumOpElts;
6053 V = convertToScalableVector(VT: M1VT, V, DAG, Subtarget);
6054 Vec = DAG.getInsertSubvector(DL, Vec, SubVec: V, Idx: InsertIdx);
6055 }
6056 return convertFromScalableVector(VT, V: Vec, DAG, Subtarget);
6057}
6058
6059// Matches a subset of compress masks with a contiguous prefix of output
6060// elements. This could be extended to allow gaps by deciding which
6061// source elements to spuriously demand.
6062static bool isCompressMask(ArrayRef<int> Mask) {
6063 int Last = -1;
6064 bool SawUndef = false;
6065 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
6066 if (M == -1) {
6067 SawUndef = true;
6068 continue;
6069 }
6070 if (SawUndef)
6071 return false;
6072 if (Idx > (unsigned)M)
6073 return false;
6074 if (M <= Last)
6075 return false;
6076 Last = M;
6077 }
6078 return true;
6079}
6080
6081/// Given a shuffle where the indices are disjoint between the two sources,
6082/// e.g.:
6083///
6084/// t2:v4i8 = vector_shuffle t0:v4i8, t1:v4i8, <2, 7, 1, 4>
6085///
6086/// Merge the two sources into one and do a single source shuffle:
6087///
6088/// t2:v4i8 = vselect t1:v4i8, t0:v4i8, <0, 1, 0, 1>
6089/// t3:v4i8 = vector_shuffle t2:v4i8, undef, <2, 3, 1, 0>
6090///
6091/// A vselect will either be merged into a masked instruction or be lowered as a
6092/// vmerge.vvm, which is cheaper than a vrgather.vv.
6093static SDValue lowerDisjointIndicesShuffle(ShuffleVectorSDNode *SVN,
6094 SelectionDAG &DAG,
6095 const RISCVSubtarget &Subtarget) {
6096 MVT VT = SVN->getSimpleValueType(ResNo: 0);
6097 MVT XLenVT = Subtarget.getXLenVT();
6098 SDLoc DL(SVN);
6099
6100 const ArrayRef<int> Mask = SVN->getMask();
6101
6102 // Work out which source each lane will come from.
6103 SmallVector<int, 16> Srcs(Mask.size(), -1);
6104
6105 for (int Idx : Mask) {
6106 if (Idx == -1)
6107 continue;
6108 unsigned SrcIdx = Idx % Mask.size();
6109 int Src = (uint32_t)Idx < Mask.size() ? 0 : 1;
6110 if (Srcs[SrcIdx] == -1)
6111 // Mark this source as using this lane.
6112 Srcs[SrcIdx] = Src;
6113 else if (Srcs[SrcIdx] != Src)
6114 // The other source is using this lane: not disjoint.
6115 return SDValue();
6116 }
6117
6118 SmallVector<SDValue> SelectMaskVals;
6119 for (int Lane : Srcs) {
6120 if (Lane == -1)
6121 SelectMaskVals.push_back(Elt: DAG.getUNDEF(VT: XLenVT));
6122 else
6123 SelectMaskVals.push_back(Elt: DAG.getConstant(Val: Lane ? 0 : 1, DL, VT: XLenVT));
6124 }
6125 MVT MaskVT = VT.changeVectorElementType(EltVT: MVT::i1);
6126 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: SelectMaskVals);
6127 SDValue Select = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask,
6128 N2: SVN->getOperand(Num: 0), N3: SVN->getOperand(Num: 1));
6129
6130 // Move all indices relative to the first source.
6131 SmallVector<int> NewMask(Mask.size());
6132 for (unsigned I = 0; I < Mask.size(); I++) {
6133 if (Mask[I] == -1)
6134 NewMask[I] = -1;
6135 else
6136 NewMask[I] = Mask[I] % Mask.size();
6137 }
6138
6139 return DAG.getVectorShuffle(VT, dl: DL, N1: Select, N2: DAG.getUNDEF(VT), Mask: NewMask);
6140}
6141
6142/// Is this mask local (i.e. elements only move within their local span), and
6143/// repeating (that is, the same rearrangement is being done within each span)?
6144static bool isLocalRepeatingShuffle(ArrayRef<int> Mask, int Span) {
6145 // Require a prefix from the original mask until the consumer code
6146 // is adjusted to rewrite the mask instead of just taking a prefix.
6147 for (auto [I, M] : enumerate(First&: Mask)) {
6148 if (M == -1)
6149 continue;
6150 if ((M / Span) != (int)(I / Span))
6151 return false;
6152 int SpanIdx = I % Span;
6153 int Expected = M % Span;
6154 if (Mask[SpanIdx] != Expected)
6155 return false;
6156 }
6157 return true;
6158}
6159
6160/// Is this mask only using elements from the first span of the input?
6161static bool isLowSourceShuffle(ArrayRef<int> Mask, int Span) {
6162 return all_of(Range&: Mask, P: [&](const auto &Idx) { return Idx == -1 || Idx < Span; });
6163}
6164
6165/// Return true for a mask which performs an arbitrary shuffle within the first
6166/// span, and then repeats that same result across all remaining spans. Note
6167/// that this doesn't check if all the inputs come from a single span!
6168static bool isSpanSplatShuffle(ArrayRef<int> Mask, int Span) {
6169 // Require a prefix from the original mask until the consumer code
6170 // is adjusted to rewrite the mask instead of just taking a prefix.
6171 for (auto [I, M] : enumerate(First&: Mask)) {
6172 if (M == -1)
6173 continue;
6174 int SpanIdx = I % Span;
6175 if (Mask[SpanIdx] != M)
6176 return false;
6177 }
6178 return true;
6179}
6180
6181/// Try to widen element type to get a new mask value for a better permutation
6182/// sequence. This doesn't try to inspect the widened mask for profitability;
6183/// we speculate the widened form is equal or better. This has the effect of
6184/// reducing mask constant sizes - allowing cheaper materialization sequences
6185/// - and index sequence sizes - reducing register pressure and materialization
6186/// cost, at the cost of (possibly) an extra VTYPE toggle.
6187static SDValue tryWidenMaskForShuffle(SDValue Op, SelectionDAG &DAG) {
6188 SDLoc DL(Op);
6189 MVT VT = Op.getSimpleValueType();
6190 MVT ScalarVT = VT.getVectorElementType();
6191 unsigned ElementSize = ScalarVT.getFixedSizeInBits();
6192 SDValue V0 = Op.getOperand(i: 0);
6193 SDValue V1 = Op.getOperand(i: 1);
6194 ArrayRef<int> Mask = cast<ShuffleVectorSDNode>(Val&: Op)->getMask();
6195
6196 // Avoid wasted work leading to isTypeLegal check failing below
6197 if (ElementSize > 32)
6198 return SDValue();
6199
6200 SmallVector<int, 8> NewMask;
6201 if (!widenShuffleMaskElts(M: Mask, NewMask))
6202 return SDValue();
6203
6204 MVT NewEltVT = VT.isFloatingPoint() ? MVT::getFloatingPointVT(BitWidth: ElementSize * 2)
6205 : MVT::getIntegerVT(BitWidth: ElementSize * 2);
6206 MVT NewVT = MVT::getVectorVT(VT: NewEltVT, NumElements: VT.getVectorNumElements() / 2);
6207 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT: NewVT))
6208 return SDValue();
6209 V0 = DAG.getBitcast(VT: NewVT, V: V0);
6210 V1 = DAG.getBitcast(VT: NewVT, V: V1);
6211 return DAG.getBitcast(VT, V: DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: V0, N2: V1, Mask: NewMask));
6212}
6213
6214static SDValue lowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG,
6215 const RISCVSubtarget &Subtarget) {
6216 SDValue V1 = Op.getOperand(i: 0);
6217 SDValue V2 = Op.getOperand(i: 1);
6218 SDLoc DL(Op);
6219 MVT XLenVT = Subtarget.getXLenVT();
6220 MVT VT = Op.getSimpleValueType();
6221 unsigned NumElts = VT.getVectorNumElements();
6222 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: Op.getNode());
6223
6224 if (VT.getVectorElementType() == MVT::i1) {
6225 // Lower to a vror.vi of a larger element type if possible before we promote
6226 // i1s to i8s.
6227 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6228 return V;
6229 if (SDValue V = lowerBitreverseShuffle(SVN, DAG, Subtarget))
6230 return V;
6231
6232 // Promote i1 shuffle to i8 shuffle.
6233 MVT WidenVT = MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount());
6234 V1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: V1);
6235 V2 = V2.isUndef() ? DAG.getUNDEF(VT: WidenVT)
6236 : DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: V2);
6237 SDValue Shuffled = DAG.getVectorShuffle(VT: WidenVT, dl: DL, N1: V1, N2: V2, Mask: SVN->getMask());
6238 return DAG.getSetCC(DL, VT, LHS: Shuffled, RHS: DAG.getConstant(Val: 0, DL, VT: WidenVT),
6239 Cond: ISD::SETNE);
6240 }
6241
6242 MVT ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
6243
6244 // Store the return value in a single variable instead of structured bindings
6245 // so that we can pass it to GetSlide below, which cannot capture structured
6246 // bindings until C++20.
6247 auto TrueMaskVL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
6248 auto [TrueMask, VL] = TrueMaskVL;
6249
6250 if (SVN->isSplat()) {
6251 const int Lane = SVN->getSplatIndex();
6252 if (Lane >= 0) {
6253 MVT SVT = VT.getVectorElementType();
6254
6255 // Turn splatted vector load into a strided load with an X0 stride.
6256 SDValue V = V1;
6257 // Peek through CONCAT_VECTORS as VectorCombine can concat a vector
6258 // with undef.
6259 // FIXME: Peek through INSERT_SUBVECTOR, EXTRACT_SUBVECTOR, bitcasts?
6260 int Offset = Lane;
6261 if (V.getOpcode() == ISD::CONCAT_VECTORS) {
6262 int OpElements =
6263 V.getOperand(i: 0).getSimpleValueType().getVectorNumElements();
6264 V = V.getOperand(i: Offset / OpElements);
6265 Offset %= OpElements;
6266 }
6267
6268 // We need to ensure the load isn't atomic or volatile.
6269 if (ISD::isNormalLoad(N: V.getNode()) && cast<LoadSDNode>(Val&: V)->isSimple()) {
6270 auto *Ld = cast<LoadSDNode>(Val&: V);
6271 Offset *= SVT.getStoreSize();
6272 SDValue NewAddr = DAG.getMemBasePlusOffset(
6273 Base: Ld->getBasePtr(), Offset: TypeSize::getFixed(ExactSize: Offset), DL);
6274
6275 // If this is SEW=64 on RV32, use a strided load with a stride of x0.
6276 if (SVT.isInteger() && SVT.bitsGT(VT: XLenVT)) {
6277 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
6278 SDValue IntID =
6279 DAG.getTargetConstant(Val: Intrinsic::riscv_vlse, DL, VT: XLenVT);
6280 SDValue Ops[] = {Ld->getChain(),
6281 IntID,
6282 DAG.getUNDEF(VT: ContainerVT),
6283 NewAddr,
6284 DAG.getRegister(Reg: RISCV::X0, VT: XLenVT),
6285 VL};
6286 SDValue NewLoad = DAG.getMemIntrinsicNode(
6287 Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT: SVT,
6288 MMO: DAG.getMachineFunction().getMachineMemOperand(
6289 MMO: Ld->getMemOperand(), Offset, Size: SVT.getStoreSize()));
6290 DAG.makeEquivalentMemoryOrdering(OldLoad: Ld, NewMemOp: NewLoad);
6291 return convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
6292 }
6293
6294 MVT SplatVT = ContainerVT;
6295
6296 // f16 with zvfhmin and bf16 need to use an integer scalar load.
6297 if (SVT == MVT::bf16 ||
6298 (SVT == MVT::f16 && !Subtarget.hasStdExtZfh())) {
6299 SVT = MVT::i16;
6300 SplatVT = ContainerVT.changeVectorElementType(EltVT: SVT);
6301 }
6302
6303 // Otherwise use a scalar load and splat. This will give the best
6304 // opportunity to fold a splat into the operation. ISel can turn it into
6305 // the x0 strided load if we aren't able to fold away the select.
6306 if (SVT.isFloatingPoint())
6307 V = DAG.getLoad(VT: SVT, dl: DL, Chain: Ld->getChain(), Ptr: NewAddr,
6308 PtrInfo: Ld->getPointerInfo().getWithOffset(O: Offset),
6309 Alignment: Ld->getBaseAlign(), MMOFlags: Ld->getMemOperand()->getFlags());
6310 else
6311 V = DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: DL, VT: XLenVT, Chain: Ld->getChain(), Ptr: NewAddr,
6312 PtrInfo: Ld->getPointerInfo().getWithOffset(O: Offset), MemVT: SVT,
6313 Alignment: Ld->getBaseAlign(),
6314 MMOFlags: Ld->getMemOperand()->getFlags());
6315 DAG.makeEquivalentMemoryOrdering(OldLoad: Ld, NewMemOp: V);
6316
6317 unsigned Opc = SplatVT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL
6318 : RISCVISD::VMV_V_X_VL;
6319 SDValue Splat =
6320 DAG.getNode(Opcode: Opc, DL, VT: SplatVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: V, N3: VL);
6321 Splat = DAG.getBitcast(VT: ContainerVT, V: Splat);
6322 return convertFromScalableVector(VT, V: Splat, DAG, Subtarget);
6323 }
6324
6325 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
6326 assert(Lane < (int)NumElts && "Unexpected lane!");
6327 SDValue Gather = DAG.getNode(Opcode: RISCVISD::VRGATHER_VX_VL, DL, VT: ContainerVT,
6328 N1: V1, N2: DAG.getConstant(Val: Lane, DL, VT: XLenVT),
6329 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
6330 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6331 }
6332 }
6333
6334 // For exact VLEN m2 or greater, try to split to m1 operations if we
6335 // can split cleanly.
6336 if (SDValue V = lowerShuffleViaVRegSplitting(SVN, DAG, Subtarget))
6337 return V;
6338
6339 ArrayRef<int> Mask = SVN->getMask();
6340
6341 if (SDValue V =
6342 lowerVECTOR_SHUFFLEAsVSlide1(DL, VT, V1, V2, Mask, Subtarget, DAG))
6343 return V;
6344
6345 if (SDValue V =
6346 lowerVECTOR_SHUFFLEAsVSlidedown(DL, VT, V1, V2, Mask, Subtarget, DAG))
6347 return V;
6348
6349 // A bitrotate will be one instruction on Zvkb, so try to lower to it first if
6350 // available.
6351 if (Subtarget.hasStdExtZvkb())
6352 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6353 return V;
6354
6355 if (ShuffleVectorInst::isReverseMask(Mask, NumSrcElts: NumElts) && V2.isUndef() &&
6356 NumElts != 2)
6357 return DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT, Operand: V1);
6358
6359 // If this is a deinterleave(2,4,8) and we can widen the vector, then we can
6360 // use shift and truncate to perform the shuffle.
6361 // TODO: For Factor=6, we can perform the first step of the deinterleave via
6362 // shift-and-trunc reducing total cost for everything except an mf8 result.
6363 // TODO: For Factor=4,8, we can do the same when the ratio isn't high enough
6364 // to do the entire operation.
6365 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6366 const unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6367 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6368 for (unsigned Factor = 2; Factor <= MaxFactor; Factor <<= 1) {
6369 unsigned Index = 0;
6370 if (ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor, Index) &&
6371 1 < count_if(Range&: Mask, P: [](int Idx) { return Idx != -1; })) {
6372 if (SDValue Src = getSingleShuffleSrc(VT, V1, V2))
6373 return getDeinterleaveShiftAndTrunc(DL, VT, Src, Factor, Index, DAG);
6374 if (1 < count_if(Range&: Mask,
6375 P: [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6376 1 < count_if(Range&: Mask, P: [&Mask](int Idx) {
6377 return Idx >= (int)Mask.size();
6378 })) {
6379 // Narrow each source and concatenate them.
6380 // FIXME: For small LMUL it is better to concatenate first.
6381 MVT EltVT = VT.getVectorElementType();
6382 auto EltCnt = VT.getVectorElementCount();
6383 MVT SubVT =
6384 MVT::getVectorVT(VT: EltVT, EC: EltCnt.divideCoefficientBy(RHS: Factor));
6385
6386 SDValue Lo =
6387 getDeinterleaveShiftAndTrunc(DL, VT: SubVT, Src: V1, Factor, Index, DAG);
6388 SDValue Hi =
6389 getDeinterleaveShiftAndTrunc(DL, VT: SubVT, Src: V2, Factor, Index, DAG);
6390
6391 SDValue Concat =
6392 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL,
6393 VT: SubVT.getDoubleNumVectorElementsVT(), N1: Lo, N2: Hi);
6394 if (Factor == 2)
6395 return Concat;
6396
6397 SDValue Vec = DAG.getUNDEF(VT);
6398 return DAG.getInsertSubvector(DL, Vec, SubVec: Concat, Idx: 0);
6399 }
6400 }
6401 }
6402 }
6403
6404 // If this is a deinterleave(2), try using vunzip{a,b}. This mostly catches
6405 // e64 which can't match above.
6406 unsigned Index = 0;
6407 if (Subtarget.hasVendorXRivosVizip() &&
6408 ShuffleVectorInst::isDeInterleaveMaskOfFactor(Mask, Factor: 2, Index) &&
6409 1 < count_if(Range&: Mask, P: [](int Idx) { return Idx != -1; })) {
6410 unsigned Opc =
6411 Index == 0 ? RISCVISD::RI_VUNZIP2A_VL : RISCVISD::RI_VUNZIP2B_VL;
6412 if (V2.isUndef())
6413 return lowerVZIP(Opc, Op0: V1, Op1: V2, DL, DAG, Subtarget);
6414 if (auto VLEN = Subtarget.getRealVLen();
6415 VLEN && VT.getSizeInBits().getKnownMinValue() % *VLEN == 0)
6416 return lowerVZIP(Opc, Op0: V1, Op1: V2, DL, DAG, Subtarget);
6417 if (SDValue Src = foldConcatVector(V1, V2)) {
6418 EVT NewVT = VT.getDoubleNumVectorElementsVT();
6419 Src = DAG.getExtractSubvector(DL, VT: NewVT, Vec: Src, Idx: 0);
6420 SDValue Res =
6421 lowerVZIP(Opc, Op0: Src, Op1: DAG.getUNDEF(VT: NewVT), DL, DAG, Subtarget);
6422 return DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0);
6423 }
6424 // Deinterleave each source and concatenate them, or concat first, then
6425 // deinterleave.
6426 if (1 < count_if(Range&: Mask,
6427 P: [&Mask](int Idx) { return Idx < (int)Mask.size(); }) &&
6428 1 < count_if(Range&: Mask,
6429 P: [&Mask](int Idx) { return Idx >= (int)Mask.size(); })) {
6430
6431 const unsigned EltSize = VT.getScalarSizeInBits();
6432 const unsigned MinVLMAX = Subtarget.getRealMinVLen() / EltSize;
6433 if (NumElts < MinVLMAX) {
6434 MVT ConcatVT = VT.getDoubleNumVectorElementsVT();
6435 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, N1: V1, N2: V2);
6436 SDValue Res =
6437 lowerVZIP(Opc, Op0: Concat, Op1: DAG.getUNDEF(VT: ConcatVT), DL, DAG, Subtarget);
6438 return DAG.getExtractSubvector(DL, VT, Vec: Res, Idx: 0);
6439 }
6440
6441 SDValue Lo = lowerVZIP(Opc, Op0: V1, Op1: DAG.getUNDEF(VT), DL, DAG, Subtarget);
6442 SDValue Hi = lowerVZIP(Opc, Op0: V2, Op1: DAG.getUNDEF(VT), DL, DAG, Subtarget);
6443
6444 MVT SubVT = VT.getHalfNumVectorElementsVT();
6445 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT,
6446 N1: DAG.getExtractSubvector(DL, VT: SubVT, Vec: Lo, Idx: 0),
6447 N2: DAG.getExtractSubvector(DL, VT: SubVT, Vec: Hi, Idx: 0));
6448 }
6449 }
6450
6451 if (SDValue V =
6452 lowerVECTOR_SHUFFLEAsVSlideup(DL, VT, V1, V2, Mask, Subtarget, DAG))
6453 return V;
6454
6455 // Detect an interleave shuffle and lower to
6456 // (vmaccu.vx (vwaddu.vx lohalf(V1), lohalf(V2)), lohalf(V2), (2^eltbits - 1))
6457 int EvenSrc, OddSrc;
6458 if (isInterleaveShuffle(Mask, VT, EvenSrc, OddSrc, Subtarget) &&
6459 !(NumElts == 2 &&
6460 ShuffleVectorInst::isSingleSourceMask(Mask, NumSrcElts: Mask.size()))) {
6461 // Extract the halves of the vectors.
6462 MVT HalfVT = VT.getHalfNumVectorElementsVT();
6463
6464 // Recognize if one half is actually undef; the matching above will
6465 // otherwise reuse the even stream for the undef one. This improves
6466 // spread(2) shuffles.
6467 bool LaneIsUndef[2] = { true, true};
6468 for (const auto &[Idx, M] : enumerate(First&: Mask))
6469 LaneIsUndef[Idx % 2] &= (M == -1);
6470
6471 int Size = Mask.size();
6472 SDValue EvenV, OddV;
6473 if (LaneIsUndef[0]) {
6474 EvenV = DAG.getUNDEF(VT: HalfVT);
6475 } else {
6476 assert(EvenSrc >= 0 && "Undef source?");
6477 EvenV = (EvenSrc / Size) == 0 ? V1 : V2;
6478 EvenV = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: EvenV, Idx: EvenSrc % Size);
6479 }
6480
6481 if (LaneIsUndef[1]) {
6482 OddV = DAG.getUNDEF(VT: HalfVT);
6483 } else {
6484 assert(OddSrc >= 0 && "Undef source?");
6485 OddV = (OddSrc / Size) == 0 ? V1 : V2;
6486 OddV = DAG.getExtractSubvector(DL, VT: HalfVT, Vec: OddV, Idx: OddSrc % Size);
6487 }
6488
6489 // Prefer vzip2a if available.
6490 // TODO: Extend to matching zip2b if EvenSrc and OddSrc allow.
6491 if (Subtarget.hasVendorXRivosVizip()) {
6492 EvenV = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: EvenV, Idx: 0);
6493 OddV = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: OddV, Idx: 0);
6494 return lowerVZIP(Opc: RISCVISD::RI_VZIP2A_VL, Op0: EvenV, Op1: OddV, DL, DAG, Subtarget);
6495 }
6496 return getWideningInterleave(EvenV, OddV, DL, DAG, Subtarget);
6497 }
6498
6499 // Recognize a pattern which can handled via a pair of vslideup/vslidedown
6500 // instructions (in any combination) with masking on the second instruction.
6501 // Also handles masked slides into an identity source, and single slides
6502 // without masking. Avoid matching bit rotates (which are not also element
6503 // rotates) as slide pairs. This is a performance heuristic, not a
6504 // functional check.
6505 std::array<std::pair<int, int>, 2> SrcInfo;
6506 unsigned RotateAmt;
6507 MVT RotateVT;
6508 if (::isMaskedSlidePair(Mask, SrcInfo) &&
6509 (isElementRotate(SrcInfo, NumElts) ||
6510 !isLegalBitRotate(Mask, VT, Subtarget, RotateVT, RotateAmt))) {
6511 SDValue Sources[2];
6512 auto GetSourceFor = [&](const std::pair<int, int> &Info) {
6513 int SrcIdx = Info.first;
6514 assert(SrcIdx == 0 || SrcIdx == 1);
6515 SDValue &Src = Sources[SrcIdx];
6516 if (!Src) {
6517 SDValue SrcV = SrcIdx == 0 ? V1 : V2;
6518 Src = convertToScalableVector(VT: ContainerVT, V: SrcV, DAG, Subtarget);
6519 }
6520 return Src;
6521 };
6522 auto GetSlide = [&](const std::pair<int, int> &Src, SDValue Mask,
6523 SDValue Passthru) {
6524 auto [TrueMask, VL] = TrueMaskVL;
6525 SDValue SrcV = GetSourceFor(Src);
6526 int SlideAmt = Src.second;
6527 if (SlideAmt == 0) {
6528 // Should never be second operation
6529 assert(Mask == TrueMask);
6530 return SrcV;
6531 }
6532 if (SlideAmt < 0)
6533 return getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: SrcV,
6534 Offset: DAG.getConstant(Val: -SlideAmt, DL, VT: XLenVT), Mask, VL,
6535 Policy: RISCVVType::TAIL_AGNOSTIC);
6536 return getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru, Op: SrcV,
6537 Offset: DAG.getConstant(Val: SlideAmt, DL, VT: XLenVT), Mask, VL,
6538 Policy: RISCVVType::TAIL_AGNOSTIC);
6539 };
6540
6541 if (SrcInfo[1].first == -1) {
6542 SDValue Res = DAG.getUNDEF(VT: ContainerVT);
6543 Res = GetSlide(SrcInfo[0], TrueMask, Res);
6544 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
6545 }
6546
6547 if (Subtarget.hasVendorXRivosVizip()) {
6548 bool TryWiden = false;
6549 unsigned Factor;
6550 if (isZipEven(SrcInfo, Mask, Factor)) {
6551 if (Factor == 1) {
6552 SDValue Src1 = SrcInfo[0].first == 0 ? V1 : V2;
6553 SDValue Src2 = SrcInfo[1].first == 0 ? V1 : V2;
6554 return lowerVZIP(Opc: RISCVISD::RI_VZIPEVEN_VL, Op0: Src1, Op1: Src2, DL, DAG,
6555 Subtarget);
6556 }
6557 TryWiden = true;
6558 }
6559 if (isZipOdd(SrcInfo, Mask, Factor)) {
6560 if (Factor == 1) {
6561 SDValue Src1 = SrcInfo[1].first == 0 ? V1 : V2;
6562 SDValue Src2 = SrcInfo[0].first == 0 ? V1 : V2;
6563 return lowerVZIP(Opc: RISCVISD::RI_VZIPODD_VL, Op0: Src1, Op1: Src2, DL, DAG,
6564 Subtarget);
6565 }
6566 TryWiden = true;
6567 }
6568 // If we found a widening oppurtunity which would let us form a
6569 // zipeven or zipodd, use the generic code to widen the shuffle
6570 // and recurse through this logic.
6571 if (TryWiden)
6572 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
6573 return V;
6574 }
6575
6576 // Build the mask. Note that vslideup unconditionally preserves elements
6577 // below the slide amount in the destination, and thus those elements are
6578 // undefined in the mask. If the mask ends up all true (or undef), it
6579 // will be folded away by general logic.
6580 SmallVector<SDValue> MaskVals;
6581 for (const auto &[Idx, M] : enumerate(First&: Mask)) {
6582 if (M < 0 ||
6583 (SrcInfo[1].second > 0 && Idx < (unsigned)SrcInfo[1].second)) {
6584 MaskVals.push_back(Elt: DAG.getUNDEF(VT: XLenVT));
6585 continue;
6586 }
6587 int Src = M >= (int)NumElts;
6588 int Diff = (int)Idx - (M % NumElts);
6589 bool C = Src == SrcInfo[1].first && Diff == SrcInfo[1].second;
6590 assert(C ^ (Src == SrcInfo[0].first && Diff == SrcInfo[0].second) &&
6591 "Must match exactly one of the two slides");
6592 MaskVals.push_back(Elt: DAG.getConstant(Val: C, DL, VT: XLenVT));
6593 }
6594 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
6595 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
6596 SDValue SelectMask = convertToScalableVector(
6597 VT: ContainerVT.changeVectorElementType(EltVT: MVT::i1),
6598 V: DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals), DAG, Subtarget);
6599
6600 SDValue Res = DAG.getUNDEF(VT: ContainerVT);
6601 Res = GetSlide(SrcInfo[0], TrueMask, Res);
6602 Res = GetSlide(SrcInfo[1], SelectMask, Res);
6603 return convertFromScalableVector(VT, V: Res, DAG, Subtarget);
6604 }
6605
6606 // Handle any remaining single source shuffles
6607 assert(!V1.isUndef() && "Unexpected shuffle canonicalization");
6608 if (V2.isUndef()) {
6609 // We might be able to express the shuffle as a bitrotate. But even if we
6610 // don't have Zvkb and have to expand, the expanded sequence of approx. 2
6611 // shifts and a vor will have a higher throughput than a vrgather.
6612 if (SDValue V = lowerVECTOR_SHUFFLEAsRotate(SVN, DAG, Subtarget))
6613 return V;
6614
6615 if (SDValue V = lowerVECTOR_SHUFFLEAsVRGatherVX(SVN, Subtarget, DAG))
6616 return V;
6617
6618 // Match a spread(4,8) which can be done via extend and shift. Spread(2)
6619 // is fully covered in interleave(2) above, so it is ignored here.
6620 if (VT.getScalarSizeInBits() < Subtarget.getELen()) {
6621 unsigned MaxFactor = Subtarget.getELen() / VT.getScalarSizeInBits();
6622 assert(MaxFactor == 2 || MaxFactor == 4 || MaxFactor == 8);
6623 for (unsigned Factor = 4; Factor <= MaxFactor; Factor <<= 1) {
6624 unsigned Index;
6625 if (RISCVTargetLowering::isSpreadMask(Mask, Factor, Index)) {
6626 MVT NarrowVT =
6627 MVT::getVectorVT(VT: VT.getVectorElementType(), NumElements: NumElts / Factor);
6628 SDValue Src = DAG.getExtractSubvector(DL, VT: NarrowVT, Vec: V1, Idx: 0);
6629 return getWideningSpread(V: Src, Factor, Index, DL, DAG);
6630 }
6631 }
6632 }
6633
6634 // If only a prefix of the source elements influence a prefix of the
6635 // destination elements, try to see if we can reduce the required LMUL
6636 unsigned MinVLen = Subtarget.getRealMinVLen();
6637 unsigned MinVLMAX = MinVLen / VT.getScalarSizeInBits();
6638 if (NumElts > MinVLMAX) {
6639 unsigned MaxIdx = 0;
6640 for (auto [I, M] : enumerate(First&: Mask)) {
6641 if (M == -1)
6642 continue;
6643 MaxIdx = std::max(a: std::max(a: (unsigned)I, b: (unsigned)M), b: MaxIdx);
6644 }
6645 unsigned NewNumElts =
6646 std::max(a: (uint64_t)MinVLMAX, b: PowerOf2Ceil(A: MaxIdx + 1));
6647 if (NewNumElts != NumElts) {
6648 MVT NewVT = MVT::getVectorVT(VT: VT.getVectorElementType(), NumElements: NewNumElts);
6649 V1 = DAG.getExtractSubvector(DL, VT: NewVT, Vec: V1, Idx: 0);
6650 SDValue Res = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT: NewVT),
6651 Mask: Mask.take_front(N: NewNumElts));
6652 return DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: Res, Idx: 0);
6653 }
6654 }
6655
6656 // Before hitting generic lowering fallbacks, try to widen the mask
6657 // to a wider SEW.
6658 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
6659 return V;
6660
6661 // Can we generate a vcompress instead of a vrgather? These scale better
6662 // at high LMUL, at the cost of not being able to fold a following select
6663 // into them. The mask constants are also smaller than the index vector
6664 // constants, and thus easier to materialize.
6665 if (isCompressMask(Mask)) {
6666 SmallVector<SDValue> MaskVals(NumElts,
6667 DAG.getConstant(Val: false, DL, VT: XLenVT));
6668 for (auto Idx : Mask) {
6669 if (Idx == -1)
6670 break;
6671 assert(Idx >= 0 && (unsigned)Idx < NumElts);
6672 MaskVals[Idx] = DAG.getConstant(Val: true, DL, VT: XLenVT);
6673 }
6674 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
6675 SDValue CompressMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
6676 return DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT, N1: V1, N2: CompressMask,
6677 N3: DAG.getUNDEF(VT));
6678 }
6679
6680 if (VT.getScalarSizeInBits() == 8 &&
6681 any_of(Range&: Mask, P: [&](const auto &Idx) { return Idx > 255; })) {
6682 // On such a vector we're unable to use i8 as the index type.
6683 // FIXME: We could promote the index to i16 and use vrgatherei16, but that
6684 // may involve vector splitting if we're already at LMUL=8, or our
6685 // user-supplied maximum fixed-length LMUL.
6686 return SDValue();
6687 }
6688
6689 // Base case for the two operand recursion below - handle the worst case
6690 // single source shuffle.
6691 unsigned GatherVVOpc = RISCVISD::VRGATHER_VV_VL;
6692 MVT IndexVT = VT.changeTypeToInteger();
6693 // Since we can't introduce illegal index types at this stage, use i16 and
6694 // vrgatherei16 if the corresponding index type for plain vrgather is greater
6695 // than XLenVT.
6696 if (IndexVT.getScalarType().bitsGT(VT: XLenVT)) {
6697 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
6698 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
6699 }
6700
6701 // If the mask allows, we can do all the index computation in 16 bits. This
6702 // requires less work and less register pressure at high LMUL, and creates
6703 // smaller constants which may be cheaper to materialize.
6704 if (IndexVT.getScalarType().bitsGT(VT: MVT::i16) && isUInt<16>(x: NumElts - 1) &&
6705 (IndexVT.getSizeInBits() / Subtarget.getRealMinVLen()) > 1) {
6706 GatherVVOpc = RISCVISD::VRGATHEREI16_VV_VL;
6707 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
6708 }
6709
6710 MVT IndexContainerVT =
6711 ContainerVT.changeVectorElementType(EltVT: IndexVT.getScalarType());
6712
6713 V1 = convertToScalableVector(VT: ContainerVT, V: V1, DAG, Subtarget);
6714 SmallVector<SDValue> GatherIndicesLHS;
6715 for (int MaskIndex : Mask) {
6716 bool IsLHSIndex = MaskIndex < (int)NumElts && MaskIndex >= 0;
6717 GatherIndicesLHS.push_back(Elt: IsLHSIndex
6718 ? DAG.getConstant(Val: MaskIndex, DL, VT: XLenVT)
6719 : DAG.getUNDEF(VT: XLenVT));
6720 }
6721 SDValue LHSIndices = DAG.getBuildVector(VT: IndexVT, DL, Ops: GatherIndicesLHS);
6722 LHSIndices =
6723 convertToScalableVector(VT: IndexContainerVT, V: LHSIndices, DAG, Subtarget);
6724 // At m1 and less, there's no point trying any of the high LMUL splitting
6725 // techniques. TODO: Should we reconsider this for DLEN < VLEN?
6726 if (NumElts <= MinVLMAX) {
6727 SDValue Gather = DAG.getNode(Opcode: GatherVVOpc, DL, VT: ContainerVT, N1: V1, N2: LHSIndices,
6728 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
6729 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6730 }
6731
6732 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
6733 EVT SubIndexVT = M1VT.changeVectorElementType(EltVT: IndexVT.getScalarType());
6734 auto [InnerTrueMask, InnerVL] =
6735 getDefaultScalableVLOps(VecVT: M1VT, DL, DAG, Subtarget);
6736 int N =
6737 ContainerVT.getVectorMinNumElements() / M1VT.getVectorMinNumElements();
6738 assert(isPowerOf2_32(N) && N <= 8);
6739
6740 // If we have a locally repeating mask, then we can reuse the first
6741 // register in the index register group for all registers within the
6742 // source register group. TODO: This generalizes to m2, and m4.
6743 if (isLocalRepeatingShuffle(Mask, Span: MinVLMAX)) {
6744 SDValue SubIndex = DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
6745 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
6746 for (int i = 0; i < N; i++) {
6747 unsigned SubIdx = M1VT.getVectorMinNumElements() * i;
6748 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: SubIdx);
6749 SDValue SubVec =
6750 DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
6751 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
6752 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec, Idx: SubIdx);
6753 }
6754 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6755 }
6756
6757 // If we have a shuffle which only uses the first register in our source
6758 // register group, and repeats the same index across all spans, we can
6759 // use a single vrgather (and possibly some register moves).
6760 // TODO: This can be generalized for m2 or m4, or for any shuffle for
6761 // which we can do a linear number of shuffles to form an m1 which
6762 // contains all the output elements.
6763 if (isLowSourceShuffle(Mask, Span: MinVLMAX) &&
6764 isSpanSplatShuffle(Mask, Span: MinVLMAX)) {
6765 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: 0);
6766 SDValue SubIndex = DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
6767 SDValue SubVec = DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
6768 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
6769 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
6770 for (int i = 0; i < N; i++)
6771 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec,
6772 Idx: M1VT.getVectorMinNumElements() * i);
6773 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6774 }
6775
6776 // If we have a shuffle which only uses the first register in our
6777 // source register group, we can do a linear number of m1 vrgathers
6778 // reusing the same source register (but with different indices)
6779 // TODO: This can be generalized for m2 or m4, or for any shuffle
6780 // for which we can do a vslidedown followed by this expansion.
6781 if (isLowSourceShuffle(Mask, Span: MinVLMAX)) {
6782 SDValue SlideAmt =
6783 DAG.getElementCount(DL, VT: XLenVT, EC: M1VT.getVectorElementCount());
6784 SDValue SubV1 = DAG.getExtractSubvector(DL, VT: M1VT, Vec: V1, Idx: 0);
6785 SDValue Gather = DAG.getUNDEF(VT: ContainerVT);
6786 for (int i = 0; i < N; i++) {
6787 if (i != 0)
6788 LHSIndices = getVSlidedown(DAG, Subtarget, DL, VT: IndexContainerVT,
6789 Passthru: DAG.getUNDEF(VT: IndexContainerVT), Op: LHSIndices,
6790 Offset: SlideAmt, Mask: TrueMask, VL);
6791 SDValue SubIndex =
6792 DAG.getExtractSubvector(DL, VT: SubIndexVT, Vec: LHSIndices, Idx: 0);
6793 SDValue SubVec =
6794 DAG.getNode(Opcode: GatherVVOpc, DL, VT: M1VT, N1: SubV1, N2: SubIndex,
6795 N3: DAG.getUNDEF(VT: M1VT), N4: InnerTrueMask, N5: InnerVL);
6796 Gather = DAG.getInsertSubvector(DL, Vec: Gather, SubVec,
6797 Idx: M1VT.getVectorMinNumElements() * i);
6798 }
6799 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6800 }
6801
6802 // Fallback to generic vrgather if we can't find anything better.
6803 // On many machines, this will be O(LMUL^2)
6804 SDValue Gather = DAG.getNode(Opcode: GatherVVOpc, DL, VT: ContainerVT, N1: V1, N2: LHSIndices,
6805 N3: DAG.getUNDEF(VT: ContainerVT), N4: TrueMask, N5: VL);
6806 return convertFromScalableVector(VT, V: Gather, DAG, Subtarget);
6807 }
6808
6809 // As a backup, shuffles can be lowered via a vrgather instruction, possibly
6810 // merged with a second vrgather.
6811 SmallVector<int> ShuffleMaskLHS, ShuffleMaskRHS;
6812
6813 // Now construct the mask that will be used by the blended vrgather operation.
6814 // Construct the appropriate indices into each vector.
6815 for (int MaskIndex : Mask) {
6816 bool IsLHSOrUndefIndex = MaskIndex < (int)NumElts;
6817 ShuffleMaskLHS.push_back(Elt: IsLHSOrUndefIndex && MaskIndex >= 0
6818 ? MaskIndex : -1);
6819 ShuffleMaskRHS.push_back(Elt: IsLHSOrUndefIndex ? -1 : (MaskIndex - NumElts));
6820 }
6821
6822 // If the mask indices are disjoint between the two sources, we can lower it
6823 // as a vselect + a single source vrgather.vv. Don't do this if we think the
6824 // operands may end up being lowered to something cheaper than a vrgather.vv.
6825 if (!DAG.isSplatValue(V: V2) && !DAG.isSplatValue(V: V1) &&
6826 !ShuffleVectorSDNode::isSplatMask(Mask: ShuffleMaskLHS) &&
6827 !ShuffleVectorSDNode::isSplatMask(Mask: ShuffleMaskRHS) &&
6828 !ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskLHS, NumSrcElts: NumElts) &&
6829 !ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskRHS, NumSrcElts: NumElts))
6830 if (SDValue V = lowerDisjointIndicesShuffle(SVN, DAG, Subtarget))
6831 return V;
6832
6833 // Before hitting generic lowering fallbacks, try to widen the mask
6834 // to a wider SEW.
6835 if (SDValue V = tryWidenMaskForShuffle(Op, DAG))
6836 return V;
6837
6838 // Try to pick a profitable operand order.
6839 bool SwapOps = DAG.isSplatValue(V: V2) && !DAG.isSplatValue(V: V1);
6840 SwapOps = SwapOps ^ ShuffleVectorInst::isIdentityMask(Mask: ShuffleMaskRHS, NumSrcElts: NumElts);
6841
6842 // Recursively invoke lowering for each operand if we had two
6843 // independent single source shuffles, and then combine the result via a
6844 // vselect. Note that the vselect will likely be folded back into the
6845 // second permute (vrgather, or other) by the post-isel combine.
6846 V1 = DAG.getVectorShuffle(VT, dl: DL, N1: V1, N2: DAG.getUNDEF(VT), Mask: ShuffleMaskLHS);
6847 V2 = DAG.getVectorShuffle(VT, dl: DL, N1: V2, N2: DAG.getUNDEF(VT), Mask: ShuffleMaskRHS);
6848
6849 SmallVector<SDValue> MaskVals;
6850 for (int MaskIndex : Mask) {
6851 bool SelectMaskVal = (MaskIndex < (int)NumElts) ^ !SwapOps;
6852 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
6853 }
6854
6855 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
6856 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, NumElements: NumElts);
6857 SDValue SelectMask = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
6858
6859 if (SwapOps)
6860 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: V1, N3: V2);
6861 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: SelectMask, N2: V2, N3: V1);
6862}
6863
6864bool RISCVTargetLowering::isShuffleMaskLegal(ArrayRef<int> M, EVT VT) const {
6865 // Only support legal VTs for other shuffles for now.
6866 if (!isTypeLegal(VT) || !Subtarget.hasVInstructions())
6867 return false;
6868
6869 // Support splats for any type. These should type legalize well.
6870 if (ShuffleVectorSDNode::isSplatMask(Mask: M))
6871 return true;
6872
6873 const unsigned NumElts = M.size();
6874 MVT SVT = VT.getSimpleVT();
6875
6876 // Not for i1 vectors.
6877 if (SVT.getScalarType() == MVT::i1)
6878 return false;
6879
6880 std::array<std::pair<int, int>, 2> SrcInfo;
6881 int Dummy1, Dummy2;
6882 return ShuffleVectorInst::isReverseMask(Mask: M, NumSrcElts: NumElts) ||
6883 (::isMaskedSlidePair(Mask: M, SrcInfo) &&
6884 isElementRotate(SrcInfo, NumElts)) ||
6885 isInterleaveShuffle(Mask: M, VT: SVT, EvenSrc&: Dummy1, OddSrc&: Dummy2, Subtarget);
6886}
6887
6888// Lower CTLZ_ZERO_UNDEF or CTTZ_ZERO_UNDEF by converting to FP and extracting
6889// the exponent.
6890SDValue
6891RISCVTargetLowering::lowerCTLZ_CTTZ_ZERO_UNDEF(SDValue Op,
6892 SelectionDAG &DAG) const {
6893 MVT VT = Op.getSimpleValueType();
6894 unsigned EltSize = VT.getScalarSizeInBits();
6895 SDValue Src = Op.getOperand(i: 0);
6896 SDLoc DL(Op);
6897 MVT ContainerVT = VT;
6898
6899 SDValue Mask, VL;
6900 if (Op->isVPOpcode()) {
6901 Mask = Op.getOperand(i: 1);
6902 if (VT.isFixedLengthVector())
6903 Mask = convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Mask, DAG,
6904 Subtarget);
6905 VL = Op.getOperand(i: 2);
6906 }
6907
6908 // We choose FP type that can represent the value if possible. Otherwise, we
6909 // use rounding to zero conversion for correct exponent of the result.
6910 // TODO: Use f16 for i8 when possible?
6911 MVT FloatEltVT = (EltSize >= 32) ? MVT::f64 : MVT::f32;
6912 if (!isTypeLegal(VT: MVT::getVectorVT(VT: FloatEltVT, EC: VT.getVectorElementCount())))
6913 FloatEltVT = MVT::f32;
6914 MVT FloatVT = MVT::getVectorVT(VT: FloatEltVT, EC: VT.getVectorElementCount());
6915
6916 // Legal types should have been checked in the RISCVTargetLowering
6917 // constructor.
6918 // TODO: Splitting may make sense in some cases.
6919 assert(DAG.getTargetLoweringInfo().isTypeLegal(FloatVT) &&
6920 "Expected legal float type!");
6921
6922 // For CTTZ_ZERO_UNDEF, we need to extract the lowest set bit using X & -X.
6923 // The trailing zero count is equal to log2 of this single bit value.
6924 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
6925 SDValue Neg = DAG.getNegative(Val: Src, DL, VT);
6926 Src = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Src, N2: Neg);
6927 } else if (Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF) {
6928 SDValue Neg = DAG.getNode(Opcode: ISD::VP_SUB, DL, VT, N1: DAG.getConstant(Val: 0, DL, VT),
6929 N2: Src, N3: Mask, N4: VL);
6930 Src = DAG.getNode(Opcode: ISD::VP_AND, DL, VT, N1: Src, N2: Neg, N3: Mask, N4: VL);
6931 }
6932
6933 // We have a legal FP type, convert to it.
6934 SDValue FloatVal;
6935 if (FloatVT.bitsGT(VT)) {
6936 if (Op->isVPOpcode())
6937 FloatVal = DAG.getNode(Opcode: ISD::VP_UINT_TO_FP, DL, VT: FloatVT, N1: Src, N2: Mask, N3: VL);
6938 else
6939 FloatVal = DAG.getNode(Opcode: ISD::UINT_TO_FP, DL, VT: FloatVT, Operand: Src);
6940 } else {
6941 // Use RTZ to avoid rounding influencing exponent of FloatVal.
6942 if (VT.isFixedLengthVector()) {
6943 ContainerVT = getContainerForFixedLengthVector(VT);
6944 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
6945 }
6946 if (!Op->isVPOpcode())
6947 std::tie(args&: Mask, args&: VL) = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
6948 SDValue RTZRM =
6949 DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: Subtarget.getXLenVT());
6950 MVT ContainerFloatVT =
6951 MVT::getVectorVT(VT: FloatEltVT, EC: ContainerVT.getVectorElementCount());
6952 FloatVal = DAG.getNode(Opcode: RISCVISD::VFCVT_RM_F_XU_VL, DL, VT: ContainerFloatVT,
6953 N1: Src, N2: Mask, N3: RTZRM, N4: VL);
6954 if (VT.isFixedLengthVector())
6955 FloatVal = convertFromScalableVector(VT: FloatVT, V: FloatVal, DAG, Subtarget);
6956 }
6957 // Bitcast to integer and shift the exponent to the LSB.
6958 EVT IntVT = FloatVT.changeVectorElementTypeToInteger();
6959 SDValue Bitcast = DAG.getBitcast(VT: IntVT, V: FloatVal);
6960 unsigned ShiftAmt = FloatEltVT == MVT::f64 ? 52 : 23;
6961
6962 SDValue Exp;
6963 // Restore back to original type. Truncation after SRL is to generate vnsrl.
6964 if (Op->isVPOpcode()) {
6965 Exp = DAG.getNode(Opcode: ISD::VP_SRL, DL, VT: IntVT, N1: Bitcast,
6966 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: IntVT), N3: Mask, N4: VL);
6967 Exp = DAG.getVPZExtOrTrunc(DL, VT, Op: Exp, Mask, EVL: VL);
6968 } else {
6969 Exp = DAG.getNode(Opcode: ISD::SRL, DL, VT: IntVT, N1: Bitcast,
6970 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: IntVT));
6971 if (IntVT.bitsLT(VT))
6972 Exp = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Exp);
6973 else if (IntVT.bitsGT(VT))
6974 Exp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Exp);
6975 }
6976
6977 // The exponent contains log2 of the value in biased form.
6978 unsigned ExponentBias = FloatEltVT == MVT::f64 ? 1023 : 127;
6979 // For trailing zeros, we just need to subtract the bias.
6980 if (Op.getOpcode() == ISD::CTTZ_ZERO_UNDEF)
6981 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Exp,
6982 N2: DAG.getConstant(Val: ExponentBias, DL, VT));
6983 if (Op.getOpcode() == ISD::VP_CTTZ_ZERO_UNDEF)
6984 return DAG.getNode(Opcode: ISD::VP_SUB, DL, VT, N1: Exp,
6985 N2: DAG.getConstant(Val: ExponentBias, DL, VT), N3: Mask, N4: VL);
6986
6987 // For leading zeros, we need to remove the bias and convert from log2 to
6988 // leading zeros. We can do this by subtracting from (Bias + (EltSize - 1)).
6989 unsigned Adjust = ExponentBias + (EltSize - 1);
6990 SDValue Res;
6991 if (Op->isVPOpcode())
6992 Res = DAG.getNode(Opcode: ISD::VP_SUB, DL, VT, N1: DAG.getConstant(Val: Adjust, DL, VT), N2: Exp,
6993 N3: Mask, N4: VL);
6994 else
6995 Res = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: DAG.getConstant(Val: Adjust, DL, VT), N2: Exp);
6996
6997 // The above result with zero input equals to Adjust which is greater than
6998 // EltSize. Hence, we can do min(Res, EltSize) for CTLZ.
6999 if (Op.getOpcode() == ISD::CTLZ)
7000 Res = DAG.getNode(Opcode: ISD::UMIN, DL, VT, N1: Res, N2: DAG.getConstant(Val: EltSize, DL, VT));
7001 else if (Op.getOpcode() == ISD::VP_CTLZ)
7002 Res = DAG.getNode(Opcode: ISD::VP_UMIN, DL, VT, N1: Res,
7003 N2: DAG.getConstant(Val: EltSize, DL, VT), N3: Mask, N4: VL);
7004 return Res;
7005}
7006
7007SDValue RISCVTargetLowering::lowerVPCttzElements(SDValue Op,
7008 SelectionDAG &DAG) const {
7009 SDLoc DL(Op);
7010 MVT XLenVT = Subtarget.getXLenVT();
7011 SDValue Source = Op->getOperand(Num: 0);
7012 MVT SrcVT = Source.getSimpleValueType();
7013 SDValue Mask = Op->getOperand(Num: 1);
7014 SDValue EVL = Op->getOperand(Num: 2);
7015
7016 if (SrcVT.isFixedLengthVector()) {
7017 MVT ContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
7018 Source = convertToScalableVector(VT: ContainerVT, V: Source, DAG, Subtarget);
7019 Mask = convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Mask, DAG,
7020 Subtarget);
7021 SrcVT = ContainerVT;
7022 }
7023
7024 // Convert to boolean vector.
7025 if (SrcVT.getScalarType() != MVT::i1) {
7026 SDValue AllZero = DAG.getConstant(Val: 0, DL, VT: SrcVT);
7027 SrcVT = MVT::getVectorVT(VT: MVT::i1, EC: SrcVT.getVectorElementCount());
7028 Source = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: SrcVT,
7029 Ops: {Source, AllZero, DAG.getCondCode(Cond: ISD::SETNE),
7030 DAG.getUNDEF(VT: SrcVT), Mask, EVL});
7031 }
7032
7033 SDValue Res = DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Source, N2: Mask, N3: EVL);
7034 if (Op->getOpcode() == ISD::VP_CTTZ_ELTS_ZERO_UNDEF)
7035 // In this case, we can interpret poison as -1, so nothing to do further.
7036 return Res;
7037
7038 // Convert -1 to VL.
7039 SDValue SetCC =
7040 DAG.getSetCC(DL, VT: XLenVT, LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETLT);
7041 Res = DAG.getSelect(DL, VT: XLenVT, Cond: SetCC, LHS: EVL, RHS: Res);
7042 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: Res);
7043}
7044
7045// While RVV has alignment restrictions, we should always be able to load as a
7046// legal equivalently-sized byte-typed vector instead. This method is
7047// responsible for re-expressing a ISD::LOAD via a correctly-aligned type. If
7048// the load is already correctly-aligned, it returns SDValue().
7049SDValue RISCVTargetLowering::expandUnalignedRVVLoad(SDValue Op,
7050 SelectionDAG &DAG) const {
7051 auto *Load = cast<LoadSDNode>(Val&: Op);
7052 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7053
7054 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7055 VT: Load->getMemoryVT(),
7056 MMO: *Load->getMemOperand()))
7057 return SDValue();
7058
7059 SDLoc DL(Op);
7060 MVT VT = Op.getSimpleValueType();
7061 unsigned EltSizeBits = VT.getScalarSizeInBits();
7062 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7063 "Unexpected unaligned RVV load type");
7064 MVT NewVT =
7065 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7066 assert(NewVT.isValid() &&
7067 "Expecting equally-sized RVV vector types to be legal");
7068 SDValue L = DAG.getLoad(VT: NewVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
7069 PtrInfo: Load->getPointerInfo(), Alignment: Load->getBaseAlign(),
7070 MMOFlags: Load->getMemOperand()->getFlags());
7071 return DAG.getMergeValues(Ops: {DAG.getBitcast(VT, V: L), L.getValue(R: 1)}, dl: DL);
7072}
7073
7074// While RVV has alignment restrictions, we should always be able to store as a
7075// legal equivalently-sized byte-typed vector instead. This method is
7076// responsible for re-expressing a ISD::STORE via a correctly-aligned type. It
7077// returns SDValue() if the store is already correctly aligned.
7078SDValue RISCVTargetLowering::expandUnalignedRVVStore(SDValue Op,
7079 SelectionDAG &DAG) const {
7080 auto *Store = cast<StoreSDNode>(Val&: Op);
7081 assert(Store && Store->getValue().getValueType().isVector() &&
7082 "Expected vector store");
7083
7084 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7085 VT: Store->getMemoryVT(),
7086 MMO: *Store->getMemOperand()))
7087 return SDValue();
7088
7089 SDLoc DL(Op);
7090 SDValue StoredVal = Store->getValue();
7091 MVT VT = StoredVal.getSimpleValueType();
7092 unsigned EltSizeBits = VT.getScalarSizeInBits();
7093 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7094 "Unexpected unaligned RVV store type");
7095 MVT NewVT =
7096 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7097 assert(NewVT.isValid() &&
7098 "Expecting equally-sized RVV vector types to be legal");
7099 StoredVal = DAG.getBitcast(VT: NewVT, V: StoredVal);
7100 return DAG.getStore(Chain: Store->getChain(), dl: DL, Val: StoredVal, Ptr: Store->getBasePtr(),
7101 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
7102 MMOFlags: Store->getMemOperand()->getFlags());
7103}
7104
7105// While RVV has alignment restrictions, we should always be able to load as a
7106// legal equivalently-sized byte-typed vector instead. This method is
7107// responsible for re-expressing a ISD::VP_LOAD via a correctly-aligned type. If
7108// the load is already correctly-aligned, it returns SDValue().
7109SDValue RISCVTargetLowering::expandUnalignedVPLoad(SDValue Op,
7110 SelectionDAG &DAG) const {
7111 auto *Load = cast<VPLoadSDNode>(Val&: Op);
7112 assert(Load && Load->getMemoryVT().isVector() && "Expected vector load");
7113
7114 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7115 VT: Load->getMemoryVT(),
7116 MMO: *Load->getMemOperand()))
7117 return SDValue();
7118
7119 SDValue Mask = Load->getMask();
7120
7121 // FIXME: Handled masked loads somehow.
7122 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
7123 return SDValue();
7124
7125 SDLoc DL(Op);
7126 MVT VT = Op.getSimpleValueType();
7127 unsigned EltSizeBits = VT.getScalarSizeInBits();
7128 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7129 "Unexpected unaligned RVV load type");
7130 MVT NewVT =
7131 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7132 assert(NewVT.isValid() &&
7133 "Expecting equally-sized RVV vector types to be legal");
7134
7135 SDValue VL = Load->getVectorLength();
7136 VL = DAG.getNode(Opcode: ISD::MUL, DL, VT: VL.getValueType(), N1: VL,
7137 N2: DAG.getConstant(Val: (EltSizeBits / 8), DL, VT: VL.getValueType()));
7138
7139 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, EC: NewVT.getVectorElementCount());
7140 SDValue L = DAG.getLoadVP(VT: NewVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
7141 Mask: DAG.getAllOnesConstant(DL, VT: MaskVT), EVL: VL,
7142 PtrInfo: Load->getPointerInfo(), Alignment: Load->getBaseAlign(),
7143 MMOFlags: Load->getMemOperand()->getFlags(), AAInfo: AAMDNodes());
7144 return DAG.getMergeValues(Ops: {DAG.getBitcast(VT, V: L), L.getValue(R: 1)}, dl: DL);
7145}
7146
7147// While RVV has alignment restrictions, we should always be able to store as a
7148// legal equivalently-sized byte-typed vector instead. This method is
7149// responsible for re-expressing a ISD::VP STORE via a correctly-aligned type.
7150// It returns SDValue() if the store is already correctly aligned.
7151SDValue RISCVTargetLowering::expandUnalignedVPStore(SDValue Op,
7152 SelectionDAG &DAG) const {
7153 auto *Store = cast<VPStoreSDNode>(Val&: Op);
7154 assert(Store && Store->getValue().getValueType().isVector() &&
7155 "Expected vector store");
7156
7157 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
7158 VT: Store->getMemoryVT(),
7159 MMO: *Store->getMemOperand()))
7160 return SDValue();
7161
7162 SDValue Mask = Store->getMask();
7163
7164 // FIXME: Handled masked stores somehow.
7165 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
7166 return SDValue();
7167
7168 SDLoc DL(Op);
7169 SDValue StoredVal = Store->getValue();
7170 MVT VT = StoredVal.getSimpleValueType();
7171 unsigned EltSizeBits = VT.getScalarSizeInBits();
7172 assert((EltSizeBits == 16 || EltSizeBits == 32 || EltSizeBits == 64) &&
7173 "Unexpected unaligned RVV store type");
7174 MVT NewVT =
7175 MVT::getVectorVT(VT: MVT::i8, EC: VT.getVectorElementCount() * (EltSizeBits / 8));
7176 assert(NewVT.isValid() &&
7177 "Expecting equally-sized RVV vector types to be legal");
7178
7179 SDValue VL = Store->getVectorLength();
7180 VL = DAG.getNode(Opcode: ISD::MUL, DL, VT: VL.getValueType(), N1: VL,
7181 N2: DAG.getConstant(Val: (EltSizeBits / 8), DL, VT: VL.getValueType()));
7182
7183 StoredVal = DAG.getBitcast(VT: NewVT, V: StoredVal);
7184
7185 LocationSize Size = LocationSize::precise(Value: NewVT.getStoreSize());
7186 MachineFunction &MF = DAG.getMachineFunction();
7187 MachineMemOperand *MMO = MF.getMachineMemOperand(
7188 PtrInfo: Store->getPointerInfo(), F: Store->getMemOperand()->getFlags(), Size,
7189 BaseAlignment: Store->getBaseAlign());
7190
7191 MVT MaskVT = MVT::getVectorVT(VT: MVT::i1, EC: NewVT.getVectorElementCount());
7192 return DAG.getStoreVP(Chain: Store->getChain(), dl: DL, Val: StoredVal, Ptr: Store->getBasePtr(),
7193 Offset: DAG.getUNDEF(VT: Store->getBasePtr().getValueType()),
7194 Mask: DAG.getAllOnesConstant(DL, VT: MaskVT), EVL: VL, MemVT: NewVT, MMO,
7195 AM: ISD::UNINDEXED);
7196}
7197
7198static SDValue lowerConstant(SDValue Op, SelectionDAG &DAG,
7199 const RISCVSubtarget &Subtarget) {
7200 assert(Op.getValueType() == MVT::i64 && "Unexpected VT");
7201
7202 int64_t Imm = cast<ConstantSDNode>(Val&: Op)->getSExtValue();
7203
7204 // All simm32 constants should be handled by isel.
7205 // NOTE: The getMaxBuildIntsCost call below should return a value >= 2 making
7206 // this check redundant, but small immediates are common so this check
7207 // should have better compile time.
7208 if (isInt<32>(x: Imm))
7209 return Op;
7210
7211 // We only need to cost the immediate, if constant pool lowering is enabled.
7212 if (!Subtarget.useConstantPoolForLargeInts())
7213 return Op;
7214
7215 RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Val: Imm, STI: Subtarget);
7216 if (Seq.size() <= Subtarget.getMaxBuildIntsCost())
7217 return Op;
7218
7219 // Optimizations below are disabled for opt size. If we're optimizing for
7220 // size, use a constant pool.
7221 if (DAG.shouldOptForSize())
7222 return SDValue();
7223
7224 // Special case. See if we can build the constant as (ADD (SLLI X, C), X) do
7225 // that if it will avoid a constant pool.
7226 // It will require an extra temporary register though.
7227 // If we have Zba we can use (ADD_UW X, (SLLI X, 32)) to handle cases where
7228 // low and high 32 bits are the same and bit 31 and 63 are set.
7229 unsigned ShiftAmt, AddOpc;
7230 RISCVMatInt::InstSeq SeqLo =
7231 RISCVMatInt::generateTwoRegInstSeq(Val: Imm, STI: Subtarget, ShiftAmt, AddOpc);
7232 if (!SeqLo.empty() && (SeqLo.size() + 2) <= Subtarget.getMaxBuildIntsCost())
7233 return Op;
7234
7235 return SDValue();
7236}
7237
7238SDValue RISCVTargetLowering::lowerConstantFP(SDValue Op,
7239 SelectionDAG &DAG) const {
7240 MVT VT = Op.getSimpleValueType();
7241 const APFloat &Imm = cast<ConstantFPSDNode>(Val&: Op)->getValueAPF();
7242
7243 // Can this constant be selected by a Zfa FLI instruction?
7244 bool Negate = false;
7245 int Index = getLegalZfaFPImm(Imm, VT);
7246
7247 // If the constant is negative, try negating.
7248 if (Index < 0 && Imm.isNegative()) {
7249 Index = getLegalZfaFPImm(Imm: -Imm, VT);
7250 Negate = true;
7251 }
7252
7253 // If we couldn't find a FLI lowering, fall back to generic code.
7254 if (Index < 0)
7255 return SDValue();
7256
7257 // Emit an FLI+FNEG. We use a custom node to hide from constant folding.
7258 SDLoc DL(Op);
7259 SDValue Const =
7260 DAG.getNode(Opcode: RISCVISD::FLI, DL, VT,
7261 Operand: DAG.getTargetConstant(Val: Index, DL, VT: Subtarget.getXLenVT()));
7262 if (!Negate)
7263 return Const;
7264
7265 return DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Const);
7266}
7267
7268static SDValue LowerPREFETCH(SDValue Op, const RISCVSubtarget &Subtarget,
7269 SelectionDAG &DAG) {
7270
7271 unsigned IsData = Op.getConstantOperandVal(i: 4);
7272
7273 // mips-p8700 we support data prefetch for now.
7274 if (Subtarget.hasVendorXMIPSCBOP() && !IsData)
7275 return Op.getOperand(i: 0);
7276 return Op;
7277}
7278
7279static SDValue LowerATOMIC_FENCE(SDValue Op, SelectionDAG &DAG,
7280 const RISCVSubtarget &Subtarget) {
7281 SDLoc dl(Op);
7282 AtomicOrdering FenceOrdering =
7283 static_cast<AtomicOrdering>(Op.getConstantOperandVal(i: 1));
7284 SyncScope::ID FenceSSID =
7285 static_cast<SyncScope::ID>(Op.getConstantOperandVal(i: 2));
7286
7287 if (Subtarget.hasStdExtZtso()) {
7288 // The only fence that needs an instruction is a sequentially-consistent
7289 // cross-thread fence.
7290 if (FenceOrdering == AtomicOrdering::SequentiallyConsistent &&
7291 FenceSSID == SyncScope::System)
7292 return Op;
7293
7294 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7295 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
7296 }
7297
7298 // singlethread fences only synchronize with signal handlers on the same
7299 // thread and thus only need to preserve instruction order, not actually
7300 // enforce memory ordering.
7301 if (FenceSSID == SyncScope::SingleThread)
7302 // MEMBARRIER is a compiler barrier; it codegens to a no-op.
7303 return DAG.getNode(Opcode: ISD::MEMBARRIER, DL: dl, VT: MVT::Other, Operand: Op.getOperand(i: 0));
7304
7305 return Op;
7306}
7307
7308SDValue RISCVTargetLowering::LowerIS_FPCLASS(SDValue Op,
7309 SelectionDAG &DAG) const {
7310 SDLoc DL(Op);
7311 MVT VT = Op.getSimpleValueType();
7312 MVT XLenVT = Subtarget.getXLenVT();
7313 unsigned Check = Op.getConstantOperandVal(i: 1);
7314 unsigned TDCMask = 0;
7315 if (Check & fcSNan)
7316 TDCMask |= RISCV::FPMASK_Signaling_NaN;
7317 if (Check & fcQNan)
7318 TDCMask |= RISCV::FPMASK_Quiet_NaN;
7319 if (Check & fcPosInf)
7320 TDCMask |= RISCV::FPMASK_Positive_Infinity;
7321 if (Check & fcNegInf)
7322 TDCMask |= RISCV::FPMASK_Negative_Infinity;
7323 if (Check & fcPosNormal)
7324 TDCMask |= RISCV::FPMASK_Positive_Normal;
7325 if (Check & fcNegNormal)
7326 TDCMask |= RISCV::FPMASK_Negative_Normal;
7327 if (Check & fcPosSubnormal)
7328 TDCMask |= RISCV::FPMASK_Positive_Subnormal;
7329 if (Check & fcNegSubnormal)
7330 TDCMask |= RISCV::FPMASK_Negative_Subnormal;
7331 if (Check & fcPosZero)
7332 TDCMask |= RISCV::FPMASK_Positive_Zero;
7333 if (Check & fcNegZero)
7334 TDCMask |= RISCV::FPMASK_Negative_Zero;
7335
7336 bool IsOneBitMask = isPowerOf2_32(Value: TDCMask);
7337
7338 SDValue TDCMaskV = DAG.getConstant(Val: TDCMask, DL, VT: XLenVT);
7339
7340 if (VT.isVector()) {
7341 SDValue Op0 = Op.getOperand(i: 0);
7342 MVT VT0 = Op.getOperand(i: 0).getSimpleValueType();
7343
7344 if (VT.isScalableVector()) {
7345 MVT DstVT = VT0.changeVectorElementTypeToInteger();
7346 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: VT0, DL, DAG, Subtarget);
7347 if (Op.getOpcode() == ISD::VP_IS_FPCLASS) {
7348 Mask = Op.getOperand(i: 2);
7349 VL = Op.getOperand(i: 3);
7350 }
7351 SDValue FPCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS_VL, DL, VT: DstVT, N1: Op0, N2: Mask,
7352 N3: VL, Flags: Op->getFlags());
7353 if (IsOneBitMask)
7354 return DAG.getSetCC(DL, VT, LHS: FPCLASS,
7355 RHS: DAG.getConstant(Val: TDCMask, DL, VT: DstVT),
7356 Cond: ISD::CondCode::SETEQ);
7357 SDValue AND = DAG.getNode(Opcode: ISD::AND, DL, VT: DstVT, N1: FPCLASS,
7358 N2: DAG.getConstant(Val: TDCMask, DL, VT: DstVT));
7359 return DAG.getSetCC(DL, VT, LHS: AND, RHS: DAG.getConstant(Val: 0, DL, VT: DstVT),
7360 Cond: ISD::SETNE);
7361 }
7362
7363 MVT ContainerVT0 = getContainerForFixedLengthVector(VT: VT0);
7364 MVT ContainerVT = getContainerForFixedLengthVector(VT);
7365 MVT ContainerDstVT = ContainerVT0.changeVectorElementTypeToInteger();
7366 auto [Mask, VL] = getDefaultVLOps(VecVT: VT0, ContainerVT: ContainerVT0, DL, DAG, Subtarget);
7367 if (Op.getOpcode() == ISD::VP_IS_FPCLASS) {
7368 Mask = Op.getOperand(i: 2);
7369 MVT MaskContainerVT =
7370 getContainerForFixedLengthVector(VT: Mask.getSimpleValueType());
7371 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
7372 VL = Op.getOperand(i: 3);
7373 }
7374 Op0 = convertToScalableVector(VT: ContainerVT0, V: Op0, DAG, Subtarget);
7375
7376 SDValue FPCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS_VL, DL, VT: ContainerDstVT, N1: Op0,
7377 N2: Mask, N3: VL, Flags: Op->getFlags());
7378
7379 TDCMaskV = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerDstVT,
7380 N1: DAG.getUNDEF(VT: ContainerDstVT), N2: TDCMaskV, N3: VL);
7381 if (IsOneBitMask) {
7382 SDValue VMSEQ =
7383 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
7384 Ops: {FPCLASS, TDCMaskV, DAG.getCondCode(Cond: ISD::SETEQ),
7385 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7386 return convertFromScalableVector(VT, V: VMSEQ, DAG, Subtarget);
7387 }
7388 SDValue AND = DAG.getNode(Opcode: RISCVISD::AND_VL, DL, VT: ContainerDstVT, N1: FPCLASS,
7389 N2: TDCMaskV, N3: DAG.getUNDEF(VT: ContainerDstVT), N4: Mask, N5: VL);
7390
7391 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
7392 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerDstVT,
7393 N1: DAG.getUNDEF(VT: ContainerDstVT), N2: SplatZero, N3: VL);
7394
7395 SDValue VMSNE = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
7396 Ops: {AND, SplatZero, DAG.getCondCode(Cond: ISD::SETNE),
7397 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7398 return convertFromScalableVector(VT, V: VMSNE, DAG, Subtarget);
7399 }
7400
7401 SDValue FCLASS = DAG.getNode(Opcode: RISCVISD::FCLASS, DL, VT: XLenVT, Operand: Op.getOperand(i: 0));
7402 SDValue AND = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: FCLASS, N2: TDCMaskV);
7403 SDValue Res = DAG.getSetCC(DL, VT: XLenVT, LHS: AND, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT),
7404 Cond: ISD::CondCode::SETNE);
7405 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
7406}
7407
7408// Lower fmaximum and fminimum. Unlike our fmax and fmin instructions, these
7409// operations propagate nans.
7410static SDValue lowerFMAXIMUM_FMINIMUM(SDValue Op, SelectionDAG &DAG,
7411 const RISCVSubtarget &Subtarget) {
7412 SDLoc DL(Op);
7413 MVT VT = Op.getSimpleValueType();
7414
7415 SDValue X = Op.getOperand(i: 0);
7416 SDValue Y = Op.getOperand(i: 1);
7417
7418 if (!VT.isVector()) {
7419 MVT XLenVT = Subtarget.getXLenVT();
7420
7421 // If X is a nan, replace Y with X. If Y is a nan, replace X with Y. This
7422 // ensures that when one input is a nan, the other will also be a nan
7423 // allowing the nan to propagate. If both inputs are nan, this will swap the
7424 // inputs which is harmless.
7425
7426 SDValue NewY = Y;
7427 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Op: X)) {
7428 SDValue XIsNonNan = DAG.getSetCC(DL, VT: XLenVT, LHS: X, RHS: X, Cond: ISD::SETOEQ);
7429 NewY = DAG.getSelect(DL, VT, Cond: XIsNonNan, LHS: Y, RHS: X);
7430 }
7431
7432 SDValue NewX = X;
7433 if (!Op->getFlags().hasNoNaNs() && !DAG.isKnownNeverNaN(Op: Y)) {
7434 SDValue YIsNonNan = DAG.getSetCC(DL, VT: XLenVT, LHS: Y, RHS: Y, Cond: ISD::SETOEQ);
7435 NewX = DAG.getSelect(DL, VT, Cond: YIsNonNan, LHS: X, RHS: Y);
7436 }
7437
7438 unsigned Opc =
7439 Op.getOpcode() == ISD::FMAXIMUM ? RISCVISD::FMAX : RISCVISD::FMIN;
7440 return DAG.getNode(Opcode: Opc, DL, VT, N1: NewX, N2: NewY);
7441 }
7442
7443 // Check no NaNs before converting to fixed vector scalable.
7444 bool XIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Op: X);
7445 bool YIsNeverNan = Op->getFlags().hasNoNaNs() || DAG.isKnownNeverNaN(Op: Y);
7446
7447 MVT ContainerVT = VT;
7448 if (VT.isFixedLengthVector()) {
7449 ContainerVT = getContainerForFixedLengthVector(DAG, VT, Subtarget);
7450 X = convertToScalableVector(VT: ContainerVT, V: X, DAG, Subtarget);
7451 Y = convertToScalableVector(VT: ContainerVT, V: Y, DAG, Subtarget);
7452 }
7453
7454 SDValue Mask, VL;
7455 if (Op->isVPOpcode()) {
7456 Mask = Op.getOperand(i: 2);
7457 if (VT.isFixedLengthVector())
7458 Mask = convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Mask, DAG,
7459 Subtarget);
7460 VL = Op.getOperand(i: 3);
7461 } else {
7462 std::tie(args&: Mask, args&: VL) = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
7463 }
7464
7465 SDValue NewY = Y;
7466 if (!XIsNeverNan) {
7467 SDValue XIsNonNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
7468 Ops: {X, X, DAG.getCondCode(Cond: ISD::SETOEQ),
7469 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7470 NewY = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: XIsNonNan, N2: Y, N3: X,
7471 N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
7472 }
7473
7474 SDValue NewX = X;
7475 if (!YIsNeverNan) {
7476 SDValue YIsNonNan = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
7477 Ops: {Y, Y, DAG.getCondCode(Cond: ISD::SETOEQ),
7478 DAG.getUNDEF(VT: ContainerVT), Mask, VL});
7479 NewX = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: YIsNonNan, N2: X, N3: Y,
7480 N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
7481 }
7482
7483 unsigned Opc =
7484 Op.getOpcode() == ISD::FMAXIMUM || Op->getOpcode() == ISD::VP_FMAXIMUM
7485 ? RISCVISD::VFMAX_VL
7486 : RISCVISD::VFMIN_VL;
7487 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: NewX, N2: NewY,
7488 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
7489 if (VT.isFixedLengthVector())
7490 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
7491 return Res;
7492}
7493
7494static SDValue lowerFABSorFNEG(SDValue Op, SelectionDAG &DAG,
7495 const RISCVSubtarget &Subtarget) {
7496 bool IsFABS = Op.getOpcode() == ISD::FABS;
7497 assert((IsFABS || Op.getOpcode() == ISD::FNEG) &&
7498 "Wrong opcode for lowering FABS or FNEG.");
7499
7500 MVT XLenVT = Subtarget.getXLenVT();
7501 MVT VT = Op.getSimpleValueType();
7502 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7503
7504 SDLoc DL(Op);
7505 SDValue Fmv =
7506 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Op.getOperand(i: 0));
7507
7508 APInt Mask = IsFABS ? APInt::getSignedMaxValue(numBits: 16) : APInt::getSignMask(BitWidth: 16);
7509 Mask = Mask.sext(width: Subtarget.getXLen());
7510
7511 unsigned LogicOpc = IsFABS ? ISD::AND : ISD::XOR;
7512 SDValue Logic =
7513 DAG.getNode(Opcode: LogicOpc, DL, VT: XLenVT, N1: Fmv, N2: DAG.getConstant(Val: Mask, DL, VT: XLenVT));
7514 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: Logic);
7515}
7516
7517static SDValue lowerFCOPYSIGN(SDValue Op, SelectionDAG &DAG,
7518 const RISCVSubtarget &Subtarget) {
7519 assert(Op.getOpcode() == ISD::FCOPYSIGN && "Unexpected opcode");
7520
7521 MVT XLenVT = Subtarget.getXLenVT();
7522 MVT VT = Op.getSimpleValueType();
7523 assert((VT == MVT::f16 || VT == MVT::bf16) && "Unexpected type");
7524
7525 SDValue Mag = Op.getOperand(i: 0);
7526 SDValue Sign = Op.getOperand(i: 1);
7527
7528 SDLoc DL(Op);
7529
7530 // Get sign bit into an integer value.
7531 unsigned SignSize = Sign.getValueSizeInBits();
7532 SDValue SignAsInt = [&]() {
7533 if (SignSize == Subtarget.getXLen())
7534 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT: XLenVT, Operand: Sign);
7535 switch (SignSize) {
7536 case 16:
7537 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Sign);
7538 case 32:
7539 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: XLenVT, Operand: Sign);
7540 case 64: {
7541 assert(XLenVT == MVT::i32 && "Unexpected type");
7542 // Copy the upper word to integer.
7543 SignSize = 32;
7544 return DAG.getNode(Opcode: RISCVISD::SplitF64, DL, ResultTys: {MVT::i32, MVT::i32}, Ops: Sign)
7545 .getValue(R: 1);
7546 }
7547 default:
7548 llvm_unreachable("Unexpected sign size");
7549 }
7550 }();
7551
7552 // Get the signbit at the right position for MagAsInt.
7553 if (int ShiftAmount = (int)SignSize - (int)Mag.getValueSizeInBits())
7554 SignAsInt = DAG.getNode(Opcode: ShiftAmount > 0 ? ISD::SRL : ISD::SHL, DL, VT: XLenVT,
7555 N1: SignAsInt,
7556 N2: DAG.getConstant(Val: std::abs(x: ShiftAmount), DL, VT: XLenVT));
7557
7558 // Mask the sign bit and any bits above it. The extra bits will be dropped
7559 // when we convert back to FP.
7560 SDValue SignMask = DAG.getConstant(
7561 Val: APInt::getSignMask(BitWidth: 16).sext(width: Subtarget.getXLen()), DL, VT: XLenVT);
7562 SDValue SignBit = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: SignAsInt, N2: SignMask);
7563
7564 // Transform Mag value to integer, and clear the sign bit.
7565 SDValue MagAsInt = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Mag);
7566 SDValue ClearSignMask = DAG.getConstant(
7567 Val: APInt::getSignedMaxValue(numBits: 16).sext(width: Subtarget.getXLen()), DL, VT: XLenVT);
7568 SDValue ClearedSign =
7569 DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: MagAsInt, N2: ClearSignMask);
7570
7571 SDValue CopiedSign = DAG.getNode(Opcode: ISD::OR, DL, VT: XLenVT, N1: ClearedSign, N2: SignBit,
7572 Flags: SDNodeFlags::Disjoint);
7573
7574 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: CopiedSign);
7575}
7576
7577/// Get a RISC-V target specified VL op for a given SDNode.
7578static unsigned getRISCVVLOp(SDValue Op) {
7579#define OP_CASE(NODE) \
7580 case ISD::NODE: \
7581 return RISCVISD::NODE##_VL;
7582#define VP_CASE(NODE) \
7583 case ISD::VP_##NODE: \
7584 return RISCVISD::NODE##_VL;
7585 // clang-format off
7586 switch (Op.getOpcode()) {
7587 default:
7588 llvm_unreachable("don't have RISC-V specified VL op for this SDNode");
7589 OP_CASE(ADD)
7590 OP_CASE(SUB)
7591 OP_CASE(MUL)
7592 OP_CASE(MULHS)
7593 OP_CASE(MULHU)
7594 OP_CASE(SDIV)
7595 OP_CASE(SREM)
7596 OP_CASE(UDIV)
7597 OP_CASE(UREM)
7598 OP_CASE(SHL)
7599 OP_CASE(SRA)
7600 OP_CASE(SRL)
7601 OP_CASE(ROTL)
7602 OP_CASE(ROTR)
7603 OP_CASE(BSWAP)
7604 OP_CASE(CTTZ)
7605 OP_CASE(CTLZ)
7606 OP_CASE(CTPOP)
7607 OP_CASE(BITREVERSE)
7608 OP_CASE(SADDSAT)
7609 OP_CASE(UADDSAT)
7610 OP_CASE(SSUBSAT)
7611 OP_CASE(USUBSAT)
7612 OP_CASE(AVGFLOORS)
7613 OP_CASE(AVGFLOORU)
7614 OP_CASE(AVGCEILS)
7615 OP_CASE(AVGCEILU)
7616 OP_CASE(FADD)
7617 OP_CASE(FSUB)
7618 OP_CASE(FMUL)
7619 OP_CASE(FDIV)
7620 OP_CASE(FNEG)
7621 OP_CASE(FABS)
7622 OP_CASE(FCOPYSIGN)
7623 OP_CASE(FSQRT)
7624 OP_CASE(SMIN)
7625 OP_CASE(SMAX)
7626 OP_CASE(UMIN)
7627 OP_CASE(UMAX)
7628 OP_CASE(ABDS)
7629 OP_CASE(ABDU)
7630 OP_CASE(STRICT_FADD)
7631 OP_CASE(STRICT_FSUB)
7632 OP_CASE(STRICT_FMUL)
7633 OP_CASE(STRICT_FDIV)
7634 OP_CASE(STRICT_FSQRT)
7635 VP_CASE(ADD) // VP_ADD
7636 VP_CASE(SUB) // VP_SUB
7637 VP_CASE(MUL) // VP_MUL
7638 VP_CASE(SDIV) // VP_SDIV
7639 VP_CASE(SREM) // VP_SREM
7640 VP_CASE(UDIV) // VP_UDIV
7641 VP_CASE(UREM) // VP_UREM
7642 VP_CASE(SHL) // VP_SHL
7643 VP_CASE(FADD) // VP_FADD
7644 VP_CASE(FSUB) // VP_FSUB
7645 VP_CASE(FMUL) // VP_FMUL
7646 VP_CASE(FDIV) // VP_FDIV
7647 VP_CASE(FNEG) // VP_FNEG
7648 VP_CASE(FABS) // VP_FABS
7649 VP_CASE(SMIN) // VP_SMIN
7650 VP_CASE(SMAX) // VP_SMAX
7651 VP_CASE(UMIN) // VP_UMIN
7652 VP_CASE(UMAX) // VP_UMAX
7653 VP_CASE(FCOPYSIGN) // VP_FCOPYSIGN
7654 VP_CASE(SETCC) // VP_SETCC
7655 VP_CASE(SINT_TO_FP) // VP_SINT_TO_FP
7656 VP_CASE(UINT_TO_FP) // VP_UINT_TO_FP
7657 VP_CASE(BITREVERSE) // VP_BITREVERSE
7658 VP_CASE(SADDSAT) // VP_SADDSAT
7659 VP_CASE(UADDSAT) // VP_UADDSAT
7660 VP_CASE(SSUBSAT) // VP_SSUBSAT
7661 VP_CASE(USUBSAT) // VP_USUBSAT
7662 VP_CASE(BSWAP) // VP_BSWAP
7663 VP_CASE(CTLZ) // VP_CTLZ
7664 VP_CASE(CTTZ) // VP_CTTZ
7665 VP_CASE(CTPOP) // VP_CTPOP
7666 case ISD::CTLZ_ZERO_UNDEF:
7667 case ISD::VP_CTLZ_ZERO_UNDEF:
7668 return RISCVISD::CTLZ_VL;
7669 case ISD::CTTZ_ZERO_UNDEF:
7670 case ISD::VP_CTTZ_ZERO_UNDEF:
7671 return RISCVISD::CTTZ_VL;
7672 case ISD::FMA:
7673 case ISD::VP_FMA:
7674 return RISCVISD::VFMADD_VL;
7675 case ISD::STRICT_FMA:
7676 return RISCVISD::STRICT_VFMADD_VL;
7677 case ISD::AND:
7678 case ISD::VP_AND:
7679 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7680 return RISCVISD::VMAND_VL;
7681 return RISCVISD::AND_VL;
7682 case ISD::OR:
7683 case ISD::VP_OR:
7684 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7685 return RISCVISD::VMOR_VL;
7686 return RISCVISD::OR_VL;
7687 case ISD::XOR:
7688 case ISD::VP_XOR:
7689 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
7690 return RISCVISD::VMXOR_VL;
7691 return RISCVISD::XOR_VL;
7692 case ISD::ANY_EXTEND:
7693 case ISD::ZERO_EXTEND:
7694 return RISCVISD::VZEXT_VL;
7695 case ISD::SIGN_EXTEND:
7696 return RISCVISD::VSEXT_VL;
7697 case ISD::SETCC:
7698 return RISCVISD::SETCC_VL;
7699 case ISD::VSELECT:
7700 return RISCVISD::VMERGE_VL;
7701 case ISD::VP_SELECT:
7702 case ISD::VP_MERGE:
7703 return RISCVISD::VMERGE_VL;
7704 case ISD::VP_SRA:
7705 return RISCVISD::SRA_VL;
7706 case ISD::VP_SRL:
7707 return RISCVISD::SRL_VL;
7708 case ISD::VP_SQRT:
7709 return RISCVISD::FSQRT_VL;
7710 case ISD::VP_SIGN_EXTEND:
7711 return RISCVISD::VSEXT_VL;
7712 case ISD::VP_ZERO_EXTEND:
7713 return RISCVISD::VZEXT_VL;
7714 case ISD::VP_FP_TO_SINT:
7715 return RISCVISD::VFCVT_RTZ_X_F_VL;
7716 case ISD::VP_FP_TO_UINT:
7717 return RISCVISD::VFCVT_RTZ_XU_F_VL;
7718 case ISD::FMINNUM:
7719 case ISD::FMINIMUMNUM:
7720 case ISD::VP_FMINNUM:
7721 return RISCVISD::VFMIN_VL;
7722 case ISD::FMAXNUM:
7723 case ISD::FMAXIMUMNUM:
7724 case ISD::VP_FMAXNUM:
7725 return RISCVISD::VFMAX_VL;
7726 case ISD::LRINT:
7727 case ISD::VP_LRINT:
7728 case ISD::LLRINT:
7729 case ISD::VP_LLRINT:
7730 return RISCVISD::VFCVT_RM_X_F_VL;
7731 }
7732 // clang-format on
7733#undef OP_CASE
7734#undef VP_CASE
7735}
7736
7737static bool isPromotedOpNeedingSplit(SDValue Op,
7738 const RISCVSubtarget &Subtarget) {
7739 return (Op.getValueType() == MVT::nxv32f16 &&
7740 (Subtarget.hasVInstructionsF16Minimal() &&
7741 !Subtarget.hasVInstructionsF16())) ||
7742 (Op.getValueType() == MVT::nxv32bf16 &&
7743 Subtarget.hasVInstructionsBF16Minimal() &&
7744 (!Subtarget.hasVInstructionsBF16() ||
7745 (!llvm::is_contained(Range: ZvfbfaOps, Element: Op.getOpcode()) &&
7746 !llvm::is_contained(Range: ZvfbfaVPOps, Element: Op.getOpcode()))));
7747}
7748
7749static SDValue SplitVectorOp(SDValue Op, SelectionDAG &DAG) {
7750 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: Op.getValueType());
7751 SDLoc DL(Op);
7752
7753 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
7754 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
7755
7756 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
7757 if (!Op.getOperand(i: j).getValueType().isVector()) {
7758 LoOperands[j] = Op.getOperand(i: j);
7759 HiOperands[j] = Op.getOperand(i: j);
7760 continue;
7761 }
7762 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
7763 DAG.SplitVector(N: Op.getOperand(i: j), DL);
7764 }
7765
7766 SDValue LoRes =
7767 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: LoVT, Ops: LoOperands, Flags: Op->getFlags());
7768 SDValue HiRes =
7769 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: HiVT, Ops: HiOperands, Flags: Op->getFlags());
7770
7771 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Op.getValueType(), N1: LoRes, N2: HiRes);
7772}
7773
7774static SDValue SplitVPOp(SDValue Op, SelectionDAG &DAG) {
7775 assert(ISD::isVPOpcode(Op.getOpcode()) && "Not a VP op");
7776 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: Op.getValueType());
7777 SDLoc DL(Op);
7778
7779 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
7780 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
7781
7782 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
7783 if (ISD::getVPExplicitVectorLengthIdx(Opcode: Op.getOpcode()) == j) {
7784 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
7785 DAG.SplitEVL(N: Op.getOperand(i: j), VecVT: Op.getValueType(), DL);
7786 continue;
7787 }
7788 if (!Op.getOperand(i: j).getValueType().isVector()) {
7789 LoOperands[j] = Op.getOperand(i: j);
7790 HiOperands[j] = Op.getOperand(i: j);
7791 continue;
7792 }
7793 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
7794 DAG.SplitVector(N: Op.getOperand(i: j), DL);
7795 }
7796
7797 SDValue LoRes =
7798 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: LoVT, Ops: LoOperands, Flags: Op->getFlags());
7799 SDValue HiRes =
7800 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: HiVT, Ops: HiOperands, Flags: Op->getFlags());
7801
7802 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Op.getValueType(), N1: LoRes, N2: HiRes);
7803}
7804
7805static SDValue SplitVectorReductionOp(SDValue Op, SelectionDAG &DAG) {
7806 SDLoc DL(Op);
7807
7808 auto [Lo, Hi] = DAG.SplitVector(N: Op.getOperand(i: 1), DL);
7809 auto [MaskLo, MaskHi] = DAG.SplitVector(N: Op.getOperand(i: 2), DL);
7810 auto [EVLLo, EVLHi] =
7811 DAG.SplitEVL(N: Op.getOperand(i: 3), VecVT: Op.getOperand(i: 1).getValueType(), DL);
7812
7813 SDValue ResLo =
7814 DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
7815 Ops: {Op.getOperand(i: 0), Lo, MaskLo, EVLLo}, Flags: Op->getFlags());
7816 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
7817 Ops: {ResLo, Hi, MaskHi, EVLHi}, Flags: Op->getFlags());
7818}
7819
7820static SDValue SplitStrictFPVectorOp(SDValue Op, SelectionDAG &DAG) {
7821
7822 assert(Op->isStrictFPOpcode());
7823
7824 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: Op->getValueType(ResNo: 0));
7825
7826 SDVTList LoVTs = DAG.getVTList(VT1: LoVT, VT2: Op->getValueType(ResNo: 1));
7827 SDVTList HiVTs = DAG.getVTList(VT1: HiVT, VT2: Op->getValueType(ResNo: 1));
7828
7829 SDLoc DL(Op);
7830
7831 SmallVector<SDValue, 4> LoOperands(Op.getNumOperands());
7832 SmallVector<SDValue, 4> HiOperands(Op.getNumOperands());
7833
7834 for (unsigned j = 0; j != Op.getNumOperands(); ++j) {
7835 if (!Op.getOperand(i: j).getValueType().isVector()) {
7836 LoOperands[j] = Op.getOperand(i: j);
7837 HiOperands[j] = Op.getOperand(i: j);
7838 continue;
7839 }
7840 std::tie(args&: LoOperands[j], args&: HiOperands[j]) =
7841 DAG.SplitVector(N: Op.getOperand(i: j), DL);
7842 }
7843
7844 SDValue LoRes =
7845 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: LoVTs, Ops: LoOperands, Flags: Op->getFlags());
7846 HiOperands[0] = LoRes.getValue(R: 1);
7847 SDValue HiRes =
7848 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: HiVTs, Ops: HiOperands, Flags: Op->getFlags());
7849
7850 SDValue V = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: Op->getValueType(ResNo: 0),
7851 N1: LoRes.getValue(R: 0), N2: HiRes.getValue(R: 0));
7852 return DAG.getMergeValues(Ops: {V, HiRes.getValue(R: 1)}, dl: DL);
7853}
7854
7855SDValue
7856RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Load(SDValue Op,
7857 SelectionDAG &DAG) const {
7858 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
7859 "Unexpected bfloat16 load lowering");
7860
7861 SDLoc DL(Op);
7862 LoadSDNode *LD = cast<LoadSDNode>(Val: Op.getNode());
7863 EVT MemVT = LD->getMemoryVT();
7864 SDValue Load = DAG.getExtLoad(
7865 ExtType: ISD::ZEXTLOAD, dl: DL, VT: Subtarget.getXLenVT(), Chain: LD->getChain(),
7866 Ptr: LD->getBasePtr(),
7867 MemVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: MemVT.getSizeInBits()),
7868 MMO: LD->getMemOperand());
7869 // Using mask to make bf16 nan-boxing valid when we don't have flh
7870 // instruction. -65536 would be treat as a small number and thus it can be
7871 // directly used lui to get the constant.
7872 SDValue mask = DAG.getSignedConstant(Val: -65536, DL, VT: Subtarget.getXLenVT());
7873 SDValue OrSixteenOne =
7874 DAG.getNode(Opcode: ISD::OR, DL, VT: Load.getValueType(), Ops: {Load, mask});
7875 SDValue ConvertedResult =
7876 DAG.getNode(Opcode: RISCVISD::NDS_FMV_BF16_X, DL, VT: MVT::bf16, Operand: OrSixteenOne);
7877 return DAG.getMergeValues(Ops: {ConvertedResult, Load.getValue(R: 1)}, dl: DL);
7878}
7879
7880SDValue
7881RISCVTargetLowering::lowerXAndesBfHCvtBFloat16Store(SDValue Op,
7882 SelectionDAG &DAG) const {
7883 assert(Subtarget.hasVendorXAndesBFHCvt() && !Subtarget.hasStdExtZfh() &&
7884 "Unexpected bfloat16 store lowering");
7885
7886 StoreSDNode *ST = cast<StoreSDNode>(Val: Op.getNode());
7887 SDLoc DL(Op);
7888 SDValue FMV = DAG.getNode(Opcode: RISCVISD::NDS_FMV_X_ANYEXTBF16, DL,
7889 VT: Subtarget.getXLenVT(), Operand: ST->getValue());
7890 return DAG.getTruncStore(
7891 Chain: ST->getChain(), dl: DL, Val: FMV, Ptr: ST->getBasePtr(),
7892 SVT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: ST->getMemoryVT().getSizeInBits()),
7893 MMO: ST->getMemOperand());
7894}
7895
7896static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
7897 const RISCVSubtarget &Subtarget);
7898
7899SDValue RISCVTargetLowering::LowerOperation(SDValue Op,
7900 SelectionDAG &DAG) const {
7901 switch (Op.getOpcode()) {
7902 default:
7903 reportFatalInternalError(
7904 reason: "Unimplemented RISCVTargetLowering::LowerOperation Case");
7905 case ISD::PREFETCH:
7906 return LowerPREFETCH(Op, Subtarget, DAG);
7907 case ISD::ATOMIC_FENCE:
7908 return LowerATOMIC_FENCE(Op, DAG, Subtarget);
7909 case ISD::GlobalAddress:
7910 return lowerGlobalAddress(Op, DAG);
7911 case ISD::BlockAddress:
7912 return lowerBlockAddress(Op, DAG);
7913 case ISD::ConstantPool:
7914 return lowerConstantPool(Op, DAG);
7915 case ISD::JumpTable:
7916 return lowerJumpTable(Op, DAG);
7917 case ISD::GlobalTLSAddress:
7918 return lowerGlobalTLSAddress(Op, DAG);
7919 case ISD::Constant:
7920 return lowerConstant(Op, DAG, Subtarget);
7921 case ISD::ConstantFP:
7922 return lowerConstantFP(Op, DAG);
7923 case ISD::SELECT:
7924 return lowerSELECT(Op, DAG);
7925 case ISD::BRCOND:
7926 return lowerBRCOND(Op, DAG);
7927 case ISD::VASTART:
7928 return lowerVASTART(Op, DAG);
7929 case ISD::FRAMEADDR:
7930 return lowerFRAMEADDR(Op, DAG);
7931 case ISD::RETURNADDR:
7932 return lowerRETURNADDR(Op, DAG);
7933 case ISD::SHL_PARTS:
7934 return lowerShiftLeftParts(Op, DAG);
7935 case ISD::SRA_PARTS:
7936 return lowerShiftRightParts(Op, DAG, IsSRA: true);
7937 case ISD::SRL_PARTS:
7938 return lowerShiftRightParts(Op, DAG, IsSRA: false);
7939 case ISD::ROTL:
7940 case ISD::ROTR:
7941 if (Op.getValueType().isFixedLengthVector()) {
7942 assert(Subtarget.hasStdExtZvkb());
7943 return lowerToScalableOp(Op, DAG);
7944 }
7945 assert(Subtarget.hasVendorXTHeadBb() &&
7946 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()) &&
7947 "Unexpected custom legalization");
7948 // XTHeadBb only supports rotate by constant.
7949 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
7950 return SDValue();
7951 return Op;
7952 case ISD::BITCAST: {
7953 SDLoc DL(Op);
7954 EVT VT = Op.getValueType();
7955 SDValue Op0 = Op.getOperand(i: 0);
7956 EVT Op0VT = Op0.getValueType();
7957 MVT XLenVT = Subtarget.getXLenVT();
7958 if (Op0VT == MVT::i16 &&
7959 ((VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
7960 (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
7961 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Op0);
7962 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: NewOp0);
7963 }
7964 if (VT == MVT::f32 && Op0VT == MVT::i32 && Subtarget.is64Bit() &&
7965 Subtarget.hasStdExtFOrZfinx()) {
7966 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: Op0);
7967 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: NewOp0);
7968 }
7969 if (VT == MVT::f64 && Op0VT == MVT::i64 && !Subtarget.is64Bit() &&
7970 Subtarget.hasStdExtDOrZdinx()) {
7971 SDValue Lo, Hi;
7972 std::tie(args&: Lo, args&: Hi) = DAG.SplitScalar(N: Op0, DL, LoVT: MVT::i32, HiVT: MVT::i32);
7973 return DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
7974 }
7975
7976 if (Subtarget.hasStdExtP()) {
7977 bool Is32BitCast =
7978 (VT == MVT::i32 && (Op0VT == MVT::v4i8 || Op0VT == MVT::v2i16)) ||
7979 (Op0VT == MVT::i32 && (VT == MVT::v4i8 || VT == MVT::v2i16));
7980 bool Is64BitCast =
7981 (VT == MVT::i64 && (Op0VT == MVT::v8i8 || Op0VT == MVT::v4i16 ||
7982 Op0VT == MVT::v2i32)) ||
7983 (Op0VT == MVT::i64 &&
7984 (VT == MVT::v8i8 || VT == MVT::v4i16 || VT == MVT::v2i32));
7985 if (Is32BitCast || Is64BitCast)
7986 return Op;
7987 }
7988
7989 // Consider other scalar<->scalar casts as legal if the types are legal.
7990 // Otherwise expand them.
7991 if (!VT.isVector() && !Op0VT.isVector()) {
7992 if (isTypeLegal(VT) && isTypeLegal(VT: Op0VT))
7993 return Op;
7994 return SDValue();
7995 }
7996
7997 assert(!VT.isScalableVector() && !Op0VT.isScalableVector() &&
7998 "Unexpected types");
7999
8000 if (VT.isFixedLengthVector()) {
8001 // We can handle fixed length vector bitcasts with a simple replacement
8002 // in isel.
8003 if (Op0VT.isFixedLengthVector())
8004 return Op;
8005 // When bitcasting from scalar to fixed-length vector, insert the scalar
8006 // into a one-element vector of the result type, and perform a vector
8007 // bitcast.
8008 if (!Op0VT.isVector()) {
8009 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: Op0VT, NumElements: 1);
8010 if (!isTypeLegal(VT: BVT))
8011 return SDValue();
8012 return DAG.getBitcast(
8013 VT, V: DAG.getInsertVectorElt(DL, Vec: DAG.getUNDEF(VT: BVT), Elt: Op0, Idx: 0));
8014 }
8015 return SDValue();
8016 }
8017 // Custom-legalize bitcasts from fixed-length vector types to scalar types
8018 // thus: bitcast the vector to a one-element vector type whose element type
8019 // is the same as the result type, and extract the first element.
8020 if (!VT.isVector() && Op0VT.isFixedLengthVector()) {
8021 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 1);
8022 if (!isTypeLegal(VT: BVT))
8023 return SDValue();
8024 SDValue BVec = DAG.getBitcast(VT: BVT, V: Op0);
8025 return DAG.getExtractVectorElt(DL, VT, Vec: BVec, Idx: 0);
8026 }
8027 return SDValue();
8028 }
8029 case ISD::INTRINSIC_WO_CHAIN:
8030 return LowerINTRINSIC_WO_CHAIN(Op, DAG);
8031 case ISD::INTRINSIC_W_CHAIN:
8032 return LowerINTRINSIC_W_CHAIN(Op, DAG);
8033 case ISD::INTRINSIC_VOID:
8034 return LowerINTRINSIC_VOID(Op, DAG);
8035 case ISD::IS_FPCLASS:
8036 return LowerIS_FPCLASS(Op, DAG);
8037 case ISD::BITREVERSE: {
8038 MVT VT = Op.getSimpleValueType();
8039 if (VT.isFixedLengthVector()) {
8040 assert(Subtarget.hasStdExtZvbb());
8041 return lowerToScalableOp(Op, DAG);
8042 }
8043 SDLoc DL(Op);
8044 assert(Subtarget.hasStdExtZbkb() && "Unexpected custom legalization");
8045 assert(Op.getOpcode() == ISD::BITREVERSE && "Unexpected opcode");
8046 // Expand bitreverse to a bswap(rev8) followed by brev8.
8047 SDValue BSwap = DAG.getNode(Opcode: ISD::BSWAP, DL, VT, Operand: Op.getOperand(i: 0));
8048 return DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT, Operand: BSwap);
8049 }
8050 case ISD::TRUNCATE:
8051 case ISD::TRUNCATE_SSAT_S:
8052 case ISD::TRUNCATE_USAT_U:
8053 // Only custom-lower vector truncates
8054 if (!Op.getSimpleValueType().isVector())
8055 return Op;
8056 return lowerVectorTruncLike(Op, DAG);
8057 case ISD::ANY_EXTEND:
8058 case ISD::ZERO_EXTEND:
8059 if (Op.getOperand(i: 0).getValueType().isVector() &&
8060 Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8061 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ ExtTrueVal: 1);
8062 if (Op.getValueType().isScalableVector())
8063 return Op;
8064 return lowerToScalableOp(Op, DAG);
8065 case ISD::SIGN_EXTEND:
8066 if (Op.getOperand(i: 0).getValueType().isVector() &&
8067 Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8068 return lowerVectorMaskExt(Op, DAG, /*ExtVal*/ ExtTrueVal: -1);
8069 if (Op.getValueType().isScalableVector())
8070 return Op;
8071 return lowerToScalableOp(Op, DAG);
8072 case ISD::SPLAT_VECTOR_PARTS:
8073 return lowerSPLAT_VECTOR_PARTS(Op, DAG);
8074 case ISD::INSERT_VECTOR_ELT:
8075 return lowerINSERT_VECTOR_ELT(Op, DAG);
8076 case ISD::EXTRACT_VECTOR_ELT:
8077 return lowerEXTRACT_VECTOR_ELT(Op, DAG);
8078 case ISD::SCALAR_TO_VECTOR: {
8079 MVT VT = Op.getSimpleValueType();
8080 SDLoc DL(Op);
8081 SDValue Scalar = Op.getOperand(i: 0);
8082 if (VT.getVectorElementType() == MVT::i1) {
8083 MVT WideVT = VT.changeVectorElementType(EltVT: MVT::i8);
8084 SDValue V = DAG.getNode(Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: WideVT, Operand: Scalar);
8085 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: V);
8086 }
8087 MVT ContainerVT = VT;
8088 if (VT.isFixedLengthVector())
8089 ContainerVT = getContainerForFixedLengthVector(VT);
8090 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
8091
8092 SDValue V;
8093 if (VT.isFloatingPoint()) {
8094 V = DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT: ContainerVT,
8095 N1: DAG.getUNDEF(VT: ContainerVT), N2: Scalar, N3: VL);
8096 } else {
8097 Scalar = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: Subtarget.getXLenVT(), Operand: Scalar);
8098 V = DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT: ContainerVT,
8099 N1: DAG.getUNDEF(VT: ContainerVT), N2: Scalar, N3: VL);
8100 }
8101 if (VT.isFixedLengthVector())
8102 V = convertFromScalableVector(VT, V, DAG, Subtarget);
8103 return V;
8104 }
8105 case ISD::VSCALE: {
8106 MVT XLenVT = Subtarget.getXLenVT();
8107 MVT VT = Op.getSimpleValueType();
8108 SDLoc DL(Op);
8109 SDValue Res = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
8110 // We define our scalable vector types for lmul=1 to use a 64 bit known
8111 // minimum size. e.g. <vscale x 2 x i32>. VLENB is in bytes so we calculate
8112 // vscale as VLENB / 8.
8113 static_assert(RISCV::RVVBitsPerBlock == 64, "Unexpected bits per block!");
8114 if (Subtarget.getRealMinVLen() < RISCV::RVVBitsPerBlock)
8115 reportFatalInternalError(reason: "Support for VLEN==32 is incomplete.");
8116 // We assume VLENB is a multiple of 8. We manually choose the best shift
8117 // here because SimplifyDemandedBits isn't always able to simplify it.
8118 uint64_t Val = Op.getConstantOperandVal(i: 0);
8119 if (isPowerOf2_64(Value: Val)) {
8120 uint64_t Log2 = Log2_64(Value: Val);
8121 if (Log2 < 3) {
8122 SDNodeFlags Flags;
8123 Flags.setExact(true);
8124 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Res,
8125 N2: DAG.getConstant(Val: 3 - Log2, DL, VT: XLenVT), Flags);
8126 } else if (Log2 > 3) {
8127 Res = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: Res,
8128 N2: DAG.getConstant(Val: Log2 - 3, DL, VT: XLenVT));
8129 }
8130 } else if ((Val % 8) == 0) {
8131 // If the multiplier is a multiple of 8, scale it down to avoid needing
8132 // to shift the VLENB value.
8133 Res = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Res,
8134 N2: DAG.getConstant(Val: Val / 8, DL, VT: XLenVT));
8135 } else {
8136 SDNodeFlags Flags;
8137 Flags.setExact(true);
8138 SDValue VScale = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Res,
8139 N2: DAG.getConstant(Val: 3, DL, VT: XLenVT), Flags);
8140 Res = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: VScale,
8141 N2: DAG.getConstant(Val, DL, VT: XLenVT));
8142 }
8143 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Res);
8144 }
8145 case ISD::FPOWI: {
8146 // Custom promote f16 powi with illegal i32 integer type on RV64. Once
8147 // promoted this will be legalized into a libcall by LegalizeIntegerTypes.
8148 if (Op.getValueType() == MVT::f16 && Subtarget.is64Bit() &&
8149 Op.getOperand(i: 1).getValueType() == MVT::i32) {
8150 SDLoc DL(Op);
8151 SDValue Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op.getOperand(i: 0));
8152 SDValue Powi =
8153 DAG.getNode(Opcode: ISD::FPOWI, DL, VT: MVT::f32, N1: Op0, N2: Op.getOperand(i: 1));
8154 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: MVT::f16, N1: Powi,
8155 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
8156 }
8157 return SDValue();
8158 }
8159 case ISD::FMAXIMUM:
8160 case ISD::FMINIMUM:
8161 if (isPromotedOpNeedingSplit(Op, Subtarget))
8162 return SplitVectorOp(Op, DAG);
8163 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
8164 case ISD::FP_EXTEND:
8165 case ISD::FP_ROUND:
8166 return lowerVectorFPExtendOrRoundLike(Op, DAG);
8167 case ISD::STRICT_FP_ROUND:
8168 case ISD::STRICT_FP_EXTEND:
8169 return lowerStrictFPExtendOrRoundLike(Op, DAG);
8170 case ISD::SINT_TO_FP:
8171 case ISD::UINT_TO_FP:
8172 if (Op.getValueType().isVector() &&
8173 ((Op.getValueType().getScalarType() == MVT::f16 &&
8174 (Subtarget.hasVInstructionsF16Minimal() &&
8175 !Subtarget.hasVInstructionsF16())) ||
8176 Op.getValueType().getScalarType() == MVT::bf16)) {
8177 if (isPromotedOpNeedingSplit(Op, Subtarget))
8178 return SplitVectorOp(Op, DAG);
8179 // int -> f32
8180 SDLoc DL(Op);
8181 MVT NVT =
8182 MVT::getVectorVT(VT: MVT::f32, EC: Op.getValueType().getVectorElementCount());
8183 SDValue NC = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: NVT, Ops: Op->ops());
8184 // f32 -> [b]f16
8185 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Op.getValueType(), N1: NC,
8186 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
8187 }
8188 [[fallthrough]];
8189 case ISD::FP_TO_SINT:
8190 case ISD::FP_TO_UINT:
8191 if (SDValue Op1 = Op.getOperand(i: 0);
8192 Op1.getValueType().isVector() &&
8193 ((Op1.getValueType().getScalarType() == MVT::f16 &&
8194 (Subtarget.hasVInstructionsF16Minimal() &&
8195 !Subtarget.hasVInstructionsF16())) ||
8196 Op1.getValueType().getScalarType() == MVT::bf16)) {
8197 if (isPromotedOpNeedingSplit(Op: Op1, Subtarget))
8198 return SplitVectorOp(Op, DAG);
8199 // [b]f16 -> f32
8200 SDLoc DL(Op);
8201 MVT NVT = MVT::getVectorVT(VT: MVT::f32,
8202 EC: Op1.getValueType().getVectorElementCount());
8203 SDValue WidenVec = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NVT, Operand: Op1);
8204 // f32 -> int
8205 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: WidenVec);
8206 }
8207 [[fallthrough]];
8208 case ISD::STRICT_FP_TO_SINT:
8209 case ISD::STRICT_FP_TO_UINT:
8210 case ISD::STRICT_SINT_TO_FP:
8211 case ISD::STRICT_UINT_TO_FP: {
8212 // RVV can only do fp<->int conversions to types half/double the size as
8213 // the source. We custom-lower any conversions that do two hops into
8214 // sequences.
8215 MVT VT = Op.getSimpleValueType();
8216 if (VT.isScalarInteger())
8217 return lowerFP_TO_INT(Op, DAG, Subtarget);
8218 bool IsStrict = Op->isStrictFPOpcode();
8219 SDValue Src = Op.getOperand(i: 0 + IsStrict);
8220 MVT SrcVT = Src.getSimpleValueType();
8221 if (SrcVT.isScalarInteger())
8222 return lowerINT_TO_FP(Op, DAG, Subtarget);
8223 if (!VT.isVector())
8224 return Op;
8225 SDLoc DL(Op);
8226 MVT EltVT = VT.getVectorElementType();
8227 MVT SrcEltVT = SrcVT.getVectorElementType();
8228 unsigned EltSize = EltVT.getSizeInBits();
8229 unsigned SrcEltSize = SrcEltVT.getSizeInBits();
8230 assert(isPowerOf2_32(EltSize) && isPowerOf2_32(SrcEltSize) &&
8231 "Unexpected vector element types");
8232
8233 bool IsInt2FP = SrcEltVT.isInteger();
8234 // Widening conversions
8235 if (EltSize > (2 * SrcEltSize)) {
8236 if (IsInt2FP) {
8237 // Do a regular integer sign/zero extension then convert to float.
8238 MVT IVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: EltSize / 2),
8239 EC: VT.getVectorElementCount());
8240 unsigned ExtOpcode = (Op.getOpcode() == ISD::UINT_TO_FP ||
8241 Op.getOpcode() == ISD::STRICT_UINT_TO_FP)
8242 ? ISD::ZERO_EXTEND
8243 : ISD::SIGN_EXTEND;
8244 SDValue Ext = DAG.getNode(Opcode: ExtOpcode, DL, VT: IVecVT, Operand: Src);
8245 if (IsStrict)
8246 return DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: Op->getVTList(),
8247 N1: Op.getOperand(i: 0), N2: Ext);
8248 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT, Operand: Ext);
8249 }
8250 // FP2Int
8251 assert(SrcEltVT == MVT::f16 && "Unexpected FP_TO_[US]INT lowering");
8252 // Do one doubling fp_extend then complete the operation by converting
8253 // to int.
8254 MVT InterimFVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
8255 if (IsStrict) {
8256 auto [FExt, Chain] =
8257 DAG.getStrictFPExtendOrRound(Op: Src, Chain: Op.getOperand(i: 0), DL, VT: InterimFVT);
8258 return DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: Op->getVTList(), N1: Chain, N2: FExt);
8259 }
8260 SDValue FExt = DAG.getFPExtendOrRound(Op: Src, DL, VT: InterimFVT);
8261 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT, Operand: FExt);
8262 }
8263
8264 // Narrowing conversions
8265 if (SrcEltSize > (2 * EltSize)) {
8266 if (IsInt2FP) {
8267 // One narrowing int_to_fp, then an fp_round.
8268 assert(EltVT == MVT::f16 && "Unexpected [US]_TO_FP lowering");
8269 MVT InterimFVT = MVT::getVectorVT(VT: MVT::f32, EC: VT.getVectorElementCount());
8270 if (IsStrict) {
8271 SDValue Int2FP = DAG.getNode(Opcode: Op.getOpcode(), DL,
8272 VTList: DAG.getVTList(VT1: InterimFVT, VT2: MVT::Other),
8273 N1: Op.getOperand(i: 0), N2: Src);
8274 SDValue Chain = Int2FP.getValue(R: 1);
8275 return DAG.getStrictFPExtendOrRound(Op: Int2FP, Chain, DL, VT).first;
8276 }
8277 SDValue Int2FP = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: InterimFVT, Operand: Src);
8278 return DAG.getFPExtendOrRound(Op: Int2FP, DL, VT);
8279 }
8280 // FP2Int
8281 // One narrowing fp_to_int, then truncate the integer. If the float isn't
8282 // representable by the integer, the result is poison.
8283 MVT IVecVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SrcEltSize / 2),
8284 EC: VT.getVectorElementCount());
8285 if (IsStrict) {
8286 SDValue FP2Int =
8287 DAG.getNode(Opcode: Op.getOpcode(), DL, VTList: DAG.getVTList(VT1: IVecVT, VT2: MVT::Other),
8288 N1: Op.getOperand(i: 0), N2: Src);
8289 SDValue Res = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FP2Int);
8290 return DAG.getMergeValues(Ops: {Res, FP2Int.getValue(R: 1)}, dl: DL);
8291 }
8292 SDValue FP2Int = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: IVecVT, Operand: Src);
8293 if (EltSize == 1)
8294 // The integer should be 0 or 1/-1, so compare the integer result to 0.
8295 return DAG.getSetCC(DL, VT, LHS: DAG.getConstant(Val: 0, DL, VT: IVecVT), RHS: FP2Int,
8296 Cond: ISD::SETNE);
8297 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FP2Int);
8298 }
8299
8300 // Scalable vectors can exit here. Patterns will handle equally-sized
8301 // conversions halving/doubling ones.
8302 if (!VT.isFixedLengthVector())
8303 return Op;
8304
8305 // For fixed-length vectors we lower to a custom "VL" node.
8306 unsigned RVVOpc = 0;
8307 switch (Op.getOpcode()) {
8308 default:
8309 llvm_unreachable("Impossible opcode");
8310 case ISD::FP_TO_SINT:
8311 RVVOpc = RISCVISD::VFCVT_RTZ_X_F_VL;
8312 break;
8313 case ISD::FP_TO_UINT:
8314 RVVOpc = RISCVISD::VFCVT_RTZ_XU_F_VL;
8315 break;
8316 case ISD::SINT_TO_FP:
8317 RVVOpc = RISCVISD::SINT_TO_FP_VL;
8318 break;
8319 case ISD::UINT_TO_FP:
8320 RVVOpc = RISCVISD::UINT_TO_FP_VL;
8321 break;
8322 case ISD::STRICT_FP_TO_SINT:
8323 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_X_F_VL;
8324 break;
8325 case ISD::STRICT_FP_TO_UINT:
8326 RVVOpc = RISCVISD::STRICT_VFCVT_RTZ_XU_F_VL;
8327 break;
8328 case ISD::STRICT_SINT_TO_FP:
8329 RVVOpc = RISCVISD::STRICT_SINT_TO_FP_VL;
8330 break;
8331 case ISD::STRICT_UINT_TO_FP:
8332 RVVOpc = RISCVISD::STRICT_UINT_TO_FP_VL;
8333 break;
8334 }
8335
8336 MVT ContainerVT = getContainerForFixedLengthVector(VT);
8337 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
8338 assert(ContainerVT.getVectorElementCount() == SrcContainerVT.getVectorElementCount() &&
8339 "Expected same element count");
8340
8341 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
8342
8343 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
8344 if (IsStrict) {
8345 Src = DAG.getNode(Opcode: RVVOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
8346 N1: Op.getOperand(i: 0), N2: Src, N3: Mask, N4: VL);
8347 SDValue SubVec = convertFromScalableVector(VT, V: Src, DAG, Subtarget);
8348 return DAG.getMergeValues(Ops: {SubVec, Src.getValue(R: 1)}, dl: DL);
8349 }
8350 Src = DAG.getNode(Opcode: RVVOpc, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
8351 return convertFromScalableVector(VT, V: Src, DAG, Subtarget);
8352 }
8353 case ISD::FP_TO_SINT_SAT:
8354 case ISD::FP_TO_UINT_SAT:
8355 return lowerFP_TO_INT_SAT(Op, DAG, Subtarget);
8356 case ISD::FP_TO_BF16: {
8357 // Custom lower to ensure the libcall return is passed in an FPR on hard
8358 // float ABIs.
8359 assert(!Subtarget.isSoftFPABI() && "Unexpected custom legalization");
8360 SDLoc DL(Op);
8361 MakeLibCallOptions CallOptions;
8362 RTLIB::Libcall LC =
8363 RTLIB::getFPROUND(OpVT: Op.getOperand(i: 0).getValueType(), RetVT: MVT::bf16);
8364 SDValue Res =
8365 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op.getOperand(i: 0), CallOptions, dl: DL).first;
8366 if (Subtarget.is64Bit())
8367 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Res);
8368 return DAG.getBitcast(VT: MVT::i32, V: Res);
8369 }
8370 case ISD::BF16_TO_FP: {
8371 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalization");
8372 MVT VT = Op.getSimpleValueType();
8373 SDLoc DL(Op);
8374 Op = DAG.getNode(
8375 Opcode: ISD::SHL, DL, VT: Op.getOperand(i: 0).getValueType(), N1: Op.getOperand(i: 0),
8376 N2: DAG.getShiftAmountConstant(Val: 16, VT: Op.getOperand(i: 0).getValueType(), DL));
8377 SDValue Res = Subtarget.is64Bit()
8378 ? DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Op)
8379 : DAG.getBitcast(VT: MVT::f32, V: Op);
8380 // fp_extend if the target VT is bigger than f32.
8381 if (VT != MVT::f32)
8382 return DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT, Operand: Res);
8383 return Res;
8384 }
8385 case ISD::STRICT_FP_TO_FP16:
8386 case ISD::FP_TO_FP16: {
8387 // Custom lower to ensure the libcall return is passed in an FPR on hard
8388 // float ABIs.
8389 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8390 SDLoc DL(Op);
8391 MakeLibCallOptions CallOptions;
8392 bool IsStrict = Op->isStrictFPOpcode();
8393 SDValue Op0 = IsStrict ? Op.getOperand(i: 1) : Op.getOperand(i: 0);
8394 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8395 RTLIB::Libcall LC = RTLIB::getFPROUND(OpVT: Op0.getValueType(), RetVT: MVT::f16);
8396 SDValue Res;
8397 std::tie(args&: Res, args&: Chain) =
8398 makeLibCall(DAG, LC, RetVT: MVT::f32, Ops: Op0, CallOptions, dl: DL, Chain);
8399 if (Subtarget.is64Bit())
8400 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Res);
8401 SDValue Result = DAG.getBitcast(VT: MVT::i32, V: IsStrict ? Res.getValue(R: 0) : Res);
8402 if (IsStrict)
8403 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
8404 return Result;
8405 }
8406 case ISD::STRICT_FP16_TO_FP:
8407 case ISD::FP16_TO_FP: {
8408 // Custom lower to ensure the libcall argument is passed in an FPR on hard
8409 // float ABIs.
8410 assert(Subtarget.hasStdExtFOrZfinx() && "Unexpected custom legalisation");
8411 SDLoc DL(Op);
8412 MakeLibCallOptions CallOptions;
8413 bool IsStrict = Op->isStrictFPOpcode();
8414 SDValue Op0 = IsStrict ? Op.getOperand(i: 1) : Op.getOperand(i: 0);
8415 SDValue Chain = IsStrict ? Op.getOperand(i: 0) : SDValue();
8416 SDValue Arg = Subtarget.is64Bit()
8417 ? DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Op0)
8418 : DAG.getBitcast(VT: MVT::f32, V: Op0);
8419 SDValue Res;
8420 std::tie(args&: Res, args&: Chain) = makeLibCall(DAG, LC: RTLIB::FPEXT_F16_F32, RetVT: MVT::f32, Ops: Arg,
8421 CallOptions, dl: DL, Chain);
8422 if (IsStrict)
8423 return DAG.getMergeValues(Ops: {Res, Chain}, dl: DL);
8424 return Res;
8425 }
8426 case ISD::FTRUNC:
8427 case ISD::FCEIL:
8428 case ISD::FFLOOR:
8429 case ISD::FNEARBYINT:
8430 case ISD::FRINT:
8431 case ISD::FROUND:
8432 case ISD::FROUNDEVEN:
8433 if (isPromotedOpNeedingSplit(Op, Subtarget))
8434 return SplitVectorOp(Op, DAG);
8435 return lowerFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
8436 case ISD::LRINT:
8437 case ISD::LLRINT:
8438 case ISD::LROUND:
8439 case ISD::LLROUND: {
8440 if (Op.getValueType().isVector())
8441 return lowerVectorXRINT_XROUND(Op, DAG, Subtarget);
8442 assert(Op.getOperand(0).getValueType() == MVT::f16 &&
8443 "Unexpected custom legalisation");
8444 SDLoc DL(Op);
8445 SDValue Ext = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op.getOperand(i: 0));
8446 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(), Operand: Ext);
8447 }
8448 case ISD::STRICT_LRINT:
8449 case ISD::STRICT_LLRINT:
8450 case ISD::STRICT_LROUND:
8451 case ISD::STRICT_LLROUND: {
8452 assert(Op.getOperand(1).getValueType() == MVT::f16 &&
8453 "Unexpected custom legalisation");
8454 SDLoc DL(Op);
8455 SDValue Ext = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
8456 Ops: {Op.getOperand(i: 0), Op.getOperand(i: 1)});
8457 return DAG.getNode(Opcode: Op.getOpcode(), DL, ResultTys: {Op.getValueType(), MVT::Other},
8458 Ops: {Ext.getValue(R: 1), Ext.getValue(R: 0)});
8459 }
8460 case ISD::VECREDUCE_ADD:
8461 case ISD::VECREDUCE_UMAX:
8462 case ISD::VECREDUCE_SMAX:
8463 case ISD::VECREDUCE_UMIN:
8464 case ISD::VECREDUCE_SMIN:
8465 return lowerVECREDUCE(Op, DAG);
8466 case ISD::VECREDUCE_AND:
8467 case ISD::VECREDUCE_OR:
8468 case ISD::VECREDUCE_XOR:
8469 if (Op.getOperand(i: 0).getValueType().getVectorElementType() == MVT::i1)
8470 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ false);
8471 return lowerVECREDUCE(Op, DAG);
8472 case ISD::VECREDUCE_FADD:
8473 case ISD::VECREDUCE_SEQ_FADD:
8474 case ISD::VECREDUCE_FMIN:
8475 case ISD::VECREDUCE_FMAX:
8476 case ISD::VECREDUCE_FMAXIMUM:
8477 case ISD::VECREDUCE_FMINIMUM:
8478 return lowerFPVECREDUCE(Op, DAG);
8479 case ISD::VP_REDUCE_ADD:
8480 case ISD::VP_REDUCE_UMAX:
8481 case ISD::VP_REDUCE_SMAX:
8482 case ISD::VP_REDUCE_UMIN:
8483 case ISD::VP_REDUCE_SMIN:
8484 case ISD::VP_REDUCE_FADD:
8485 case ISD::VP_REDUCE_SEQ_FADD:
8486 case ISD::VP_REDUCE_FMIN:
8487 case ISD::VP_REDUCE_FMAX:
8488 case ISD::VP_REDUCE_FMINIMUM:
8489 case ISD::VP_REDUCE_FMAXIMUM:
8490 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 1), Subtarget))
8491 return SplitVectorReductionOp(Op, DAG);
8492 return lowerVPREDUCE(Op, DAG);
8493 case ISD::VP_REDUCE_AND:
8494 case ISD::VP_REDUCE_OR:
8495 case ISD::VP_REDUCE_XOR:
8496 if (Op.getOperand(i: 1).getValueType().getVectorElementType() == MVT::i1)
8497 return lowerVectorMaskVecReduction(Op, DAG, /*IsVP*/ true);
8498 return lowerVPREDUCE(Op, DAG);
8499 case ISD::VP_CTTZ_ELTS:
8500 case ISD::VP_CTTZ_ELTS_ZERO_UNDEF:
8501 return lowerVPCttzElements(Op, DAG);
8502 case ISD::UNDEF: {
8503 MVT ContainerVT = getContainerForFixedLengthVector(VT: Op.getSimpleValueType());
8504 return convertFromScalableVector(VT: Op.getSimpleValueType(),
8505 V: DAG.getUNDEF(VT: ContainerVT), DAG, Subtarget);
8506 }
8507 case ISD::INSERT_SUBVECTOR:
8508 return lowerINSERT_SUBVECTOR(Op, DAG);
8509 case ISD::EXTRACT_SUBVECTOR:
8510 return lowerEXTRACT_SUBVECTOR(Op, DAG);
8511 case ISD::VECTOR_DEINTERLEAVE:
8512 return lowerVECTOR_DEINTERLEAVE(Op, DAG);
8513 case ISD::VECTOR_INTERLEAVE:
8514 return lowerVECTOR_INTERLEAVE(Op, DAG);
8515 case ISD::STEP_VECTOR:
8516 return lowerSTEP_VECTOR(Op, DAG);
8517 case ISD::VECTOR_REVERSE:
8518 return lowerVECTOR_REVERSE(Op, DAG);
8519 case ISD::VECTOR_SPLICE_LEFT:
8520 case ISD::VECTOR_SPLICE_RIGHT:
8521 return lowerVECTOR_SPLICE(Op, DAG);
8522 case ISD::BUILD_VECTOR: {
8523 MVT VT = Op.getSimpleValueType();
8524 MVT EltVT = VT.getVectorElementType();
8525 if (!Subtarget.is64Bit() && EltVT == MVT::i64)
8526 return lowerBuildVectorViaVID(Op, DAG, Subtarget);
8527 return lowerBUILD_VECTOR(Op, DAG, Subtarget);
8528 }
8529 case ISD::SPLAT_VECTOR: {
8530 MVT VT = Op.getSimpleValueType();
8531 MVT EltVT = VT.getVectorElementType();
8532 if ((EltVT == MVT::f16 && !Subtarget.hasStdExtZvfh()) ||
8533 EltVT == MVT::bf16) {
8534 SDLoc DL(Op);
8535 SDValue Elt;
8536 if ((EltVT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) ||
8537 (EltVT == MVT::f16 && Subtarget.hasStdExtZfhmin()))
8538 Elt = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: Subtarget.getXLenVT(),
8539 Operand: Op.getOperand(i: 0));
8540 else
8541 Elt = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Op.getOperand(i: 0));
8542 MVT IVT = VT.changeVectorElementType(EltVT: MVT::i16);
8543 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT,
8544 Operand: DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: IVT, Operand: Elt));
8545 }
8546
8547 if (EltVT == MVT::i1)
8548 return lowerVectorMaskSplat(Op, DAG);
8549 return SDValue();
8550 }
8551 case ISD::VECTOR_SHUFFLE:
8552 return lowerVECTOR_SHUFFLE(Op, DAG, Subtarget);
8553 case ISD::CONCAT_VECTORS: {
8554 // Split CONCAT_VECTORS into a series of INSERT_SUBVECTOR nodes. This is
8555 // better than going through the stack, as the default expansion does.
8556 SDLoc DL(Op);
8557 MVT VT = Op.getSimpleValueType();
8558 MVT ContainerVT = VT;
8559 if (VT.isFixedLengthVector())
8560 ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
8561
8562 // Recursively split concat_vectors with more than 2 operands:
8563 //
8564 // concat_vector op1, op2, op3, op4
8565 // ->
8566 // concat_vector (concat_vector op1, op2), (concat_vector op3, op4)
8567 //
8568 // This reduces the length of the chain of vslideups and allows us to
8569 // perform the vslideups at a smaller LMUL, limited to MF2.
8570 if (Op.getNumOperands() > 2 &&
8571 ContainerVT.bitsGE(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT))) {
8572 MVT HalfVT = VT.getHalfNumVectorElementsVT();
8573 assert(isPowerOf2_32(Op.getNumOperands()));
8574 size_t HalfNumOps = Op.getNumOperands() / 2;
8575 SDValue Lo = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
8576 Ops: Op->ops().take_front(N: HalfNumOps));
8577 SDValue Hi = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: HalfVT,
8578 Ops: Op->ops().drop_front(N: HalfNumOps));
8579 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, N1: Lo, N2: Hi);
8580 }
8581
8582 unsigned NumOpElts =
8583 Op.getOperand(i: 0).getSimpleValueType().getVectorMinNumElements();
8584 SDValue Vec = DAG.getUNDEF(VT);
8585 for (const auto &OpIdx : enumerate(First: Op->ops())) {
8586 SDValue SubVec = OpIdx.value();
8587 // Don't insert undef subvectors.
8588 if (SubVec.isUndef())
8589 continue;
8590 Vec = DAG.getInsertSubvector(DL, Vec, SubVec, Idx: OpIdx.index() * NumOpElts);
8591 }
8592 return Vec;
8593 }
8594 case ISD::LOAD: {
8595 auto *Load = cast<LoadSDNode>(Val&: Op);
8596 EVT VT = Load->getValueType(ResNo: 0);
8597 if (VT == MVT::f64) {
8598 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
8599 !Subtarget.is64Bit() && "Unexpected custom legalisation");
8600
8601 // Replace a double precision load with two i32 loads and a BuildPairF64.
8602 SDLoc DL(Op);
8603 SDValue BasePtr = Load->getBasePtr();
8604 SDValue Chain = Load->getChain();
8605
8606 SDValue Lo =
8607 DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo(),
8608 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
8609 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: 4));
8610 SDValue Hi = DAG.getLoad(
8611 VT: MVT::i32, dl: DL, Chain, Ptr: BasePtr, PtrInfo: Load->getPointerInfo().getWithOffset(O: 4),
8612 Alignment: Load->getBaseAlign(), MMOFlags: Load->getMemOperand()->getFlags());
8613 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: Lo.getValue(R: 1),
8614 N2: Hi.getValue(R: 1));
8615
8616 // For big-endian, swap the order of Lo and Hi.
8617 if (!Subtarget.isLittleEndian())
8618 std::swap(a&: Lo, b&: Hi);
8619
8620 SDValue Pair = DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
8621 return DAG.getMergeValues(Ops: {Pair, Chain}, dl: DL);
8622 }
8623
8624 if (VT == MVT::bf16)
8625 return lowerXAndesBfHCvtBFloat16Load(Op, DAG);
8626
8627 // Handle normal vector tuple load.
8628 if (VT.isRISCVVectorTuple()) {
8629 SDLoc DL(Op);
8630 MVT XLenVT = Subtarget.getXLenVT();
8631 unsigned NF = VT.getRISCVVectorTupleNumFields();
8632 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
8633 unsigned NumElts = Sz / (NF * 8);
8634 int Log2LMUL = Log2_64(Value: NumElts) - 3;
8635
8636 auto Flag = SDNodeFlags();
8637 Flag.setNoUnsignedWrap(true);
8638 SDValue Ret = DAG.getUNDEF(VT);
8639 SDValue BasePtr = Load->getBasePtr();
8640 SDValue VROffset = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
8641 VROffset =
8642 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VROffset,
8643 N2: DAG.getConstant(Val: std::max(a: Log2LMUL, b: 0), DL, VT: XLenVT));
8644 SmallVector<SDValue, 8> OutChains;
8645
8646 // Load NF vector registers and combine them to a vector tuple.
8647 for (unsigned i = 0; i < NF; ++i) {
8648 SDValue LoadVal = DAG.getLoad(
8649 VT: MVT::getScalableVectorVT(VT: MVT::i8, NumElements: NumElts), dl: DL, Chain: Load->getChain(),
8650 Ptr: BasePtr, PtrInfo: MachinePointerInfo(Load->getAddressSpace()), Alignment: Align(8));
8651 OutChains.push_back(Elt: LoadVal.getValue(R: 1));
8652 Ret = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT, N1: Ret, N2: LoadVal,
8653 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
8654 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: BasePtr, N2: VROffset, Flags: Flag);
8655 }
8656 return DAG.getMergeValues(
8657 Ops: {Ret, DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains)}, dl: DL);
8658 }
8659
8660 if (auto V = expandUnalignedRVVLoad(Op, DAG))
8661 return V;
8662 if (Op.getValueType().isFixedLengthVector())
8663 return lowerFixedLengthVectorLoadToRVV(Op, DAG);
8664 return Op;
8665 }
8666 case ISD::STORE: {
8667 auto *Store = cast<StoreSDNode>(Val&: Op);
8668 SDValue StoredVal = Store->getValue();
8669 EVT VT = StoredVal.getValueType();
8670 if (Subtarget.hasStdExtP()) {
8671 if (VT == MVT::v2i16 || VT == MVT::v4i8) {
8672 SDValue DL(Op);
8673 SDValue Cast = DAG.getBitcast(VT: MVT::i32, V: StoredVal);
8674 SDValue NewStore =
8675 DAG.getStore(Chain: Store->getChain(), dl: DL, Val: Cast, Ptr: Store->getBasePtr(),
8676 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
8677 MMOFlags: Store->getMemOperand()->getFlags());
8678 return NewStore;
8679 }
8680 }
8681 if (VT == MVT::f64) {
8682 assert(Subtarget.hasStdExtZdinx() && !Subtarget.hasStdExtZilsd() &&
8683 !Subtarget.is64Bit() && "Unexpected custom legalisation");
8684
8685 // Replace a double precision store with a SplitF64 and i32 stores.
8686 SDValue DL(Op);
8687 SDValue BasePtr = Store->getBasePtr();
8688 SDValue Chain = Store->getChain();
8689 SDValue Split = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
8690 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: StoredVal);
8691
8692 SDValue Lo = Split.getValue(R: 0);
8693 SDValue Hi = Split.getValue(R: 1);
8694
8695 // For big-endian, swap the order of Lo and Hi before storing.
8696 if (!Subtarget.isLittleEndian())
8697 std::swap(a&: Lo, b&: Hi);
8698
8699 SDValue LoStore = DAG.getStore(
8700 Chain, dl: DL, Val: Lo, Ptr: BasePtr, PtrInfo: Store->getPointerInfo(),
8701 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
8702 BasePtr = DAG.getObjectPtrOffset(SL: DL, Ptr: BasePtr, Offset: TypeSize::getFixed(ExactSize: 4));
8703 SDValue HiStore = DAG.getStore(
8704 Chain, dl: DL, Val: Hi, Ptr: BasePtr, PtrInfo: Store->getPointerInfo().getWithOffset(O: 4),
8705 Alignment: Store->getBaseAlign(), MMOFlags: Store->getMemOperand()->getFlags());
8706 return DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, N1: LoStore, N2: HiStore);
8707 }
8708 if (VT == MVT::i64) {
8709 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
8710 "Unexpected custom legalisation");
8711 if (Store->isTruncatingStore())
8712 return SDValue();
8713
8714 if (Store->getAlign() < Subtarget.getZilsdAlign())
8715 return SDValue();
8716
8717 SDLoc DL(Op);
8718 SDValue Lo = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i32, N1: StoredVal,
8719 N2: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
8720 SDValue Hi = DAG.getNode(Opcode: ISD::EXTRACT_ELEMENT, DL, VT: MVT::i32, N1: StoredVal,
8721 N2: DAG.getTargetConstant(Val: 1, DL, VT: MVT::i32));
8722
8723 return DAG.getMemIntrinsicNode(
8724 Opcode: RISCVISD::SD_RV32, dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
8725 Ops: {Store->getChain(), Lo, Hi, Store->getBasePtr()}, MemVT: MVT::i64,
8726 MMO: Store->getMemOperand());
8727 }
8728
8729 if (VT == MVT::bf16)
8730 return lowerXAndesBfHCvtBFloat16Store(Op, DAG);
8731
8732 // Handle normal vector tuple store.
8733 if (VT.isRISCVVectorTuple()) {
8734 SDLoc DL(Op);
8735 MVT XLenVT = Subtarget.getXLenVT();
8736 unsigned NF = VT.getRISCVVectorTupleNumFields();
8737 unsigned Sz = VT.getSizeInBits().getKnownMinValue();
8738 unsigned NumElts = Sz / (NF * 8);
8739 int Log2LMUL = Log2_64(Value: NumElts) - 3;
8740
8741 auto Flag = SDNodeFlags();
8742 Flag.setNoUnsignedWrap(true);
8743 SDValue Ret;
8744 SDValue Chain = Store->getChain();
8745 SDValue BasePtr = Store->getBasePtr();
8746 SDValue VROffset = DAG.getNode(Opcode: RISCVISD::READ_VLENB, DL, VT: XLenVT);
8747 VROffset =
8748 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VROffset,
8749 N2: DAG.getConstant(Val: std::max(a: Log2LMUL, b: 0), DL, VT: XLenVT));
8750
8751 // Extract subregisters in a vector tuple and store them individually.
8752 for (unsigned i = 0; i < NF; ++i) {
8753 auto Extract =
8754 DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL,
8755 VT: MVT::getScalableVectorVT(VT: MVT::i8, NumElements: NumElts), N1: StoredVal,
8756 N2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
8757 Ret = DAG.getStore(Chain, dl: DL, Val: Extract, Ptr: BasePtr,
8758 PtrInfo: MachinePointerInfo(Store->getAddressSpace()),
8759 Alignment: Store->getBaseAlign(),
8760 MMOFlags: Store->getMemOperand()->getFlags());
8761 Chain = Ret.getValue(R: 0);
8762 BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: BasePtr, N2: VROffset, Flags: Flag);
8763 }
8764 return Ret;
8765 }
8766
8767 if (auto V = expandUnalignedRVVStore(Op, DAG))
8768 return V;
8769 if (Op.getOperand(i: 1).getValueType().isFixedLengthVector())
8770 return lowerFixedLengthVectorStoreToRVV(Op, DAG);
8771 return Op;
8772 }
8773 case ISD::VP_LOAD:
8774 if (SDValue V = expandUnalignedVPLoad(Op, DAG))
8775 return V;
8776 [[fallthrough]];
8777 case ISD::MLOAD:
8778 return lowerMaskedLoad(Op, DAG);
8779 case ISD::VP_LOAD_FF:
8780 return lowerLoadFF(Op, DAG);
8781 case ISD::VP_STORE:
8782 if (SDValue V = expandUnalignedVPStore(Op, DAG))
8783 return V;
8784 [[fallthrough]];
8785 case ISD::MSTORE:
8786 return lowerMaskedStore(Op, DAG);
8787 case ISD::VECTOR_COMPRESS:
8788 return lowerVectorCompress(Op, DAG);
8789 case ISD::SELECT_CC: {
8790 // This occurs because we custom legalize SETGT and SETUGT for setcc. That
8791 // causes LegalizeDAG to think we need to custom legalize select_cc. Expand
8792 // into separate SETCC+SELECT just like LegalizeDAG.
8793 SDValue Tmp1 = Op.getOperand(i: 0);
8794 SDValue Tmp2 = Op.getOperand(i: 1);
8795 SDValue True = Op.getOperand(i: 2);
8796 SDValue False = Op.getOperand(i: 3);
8797 EVT VT = Op.getValueType();
8798 SDValue CC = Op.getOperand(i: 4);
8799 EVT CmpVT = Tmp1.getValueType();
8800 EVT CCVT =
8801 getSetCCResultType(DL: DAG.getDataLayout(), Context&: *DAG.getContext(), VT: CmpVT);
8802 SDLoc DL(Op);
8803 SDValue Cond =
8804 DAG.getNode(Opcode: ISD::SETCC, DL, VT: CCVT, N1: Tmp1, N2: Tmp2, N3: CC, Flags: Op->getFlags());
8805 return DAG.getSelect(DL, VT, Cond, LHS: True, RHS: False);
8806 }
8807 case ISD::SETCC: {
8808 MVT OpVT = Op.getOperand(i: 0).getSimpleValueType();
8809 if (OpVT.isScalarInteger()) {
8810 MVT VT = Op.getSimpleValueType();
8811 SDValue LHS = Op.getOperand(i: 0);
8812 SDValue RHS = Op.getOperand(i: 1);
8813 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
8814 assert((CCVal == ISD::SETGT || CCVal == ISD::SETUGT) &&
8815 "Unexpected CondCode");
8816
8817 SDLoc DL(Op);
8818
8819 // If the RHS is a constant in the range [-2049, 0) or (0, 2046], we can
8820 // convert this to the equivalent of (set(u)ge X, C+1) by using
8821 // (xori (slti(u) X, C+1), 1). This avoids materializing a small constant
8822 // in a register.
8823 if (isa<ConstantSDNode>(Val: RHS)) {
8824 int64_t Imm = cast<ConstantSDNode>(Val&: RHS)->getSExtValue();
8825 if (Imm != 0 && isInt<12>(x: (uint64_t)Imm + 1)) {
8826 // If this is an unsigned compare and the constant is -1, incrementing
8827 // the constant would change behavior. The result should be false.
8828 if (CCVal == ISD::SETUGT && Imm == -1)
8829 return DAG.getConstant(Val: 0, DL, VT);
8830 // Using getSetCCSwappedOperands will convert SET(U)GT->SET(U)LT.
8831 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
8832 SDValue SetCC = DAG.getSetCC(
8833 DL, VT, LHS, RHS: DAG.getSignedConstant(Val: Imm + 1, DL, VT: OpVT), Cond: CCVal);
8834 return DAG.getLogicalNOT(DL, Val: SetCC, VT);
8835 }
8836 // Lower (setugt X, 2047) as (setne (srl X, 11), 0).
8837 if (CCVal == ISD::SETUGT && Imm == 2047) {
8838 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL, VT: OpVT, N1: LHS,
8839 N2: DAG.getShiftAmountConstant(Val: 11, VT: OpVT, DL));
8840 return DAG.getSetCC(DL, VT, LHS: Shift, RHS: DAG.getConstant(Val: 0, DL, VT: OpVT),
8841 Cond: ISD::SETNE);
8842 }
8843 }
8844
8845 // Not a constant we could handle, swap the operands and condition code to
8846 // SETLT/SETULT.
8847 CCVal = ISD::getSetCCSwappedOperands(Operation: CCVal);
8848 return DAG.getSetCC(DL, VT, LHS: RHS, RHS: LHS, Cond: CCVal);
8849 }
8850
8851 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 0), Subtarget))
8852 return SplitVectorOp(Op, DAG);
8853
8854 return lowerToScalableOp(Op, DAG);
8855 }
8856 case ISD::ADD:
8857 case ISD::SUB:
8858 case ISD::MUL:
8859 case ISD::MULHS:
8860 case ISD::MULHU:
8861 case ISD::AND:
8862 case ISD::OR:
8863 case ISD::XOR:
8864 case ISD::SDIV:
8865 case ISD::SREM:
8866 case ISD::UDIV:
8867 case ISD::UREM:
8868 case ISD::BSWAP:
8869 case ISD::CTPOP:
8870 case ISD::VSELECT:
8871 return lowerToScalableOp(Op, DAG);
8872 case ISD::SHL:
8873 case ISD::SRL:
8874 case ISD::SRA:
8875 if (Op.getSimpleValueType().isFixedLengthVector()) {
8876 if (Subtarget.hasStdExtP()) {
8877 SDValue ShAmtVec = Op.getOperand(i: 1);
8878 SDValue SplatVal;
8879 if (ShAmtVec.getOpcode() == ISD::SPLAT_VECTOR)
8880 SplatVal = ShAmtVec.getOperand(i: 0);
8881 else if (ShAmtVec.getOpcode() == ISD::BUILD_VECTOR)
8882 SplatVal = cast<BuildVectorSDNode>(Val&: ShAmtVec)->getSplatValue();
8883
8884 if (!SplatVal)
8885 return DAG.UnrollVectorOp(N: Op.getNode());
8886
8887 unsigned Opc;
8888 switch (Op.getOpcode()) {
8889 default:
8890 llvm_unreachable("Unexpected opcode");
8891 case ISD::SHL:
8892 Opc = RISCVISD::PSHL;
8893 break;
8894 case ISD::SRL:
8895 Opc = RISCVISD::PSRL;
8896 break;
8897 case ISD::SRA:
8898 Opc = RISCVISD::PSRA;
8899 break;
8900 }
8901 return DAG.getNode(Opcode: Opc, DL: SDLoc(Op), VT: Op.getValueType(), N1: Op.getOperand(i: 0),
8902 N2: SplatVal);
8903 }
8904 return lowerToScalableOp(Op, DAG);
8905 }
8906 // This can be called for an i32 shift amount that needs to be promoted.
8907 assert(Op.getOperand(1).getValueType() == MVT::i32 && Subtarget.is64Bit() &&
8908 "Unexpected custom legalisation");
8909 return SDValue();
8910 case ISD::SSHLSAT: {
8911 MVT VT = Op.getSimpleValueType();
8912 assert(VT.isFixedLengthVector() && Subtarget.hasStdExtP() &&
8913 "Unexptect custom legalisation");
8914 APInt Splat;
8915 if (!ISD::isConstantSplatVector(N: Op.getOperand(i: 1).getNode(), SplatValue&: Splat))
8916 return SDValue();
8917 uint64_t ShAmt = Splat.getZExtValue();
8918 if (ShAmt >= VT.getVectorElementType().getSizeInBits())
8919 return SDValue();
8920 SDLoc DL(Op);
8921 return DAG.getNode(Opcode: RISCVISD::PSSLAI, DL, VT, N1: Op.getOperand(i: 0),
8922 N2: DAG.getTargetConstant(Val: ShAmt, DL, VT: Subtarget.getXLenVT()));
8923 }
8924 case ISD::FABS:
8925 case ISD::FNEG:
8926 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
8927 return lowerFABSorFNEG(Op, DAG, Subtarget);
8928 [[fallthrough]];
8929 case ISD::FADD:
8930 case ISD::FSUB:
8931 case ISD::FMUL:
8932 case ISD::FDIV:
8933 case ISD::FSQRT:
8934 case ISD::FMA:
8935 case ISD::FMINNUM:
8936 case ISD::FMAXNUM:
8937 case ISD::FMINIMUMNUM:
8938 case ISD::FMAXIMUMNUM:
8939 if (isPromotedOpNeedingSplit(Op, Subtarget))
8940 return SplitVectorOp(Op, DAG);
8941 [[fallthrough]];
8942 case ISD::AVGFLOORS:
8943 case ISD::AVGFLOORU:
8944 case ISD::AVGCEILS:
8945 case ISD::AVGCEILU:
8946 case ISD::SMIN:
8947 case ISD::SMAX:
8948 case ISD::UMIN:
8949 case ISD::UMAX:
8950 case ISD::UADDSAT:
8951 case ISD::USUBSAT:
8952 case ISD::SADDSAT:
8953 case ISD::SSUBSAT:
8954 return lowerToScalableOp(Op, DAG);
8955 case ISD::ABDS:
8956 case ISD::ABDU: {
8957 EVT VT = Op->getValueType(ResNo: 0);
8958 // Only SEW=8/16 are supported in Zvabd.
8959 if (Subtarget.hasStdExtZvabd() && VT.isVector() &&
8960 (VT.getVectorElementType() == MVT::i8 ||
8961 VT.getVectorElementType() == MVT::i16))
8962 return lowerToScalableOp(Op, DAG);
8963
8964 SDLoc dl(Op);
8965 SDValue LHS = DAG.getFreeze(V: Op->getOperand(Num: 0));
8966 SDValue RHS = DAG.getFreeze(V: Op->getOperand(Num: 1));
8967 bool IsSigned = Op->getOpcode() == ISD::ABDS;
8968
8969 // abds(lhs, rhs) -> sub(smax(lhs,rhs), smin(lhs,rhs))
8970 // abdu(lhs, rhs) -> sub(umax(lhs,rhs), umin(lhs,rhs))
8971 unsigned MaxOpc = IsSigned ? ISD::SMAX : ISD::UMAX;
8972 unsigned MinOpc = IsSigned ? ISD::SMIN : ISD::UMIN;
8973 SDValue Max = DAG.getNode(Opcode: MaxOpc, DL: dl, VT, N1: LHS, N2: RHS);
8974 SDValue Min = DAG.getNode(Opcode: MinOpc, DL: dl, VT, N1: LHS, N2: RHS);
8975 return DAG.getNode(Opcode: ISD::SUB, DL: dl, VT, N1: Max, N2: Min);
8976 }
8977 case ISD::ABS:
8978 case ISD::VP_ABS:
8979 return lowerABS(Op, DAG);
8980 case ISD::CTLZ:
8981 case ISD::CTLZ_ZERO_UNDEF:
8982 case ISD::CTTZ:
8983 case ISD::CTTZ_ZERO_UNDEF:
8984 if (Subtarget.hasStdExtZvbb())
8985 return lowerToScalableOp(Op, DAG);
8986 assert(Op.getOpcode() != ISD::CTTZ);
8987 return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
8988 case ISD::CLMUL: {
8989 MVT VT = Op.getSimpleValueType();
8990 assert(VT.isScalableVector() && Subtarget.hasStdExtZvbc() &&
8991 "Unexpected custom legalisation");
8992 // Promote to i64 vector.
8993 MVT I64VecVT = VT.changeVectorElementType(EltVT: MVT::i64);
8994 SDLoc DL(Op);
8995 SDValue Op0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: I64VecVT, Operand: Op.getOperand(i: 0));
8996 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: I64VecVT, Operand: Op.getOperand(i: 1));
8997 SDValue CLMUL = DAG.getNode(Opcode: ISD::CLMUL, DL, VT: I64VecVT, N1: Op0, N2: Op1);
8998 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: CLMUL);
8999 }
9000 case ISD::FCOPYSIGN:
9001 if (Op.getValueType() == MVT::f16 || Op.getValueType() == MVT::bf16)
9002 return lowerFCOPYSIGN(Op, DAG, Subtarget);
9003 if (isPromotedOpNeedingSplit(Op, Subtarget))
9004 return SplitVectorOp(Op, DAG);
9005 return lowerToScalableOp(Op, DAG);
9006 case ISD::STRICT_FADD:
9007 case ISD::STRICT_FSUB:
9008 case ISD::STRICT_FMUL:
9009 case ISD::STRICT_FDIV:
9010 case ISD::STRICT_FSQRT:
9011 case ISD::STRICT_FMA:
9012 if (isPromotedOpNeedingSplit(Op, Subtarget))
9013 return SplitStrictFPVectorOp(Op, DAG);
9014 return lowerToScalableOp(Op, DAG);
9015 case ISD::STRICT_FSETCC:
9016 case ISD::STRICT_FSETCCS:
9017 return lowerVectorStrictFSetcc(Op, DAG);
9018 case ISD::STRICT_FCEIL:
9019 case ISD::STRICT_FRINT:
9020 case ISD::STRICT_FFLOOR:
9021 case ISD::STRICT_FTRUNC:
9022 case ISD::STRICT_FNEARBYINT:
9023 case ISD::STRICT_FROUND:
9024 case ISD::STRICT_FROUNDEVEN:
9025 return lowerVectorStrictFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
9026 case ISD::MGATHER:
9027 case ISD::VP_GATHER:
9028 return lowerMaskedGather(Op, DAG);
9029 case ISD::MSCATTER:
9030 case ISD::VP_SCATTER:
9031 return lowerMaskedScatter(Op, DAG);
9032 case ISD::GET_ROUNDING:
9033 return lowerGET_ROUNDING(Op, DAG);
9034 case ISD::SET_ROUNDING:
9035 return lowerSET_ROUNDING(Op, DAG);
9036 case ISD::GET_FPENV:
9037 return lowerGET_FPENV(Op, DAG);
9038 case ISD::SET_FPENV:
9039 return lowerSET_FPENV(Op, DAG);
9040 case ISD::RESET_FPENV:
9041 return lowerRESET_FPENV(Op, DAG);
9042 case ISD::GET_FPMODE:
9043 return lowerGET_FPMODE(Op, DAG);
9044 case ISD::SET_FPMODE:
9045 return lowerSET_FPMODE(Op, DAG);
9046 case ISD::RESET_FPMODE:
9047 return lowerRESET_FPMODE(Op, DAG);
9048 case ISD::EH_DWARF_CFA:
9049 return lowerEH_DWARF_CFA(Op, DAG);
9050 case ISD::VP_MERGE:
9051 if (Op.getSimpleValueType().getVectorElementType() == MVT::i1)
9052 return lowerVPMergeMask(Op, DAG);
9053 [[fallthrough]];
9054 case ISD::VP_SELECT:
9055 case ISD::VP_ADD:
9056 case ISD::VP_SUB:
9057 case ISD::VP_MUL:
9058 case ISD::VP_SDIV:
9059 case ISD::VP_UDIV:
9060 case ISD::VP_SREM:
9061 case ISD::VP_UREM:
9062 case ISD::VP_UADDSAT:
9063 case ISD::VP_USUBSAT:
9064 case ISD::VP_SADDSAT:
9065 case ISD::VP_SSUBSAT:
9066 case ISD::VP_LRINT:
9067 case ISD::VP_LLRINT:
9068 return lowerVPOp(Op, DAG);
9069 case ISD::VP_AND:
9070 case ISD::VP_OR:
9071 case ISD::VP_XOR:
9072 return lowerLogicVPOp(Op, DAG);
9073 case ISD::VP_FADD:
9074 case ISD::VP_FSUB:
9075 case ISD::VP_FMUL:
9076 case ISD::VP_FDIV:
9077 case ISD::VP_FNEG:
9078 case ISD::VP_FABS:
9079 case ISD::VP_SQRT:
9080 case ISD::VP_FMA:
9081 case ISD::VP_FMINNUM:
9082 case ISD::VP_FMAXNUM:
9083 case ISD::VP_FCOPYSIGN:
9084 if (isPromotedOpNeedingSplit(Op, Subtarget))
9085 return SplitVPOp(Op, DAG);
9086 [[fallthrough]];
9087 case ISD::VP_SRA:
9088 case ISD::VP_SRL:
9089 case ISD::VP_SHL:
9090 return lowerVPOp(Op, DAG);
9091 case ISD::VP_IS_FPCLASS:
9092 return LowerIS_FPCLASS(Op, DAG);
9093 case ISD::VP_SIGN_EXTEND:
9094 case ISD::VP_ZERO_EXTEND:
9095 if (Op.getOperand(i: 0).getSimpleValueType().getVectorElementType() == MVT::i1)
9096 return lowerVPExtMaskOp(Op, DAG);
9097 return lowerVPOp(Op, DAG);
9098 case ISD::VP_TRUNCATE:
9099 return lowerVectorTruncLike(Op, DAG);
9100 case ISD::VP_FP_EXTEND:
9101 case ISD::VP_FP_ROUND:
9102 return lowerVectorFPExtendOrRoundLike(Op, DAG);
9103 case ISD::VP_SINT_TO_FP:
9104 case ISD::VP_UINT_TO_FP:
9105 if (Op.getValueType().isVector() &&
9106 ((Op.getValueType().getScalarType() == MVT::f16 &&
9107 (Subtarget.hasVInstructionsF16Minimal() &&
9108 !Subtarget.hasVInstructionsF16())) ||
9109 Op.getValueType().getScalarType() == MVT::bf16)) {
9110 if (isPromotedOpNeedingSplit(Op, Subtarget))
9111 return SplitVectorOp(Op, DAG);
9112 // int -> f32
9113 SDLoc DL(Op);
9114 MVT NVT =
9115 MVT::getVectorVT(VT: MVT::f32, EC: Op.getValueType().getVectorElementCount());
9116 auto NC = DAG.getNode(Opcode: Op.getOpcode(), DL, VT: NVT, Ops: Op->ops());
9117 // f32 -> [b]f16
9118 return DAG.getNode(Opcode: ISD::FP_ROUND, DL, VT: Op.getValueType(), N1: NC,
9119 N2: DAG.getIntPtrConstant(Val: 0, DL, /*isTarget=*/true));
9120 }
9121 [[fallthrough]];
9122 case ISD::VP_FP_TO_SINT:
9123 case ISD::VP_FP_TO_UINT:
9124 if (SDValue Op1 = Op.getOperand(i: 0);
9125 Op1.getValueType().isVector() &&
9126 ((Op1.getValueType().getScalarType() == MVT::f16 &&
9127 (Subtarget.hasVInstructionsF16Minimal() &&
9128 !Subtarget.hasVInstructionsF16())) ||
9129 Op1.getValueType().getScalarType() == MVT::bf16)) {
9130 if (isPromotedOpNeedingSplit(Op: Op1, Subtarget))
9131 return SplitVectorOp(Op, DAG);
9132 // [b]f16 -> f32
9133 SDLoc DL(Op);
9134 MVT NVT = MVT::getVectorVT(VT: MVT::f32,
9135 EC: Op1.getValueType().getVectorElementCount());
9136 SDValue WidenVec = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: NVT, Operand: Op1);
9137 // f32 -> int
9138 return DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
9139 Ops: {WidenVec, Op.getOperand(i: 1), Op.getOperand(i: 2)});
9140 }
9141 return lowerVPFPIntConvOp(Op, DAG);
9142 case ISD::VP_SETCC:
9143 if (isPromotedOpNeedingSplit(Op: Op.getOperand(i: 0), Subtarget))
9144 return SplitVPOp(Op, DAG);
9145 if (Op.getOperand(i: 0).getSimpleValueType().getVectorElementType() == MVT::i1)
9146 return lowerVPSetCCMaskOp(Op, DAG);
9147 [[fallthrough]];
9148 case ISD::VP_SMIN:
9149 case ISD::VP_SMAX:
9150 case ISD::VP_UMIN:
9151 case ISD::VP_UMAX:
9152 case ISD::VP_BITREVERSE:
9153 case ISD::VP_BSWAP:
9154 return lowerVPOp(Op, DAG);
9155 case ISD::VP_CTLZ:
9156 case ISD::VP_CTLZ_ZERO_UNDEF:
9157 if (Subtarget.hasStdExtZvbb())
9158 return lowerVPOp(Op, DAG);
9159 return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
9160 case ISD::VP_CTTZ:
9161 case ISD::VP_CTTZ_ZERO_UNDEF:
9162 if (Subtarget.hasStdExtZvbb())
9163 return lowerVPOp(Op, DAG);
9164 return lowerCTLZ_CTTZ_ZERO_UNDEF(Op, DAG);
9165 case ISD::VP_CTPOP:
9166 return lowerVPOp(Op, DAG);
9167 case ISD::EXPERIMENTAL_VP_STRIDED_LOAD:
9168 return lowerVPStridedLoad(Op, DAG);
9169 case ISD::EXPERIMENTAL_VP_STRIDED_STORE:
9170 return lowerVPStridedStore(Op, DAG);
9171 case ISD::VP_FCEIL:
9172 case ISD::VP_FFLOOR:
9173 case ISD::VP_FRINT:
9174 case ISD::VP_FNEARBYINT:
9175 case ISD::VP_FROUND:
9176 case ISD::VP_FROUNDEVEN:
9177 case ISD::VP_FROUNDTOZERO:
9178 if (isPromotedOpNeedingSplit(Op, Subtarget))
9179 return SplitVPOp(Op, DAG);
9180 return lowerVectorFTRUNC_FCEIL_FFLOOR_FROUND(Op, DAG, Subtarget);
9181 case ISD::VP_FMAXIMUM:
9182 case ISD::VP_FMINIMUM:
9183 if (isPromotedOpNeedingSplit(Op, Subtarget))
9184 return SplitVPOp(Op, DAG);
9185 return lowerFMAXIMUM_FMINIMUM(Op, DAG, Subtarget);
9186 case ISD::EXPERIMENTAL_VP_SPLICE:
9187 return lowerVPSpliceExperimental(Op, DAG);
9188 case ISD::EXPERIMENTAL_VP_REVERSE:
9189 return lowerVPReverseExperimental(Op, DAG);
9190 case ISD::CLEAR_CACHE: {
9191 assert(getTargetMachine().getTargetTriple().isOSLinux() &&
9192 "llvm.clear_cache only needs custom lower on Linux targets");
9193 SDLoc DL(Op);
9194 SDValue Flags = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
9195 return emitFlushICache(DAG, InChain: Op.getOperand(i: 0), Start: Op.getOperand(i: 1),
9196 End: Op.getOperand(i: 2), Flags, DL);
9197 }
9198 case ISD::DYNAMIC_STACKALLOC:
9199 return lowerDYNAMIC_STACKALLOC(Op, DAG);
9200 case ISD::INIT_TRAMPOLINE:
9201 return lowerINIT_TRAMPOLINE(Op, DAG);
9202 case ISD::ADJUST_TRAMPOLINE:
9203 return lowerADJUST_TRAMPOLINE(Op, DAG);
9204 case ISD::PARTIAL_REDUCE_UMLA:
9205 case ISD::PARTIAL_REDUCE_SMLA:
9206 case ISD::PARTIAL_REDUCE_SUMLA:
9207 return lowerPARTIAL_REDUCE_MLA(Op, DAG);
9208 case ISD::CTTZ_ELTS:
9209 case ISD::CTTZ_ELTS_ZERO_POISON:
9210 return lowerCttzElts(Op, DAG, Subtarget);
9211 }
9212}
9213
9214SDValue RISCVTargetLowering::emitFlushICache(SelectionDAG &DAG, SDValue InChain,
9215 SDValue Start, SDValue End,
9216 SDValue Flags, SDLoc DL) const {
9217 MakeLibCallOptions CallOptions;
9218 std::pair<SDValue, SDValue> CallResult =
9219 makeLibCall(DAG, LC: RTLIB::RISCV_FLUSH_ICACHE, RetVT: MVT::isVoid,
9220 Ops: {Start, End, Flags}, CallOptions, dl: DL, Chain: InChain);
9221
9222 // This function returns void so only the out chain matters.
9223 return CallResult.second;
9224}
9225
9226SDValue RISCVTargetLowering::lowerINIT_TRAMPOLINE(SDValue Op,
9227 SelectionDAG &DAG) const {
9228 if (!Subtarget.is64Bit())
9229 llvm::reportFatalUsageError(reason: "Trampolines only implemented for RV64");
9230
9231 // Create an MCCodeEmitter to encode instructions.
9232 TargetLoweringObjectFile *TLO = getTargetMachine().getObjFileLowering();
9233 assert(TLO);
9234 MCContext &MCCtx = TLO->getContext();
9235
9236 std::unique_ptr<MCCodeEmitter> CodeEmitter(
9237 createRISCVMCCodeEmitter(MCII: *getTargetMachine().getMCInstrInfo(), Ctx&: MCCtx));
9238
9239 SDValue Root = Op.getOperand(i: 0);
9240 SDValue Trmp = Op.getOperand(i: 1); // trampoline
9241 SDLoc dl(Op);
9242
9243 const Value *TrmpAddr = cast<SrcValueSDNode>(Val: Op.getOperand(i: 4))->getValue();
9244
9245 // We store in the trampoline buffer the following instructions and data.
9246 // Offset:
9247 // 0: auipc t2, 0
9248 // 4: ld t0, 24(t2)
9249 // 8: ld t2, 16(t2)
9250 // 12: jalr t0
9251 // 16: <StaticChainOffset>
9252 // 24: <FunctionAddressOffset>
9253 // 32:
9254 // Offset with branch control flow protection enabled:
9255 // 0: lpad <imm20>
9256 // 4: auipc t3, 0
9257 // 8: ld t2, 28(t3)
9258 // 12: ld t3, 20(t3)
9259 // 16: jalr t2
9260 // 20: <StaticChainOffset>
9261 // 28: <FunctionAddressOffset>
9262 // 36:
9263
9264 const bool HasCFBranch =
9265 Subtarget.hasStdExtZicfilp() &&
9266 DAG.getMachineFunction().getFunction().getParent()->getModuleFlag(
9267 Key: "cf-protection-branch");
9268 const unsigned StaticChainIdx = HasCFBranch ? 5 : 4;
9269 const unsigned StaticChainOffset = StaticChainIdx * 4;
9270 const unsigned FunctionAddressOffset = StaticChainOffset + 8;
9271
9272 const MCSubtargetInfo *STI = getTargetMachine().getMCSubtargetInfo();
9273 assert(STI);
9274 auto GetEncoding = [&](const MCInst &MC) {
9275 SmallVector<char, 4> CB;
9276 SmallVector<MCFixup> Fixups;
9277 CodeEmitter->encodeInstruction(Inst: MC, CB, Fixups, STI: *STI);
9278 uint32_t Encoding = support::endian::read32le(P: CB.data());
9279 return Encoding;
9280 };
9281
9282 SmallVector<SDValue> OutChains;
9283
9284 SmallVector<uint32_t> Encodings;
9285 if (!HasCFBranch) {
9286 Encodings.append(
9287 IL: {// auipc t2, 0
9288 // Loads the current PC into t2.
9289 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X7).addImm(Val: 0)),
9290 // ld t0, 24(t2)
9291 // Loads the function address into t0. Note that we are using offsets
9292 // pc-relative to the first instruction of the trampoline.
9293 GetEncoding(MCInstBuilder(RISCV::LD)
9294 .addReg(Reg: RISCV::X5)
9295 .addReg(Reg: RISCV::X7)
9296 .addImm(Val: FunctionAddressOffset)),
9297 // ld t2, 16(t2)
9298 // Load the value of the static chain.
9299 GetEncoding(MCInstBuilder(RISCV::LD)
9300 .addReg(Reg: RISCV::X7)
9301 .addReg(Reg: RISCV::X7)
9302 .addImm(Val: StaticChainOffset)),
9303 // jalr t0
9304 // Jump to the function.
9305 GetEncoding(MCInstBuilder(RISCV::JALR)
9306 .addReg(Reg: RISCV::X0)
9307 .addReg(Reg: RISCV::X5)
9308 .addImm(Val: 0))});
9309 } else {
9310 Encodings.append(
9311 IL: {// auipc x0, <imm20> (lpad <imm20>)
9312 // Landing pad.
9313 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X0).addImm(Val: 0)),
9314 // auipc t3, 0
9315 // Loads the current PC into t3.
9316 GetEncoding(MCInstBuilder(RISCV::AUIPC).addReg(Reg: RISCV::X28).addImm(Val: 0)),
9317 // ld t2, (FunctionAddressOffset - 4)(t3)
9318 // Loads the function address into t2. Note that we are using offsets
9319 // pc-relative to the SECOND instruction of the trampoline.
9320 GetEncoding(MCInstBuilder(RISCV::LD)
9321 .addReg(Reg: RISCV::X7)
9322 .addReg(Reg: RISCV::X28)
9323 .addImm(Val: FunctionAddressOffset - 4)),
9324 // ld t3, (StaticChainOffset - 4)(t3)
9325 // Load the value of the static chain.
9326 GetEncoding(MCInstBuilder(RISCV::LD)
9327 .addReg(Reg: RISCV::X28)
9328 .addReg(Reg: RISCV::X28)
9329 .addImm(Val: StaticChainOffset - 4)),
9330 // jalr t2
9331 // Software-guarded jump to the function.
9332 GetEncoding(MCInstBuilder(RISCV::JALR)
9333 .addReg(Reg: RISCV::X0)
9334 .addReg(Reg: RISCV::X7)
9335 .addImm(Val: 0))});
9336 }
9337
9338 // Store encoded instructions.
9339 for (auto [Idx, Encoding] : llvm::enumerate(First&: Encodings)) {
9340 SDValue Addr = Idx > 0 ? DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Trmp,
9341 N2: DAG.getConstant(Val: Idx * 4, DL: dl, VT: MVT::i64))
9342 : Trmp;
9343 OutChains.push_back(Elt: DAG.getTruncStore(
9344 Chain: Root, dl, Val: DAG.getConstant(Val: Encoding, DL: dl, VT: MVT::i64), Ptr: Addr,
9345 PtrInfo: MachinePointerInfo(TrmpAddr, Idx * 4), SVT: MVT::i32));
9346 }
9347
9348 // Now store the variable part of the trampoline.
9349 SDValue FunctionAddress = Op.getOperand(i: 2);
9350 SDValue StaticChain = Op.getOperand(i: 3);
9351
9352 // Store the given static chain and function pointer in the trampoline buffer.
9353 struct OffsetValuePair {
9354 const unsigned Offset;
9355 const SDValue Value;
9356 SDValue Addr = SDValue(); // Used to cache the address.
9357 } OffsetValues[] = {
9358 {.Offset: StaticChainOffset, .Value: StaticChain},
9359 {.Offset: FunctionAddressOffset, .Value: FunctionAddress},
9360 };
9361 for (auto &OffsetValue : OffsetValues) {
9362 SDValue Addr =
9363 DAG.getNode(Opcode: ISD::ADD, DL: dl, VT: MVT::i64, N1: Trmp,
9364 N2: DAG.getConstant(Val: OffsetValue.Offset, DL: dl, VT: MVT::i64));
9365 OffsetValue.Addr = Addr;
9366 OutChains.push_back(
9367 Elt: DAG.getStore(Chain: Root, dl, Val: OffsetValue.Value, Ptr: Addr,
9368 PtrInfo: MachinePointerInfo(TrmpAddr, OffsetValue.Offset)));
9369 }
9370
9371 assert(OutChains.size() == StaticChainIdx + 2 &&
9372 "Size of OutChains mismatch");
9373 SDValue StoreToken = DAG.getNode(Opcode: ISD::TokenFactor, DL: dl, VT: MVT::Other, Ops: OutChains);
9374
9375 // The end of instructions of trampoline is the same as the static chain
9376 // address that we computed earlier.
9377 SDValue EndOfTrmp = OffsetValues[0].Addr;
9378
9379 // Call clear cache on the trampoline instructions.
9380 SDValue Chain = DAG.getNode(Opcode: ISD::CLEAR_CACHE, DL: dl, VT: MVT::Other, N1: StoreToken,
9381 N2: Trmp, N3: EndOfTrmp);
9382
9383 return Chain;
9384}
9385
9386SDValue RISCVTargetLowering::lowerADJUST_TRAMPOLINE(SDValue Op,
9387 SelectionDAG &DAG) const {
9388 if (!Subtarget.is64Bit())
9389 llvm::reportFatalUsageError(reason: "Trampolines only implemented for RV64");
9390
9391 return Op.getOperand(i: 0);
9392}
9393
9394SDValue RISCVTargetLowering::lowerPARTIAL_REDUCE_MLA(SDValue Op,
9395 SelectionDAG &DAG) const {
9396 // Currently, only the vdota4 and vdota4u case (from zvdot4a8i) should be
9397 // legal.
9398 // TODO: There are many other sub-cases we could potentially lower, are
9399 // any of them worthwhile? Ex: via vredsum, vwredsum, vwwmaccu, etc..
9400 SDLoc DL(Op);
9401 MVT VT = Op.getSimpleValueType();
9402 SDValue Accum = Op.getOperand(i: 0);
9403 assert(Accum.getSimpleValueType() == VT &&
9404 VT.getVectorElementType() == MVT::i32);
9405 SDValue A = Op.getOperand(i: 1);
9406 SDValue B = Op.getOperand(i: 2);
9407 MVT ArgVT = A.getSimpleValueType();
9408 assert(ArgVT == B.getSimpleValueType() &&
9409 ArgVT.getVectorElementType() == MVT::i8);
9410 (void)ArgVT;
9411
9412 // The zvdot4a8i pseudos are defined with sources and destination both
9413 // being i32. This cast is needed for correctness to avoid incorrect
9414 // .vx matching of i8 splats.
9415 A = DAG.getBitcast(VT, V: A);
9416 B = DAG.getBitcast(VT, V: B);
9417
9418 MVT ContainerVT = VT;
9419 if (VT.isFixedLengthVector()) {
9420 ContainerVT = getContainerForFixedLengthVector(VT);
9421 Accum = convertToScalableVector(VT: ContainerVT, V: Accum, DAG, Subtarget);
9422 A = convertToScalableVector(VT: ContainerVT, V: A, DAG, Subtarget);
9423 B = convertToScalableVector(VT: ContainerVT, V: B, DAG, Subtarget);
9424 }
9425
9426 unsigned Opc;
9427 switch (Op.getOpcode()) {
9428 case ISD::PARTIAL_REDUCE_SMLA:
9429 Opc = RISCVISD::VDOTA4_VL;
9430 break;
9431 case ISD::PARTIAL_REDUCE_UMLA:
9432 Opc = RISCVISD::VDOTA4U_VL;
9433 break;
9434 case ISD::PARTIAL_REDUCE_SUMLA:
9435 Opc = RISCVISD::VDOTA4SU_VL;
9436 break;
9437 default:
9438 llvm_unreachable("Unexpected opcode");
9439 }
9440 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
9441 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, Ops: {A, B, Accum, Mask, VL});
9442 if (VT.isFixedLengthVector())
9443 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
9444 return Res;
9445}
9446
9447static SDValue getTargetNode(GlobalAddressSDNode *N, const SDLoc &DL, EVT Ty,
9448 SelectionDAG &DAG, unsigned Flags) {
9449 return DAG.getTargetGlobalAddress(GV: N->getGlobal(), DL, VT: Ty, offset: 0, TargetFlags: Flags);
9450}
9451
9452static SDValue getTargetNode(BlockAddressSDNode *N, const SDLoc &DL, EVT Ty,
9453 SelectionDAG &DAG, unsigned Flags) {
9454 return DAG.getTargetBlockAddress(BA: N->getBlockAddress(), VT: Ty, Offset: N->getOffset(),
9455 TargetFlags: Flags);
9456}
9457
9458static SDValue getTargetNode(ConstantPoolSDNode *N, const SDLoc &DL, EVT Ty,
9459 SelectionDAG &DAG, unsigned Flags) {
9460 return DAG.getTargetConstantPool(C: N->getConstVal(), VT: Ty, Align: N->getAlign(),
9461 Offset: N->getOffset(), TargetFlags: Flags);
9462}
9463
9464static SDValue getTargetNode(JumpTableSDNode *N, const SDLoc &DL, EVT Ty,
9465 SelectionDAG &DAG, unsigned Flags) {
9466 return DAG.getTargetJumpTable(JTI: N->getIndex(), VT: Ty, TargetFlags: Flags);
9467}
9468
9469static SDValue getLargeGlobalAddress(GlobalAddressSDNode *N, const SDLoc &DL,
9470 EVT Ty, SelectionDAG &DAG) {
9471 RISCVConstantPoolValue *CPV = RISCVConstantPoolValue::Create(GV: N->getGlobal());
9472 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: Ty, Align: Align(8));
9473 SDValue LC = DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: CPAddr);
9474 return DAG.getLoad(
9475 VT: Ty, dl: DL, Chain: DAG.getEntryNode(), Ptr: LC,
9476 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
9477}
9478
9479static SDValue getLargeExternalSymbol(ExternalSymbolSDNode *N, const SDLoc &DL,
9480 EVT Ty, SelectionDAG &DAG) {
9481 RISCVConstantPoolValue *CPV =
9482 RISCVConstantPoolValue::Create(C&: *DAG.getContext(), S: N->getSymbol());
9483 SDValue CPAddr = DAG.getTargetConstantPool(C: CPV, VT: Ty, Align: Align(8));
9484 SDValue LC = DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: CPAddr);
9485 return DAG.getLoad(
9486 VT: Ty, dl: DL, Chain: DAG.getEntryNode(), Ptr: LC,
9487 PtrInfo: MachinePointerInfo::getConstantPool(MF&: DAG.getMachineFunction()));
9488}
9489
9490template <class NodeTy>
9491SDValue RISCVTargetLowering::getAddr(NodeTy *N, SelectionDAG &DAG,
9492 bool IsLocal, bool IsExternWeak) const {
9493 SDLoc DL(N);
9494 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
9495
9496 // When HWASAN is used and tagging of global variables is enabled
9497 // they should be accessed via the GOT, since the tagged address of a global
9498 // is incompatible with existing code models. This also applies to non-pic
9499 // mode.
9500 if (isPositionIndependent() || Subtarget.allowTaggedGlobals()) {
9501 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9502 if (IsLocal && !Subtarget.allowTaggedGlobals())
9503 // Use PC-relative addressing to access the symbol. This generates the
9504 // pattern (PseudoLLA sym), which expands to (addi (auipc %pcrel_hi(sym))
9505 // %pcrel_lo(auipc)).
9506 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
9507
9508 // Use PC-relative addressing to access the GOT for this symbol, then load
9509 // the address from the GOT. This generates the pattern (PseudoLGA sym),
9510 // which expands to (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
9511 SDValue Load =
9512 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLGA, dl: DL, VT: Ty, Op1: Addr), 0);
9513 MachineFunction &MF = DAG.getMachineFunction();
9514 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9515 PtrInfo: MachinePointerInfo::getGOT(MF),
9516 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9517 MachineMemOperand::MOInvariant,
9518 MemTy: LLT(Ty.getSimpleVT()), base_alignment: Align(Ty.getFixedSizeInBits() / 8));
9519 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
9520 return Load;
9521 }
9522
9523 switch (getTargetMachine().getCodeModel()) {
9524 default:
9525 reportFatalUsageError(reason: "Unsupported code model for lowering");
9526 case CodeModel::Small: {
9527 // Generate a sequence for accessing addresses within the first 2 GiB of
9528 // address space.
9529 if (Subtarget.hasVendorXqcili()) {
9530 // Use QC.E.LI to generate the address, as this is easier to relax than
9531 // LUI/ADDI.
9532 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9533 return DAG.getNode(Opcode: RISCVISD::QC_E_LI, DL, VT: Ty, Operand: Addr);
9534 }
9535
9536 // This generates the pattern (addi (lui %hi(sym)) %lo(sym)).
9537 SDValue AddrHi = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_HI);
9538 SDValue AddrLo = getTargetNode(N, DL, Ty, DAG, RISCVII::MO_LO);
9539 SDValue MNHi = DAG.getNode(Opcode: RISCVISD::HI, DL, VT: Ty, Operand: AddrHi);
9540 return DAG.getNode(Opcode: RISCVISD::ADD_LO, DL, VT: Ty, N1: MNHi, N2: AddrLo);
9541 }
9542 case CodeModel::Medium: {
9543 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9544 if (IsExternWeak) {
9545 // An extern weak symbol may be undefined, i.e. have value 0, which may
9546 // not be within 2GiB of PC, so use GOT-indirect addressing to access the
9547 // symbol. This generates the pattern (PseudoLGA sym), which expands to
9548 // (ld (addi (auipc %got_pcrel_hi(sym)) %pcrel_lo(auipc))).
9549 SDValue Load =
9550 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLGA, dl: DL, VT: Ty, Op1: Addr), 0);
9551 MachineFunction &MF = DAG.getMachineFunction();
9552 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9553 PtrInfo: MachinePointerInfo::getGOT(MF),
9554 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9555 MachineMemOperand::MOInvariant,
9556 MemTy: LLT(Ty.getSimpleVT()), base_alignment: Align(Ty.getFixedSizeInBits() / 8));
9557 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
9558 return Load;
9559 }
9560
9561 // Generate a sequence for accessing addresses within any 2GiB range within
9562 // the address space. This generates the pattern (PseudoLLA sym), which
9563 // expands to (addi (auipc %pcrel_hi(sym)) %pcrel_lo(auipc)).
9564 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
9565 }
9566 case CodeModel::Large: {
9567 if (GlobalAddressSDNode *G = dyn_cast<GlobalAddressSDNode>(N))
9568 return getLargeGlobalAddress(N: G, DL, Ty, DAG);
9569
9570 // Using pc-relative mode for other node type.
9571 SDValue Addr = getTargetNode(N, DL, Ty, DAG, 0);
9572 return DAG.getNode(Opcode: RISCVISD::LLA, DL, VT: Ty, Operand: Addr);
9573 }
9574 }
9575}
9576
9577SDValue RISCVTargetLowering::lowerGlobalAddress(SDValue Op,
9578 SelectionDAG &DAG) const {
9579 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
9580 assert(N->getOffset() == 0 && "unexpected offset in global node");
9581 const GlobalValue *GV = N->getGlobal();
9582 bool IsLocal = getTargetMachine().shouldAssumeDSOLocal(GV);
9583 return getAddr(N, DAG, IsLocal, IsExternWeak: GV->hasExternalWeakLinkage());
9584}
9585
9586SDValue RISCVTargetLowering::lowerBlockAddress(SDValue Op,
9587 SelectionDAG &DAG) const {
9588 BlockAddressSDNode *N = cast<BlockAddressSDNode>(Val&: Op);
9589
9590 return getAddr(N, DAG);
9591}
9592
9593SDValue RISCVTargetLowering::lowerConstantPool(SDValue Op,
9594 SelectionDAG &DAG) const {
9595 ConstantPoolSDNode *N = cast<ConstantPoolSDNode>(Val&: Op);
9596
9597 return getAddr(N, DAG);
9598}
9599
9600SDValue RISCVTargetLowering::lowerJumpTable(SDValue Op,
9601 SelectionDAG &DAG) const {
9602 JumpTableSDNode *N = cast<JumpTableSDNode>(Val&: Op);
9603
9604 return getAddr(N, DAG);
9605}
9606
9607SDValue RISCVTargetLowering::getStaticTLSAddr(GlobalAddressSDNode *N,
9608 SelectionDAG &DAG,
9609 bool UseGOT) const {
9610 SDLoc DL(N);
9611 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
9612 const GlobalValue *GV = N->getGlobal();
9613 MVT XLenVT = Subtarget.getXLenVT();
9614
9615 if (UseGOT) {
9616 // Use PC-relative addressing to access the GOT for this TLS symbol, then
9617 // load the address from the GOT and add the thread pointer. This generates
9618 // the pattern (PseudoLA_TLS_IE sym), which expands to
9619 // (ld (auipc %tls_ie_pcrel_hi(sym)) %pcrel_lo(auipc)).
9620 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
9621 SDValue Load =
9622 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLS_IE, dl: DL, VT: Ty, Op1: Addr), 0);
9623 MachineFunction &MF = DAG.getMachineFunction();
9624 MachineMemOperand *MemOp = MF.getMachineMemOperand(
9625 PtrInfo: MachinePointerInfo::getGOT(MF),
9626 f: MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable |
9627 MachineMemOperand::MOInvariant,
9628 MemTy: LLT(Ty.getSimpleVT()), base_alignment: Align(Ty.getFixedSizeInBits() / 8));
9629 DAG.setNodeMemRefs(N: cast<MachineSDNode>(Val: Load.getNode()), NewMemRefs: {MemOp});
9630
9631 // Add the thread pointer.
9632 SDValue TPReg = DAG.getRegister(Reg: RISCV::X4, VT: XLenVT);
9633 return DAG.getNode(Opcode: ISD::ADD, DL, VT: Ty, N1: Load, N2: TPReg);
9634 }
9635
9636 // Generate a sequence for accessing the address relative to the thread
9637 // pointer, with the appropriate adjustment for the thread pointer offset.
9638 // This generates the pattern
9639 // (add (add_tprel (lui %tprel_hi(sym)) tp %tprel_add(sym)) %tprel_lo(sym))
9640 SDValue AddrHi =
9641 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_HI);
9642 SDValue AddrAdd =
9643 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_ADD);
9644 SDValue AddrLo =
9645 DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: RISCVII::MO_TPREL_LO);
9646
9647 SDValue MNHi = DAG.getNode(Opcode: RISCVISD::HI, DL, VT: Ty, Operand: AddrHi);
9648 SDValue TPReg = DAG.getRegister(Reg: RISCV::X4, VT: XLenVT);
9649 SDValue MNAdd =
9650 DAG.getNode(Opcode: RISCVISD::ADD_TPREL, DL, VT: Ty, N1: MNHi, N2: TPReg, N3: AddrAdd);
9651 return DAG.getNode(Opcode: RISCVISD::ADD_LO, DL, VT: Ty, N1: MNAdd, N2: AddrLo);
9652}
9653
9654SDValue RISCVTargetLowering::getDynamicTLSAddr(GlobalAddressSDNode *N,
9655 SelectionDAG &DAG) const {
9656 SDLoc DL(N);
9657 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
9658 IntegerType *CallTy = Type::getIntNTy(C&: *DAG.getContext(), N: Ty.getSizeInBits());
9659 const GlobalValue *GV = N->getGlobal();
9660
9661 // Use a PC-relative addressing mode to access the global dynamic GOT address.
9662 // This generates the pattern (PseudoLA_TLS_GD sym), which expands to
9663 // (addi (auipc %tls_gd_pcrel_hi(sym)) %pcrel_lo(auipc)).
9664 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
9665 SDValue Load =
9666 SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLS_GD, dl: DL, VT: Ty, Op1: Addr), 0);
9667
9668 // Prepare argument list to generate call.
9669 ArgListTy Args;
9670 Args.emplace_back(args&: Load, args&: CallTy);
9671
9672 // Setup call to __tls_get_addr.
9673 TargetLowering::CallLoweringInfo CLI(DAG);
9674 CLI.setDebugLoc(DL)
9675 .setChain(DAG.getEntryNode())
9676 .setLibCallee(CC: CallingConv::C, ResultType: CallTy,
9677 Target: DAG.getExternalSymbol(Sym: "__tls_get_addr", VT: Ty),
9678 ArgsList: std::move(Args));
9679
9680 return LowerCallTo(CLI).first;
9681}
9682
9683SDValue RISCVTargetLowering::getTLSDescAddr(GlobalAddressSDNode *N,
9684 SelectionDAG &DAG) const {
9685 SDLoc DL(N);
9686 EVT Ty = getPointerTy(DL: DAG.getDataLayout());
9687 const GlobalValue *GV = N->getGlobal();
9688
9689 // Use a PC-relative addressing mode to access the global dynamic GOT address.
9690 // This generates the pattern (PseudoLA_TLSDESC sym), which expands to
9691 //
9692 // auipc tX, %tlsdesc_hi(symbol) // R_RISCV_TLSDESC_HI20(symbol)
9693 // lw tY, tX, %tlsdesc_load_lo(label) // R_RISCV_TLSDESC_LOAD_LO12(label)
9694 // addi a0, tX, %tlsdesc_add_lo(label) // R_RISCV_TLSDESC_ADD_LO12(label)
9695 // jalr t0, tY // R_RISCV_TLSDESC_CALL(label)
9696 SDValue Addr = DAG.getTargetGlobalAddress(GV, DL, VT: Ty, offset: 0, TargetFlags: 0);
9697 return SDValue(DAG.getMachineNode(Opcode: RISCV::PseudoLA_TLSDESC, dl: DL, VT: Ty, Op1: Addr), 0);
9698}
9699
9700SDValue RISCVTargetLowering::lowerGlobalTLSAddress(SDValue Op,
9701 SelectionDAG &DAG) const {
9702 GlobalAddressSDNode *N = cast<GlobalAddressSDNode>(Val&: Op);
9703 assert(N->getOffset() == 0 && "unexpected offset in global node");
9704
9705 if (DAG.getTarget().useEmulatedTLS())
9706 return LowerToTLSEmulatedModel(GA: N, DAG);
9707
9708 TLSModel::Model Model = getTargetMachine().getTLSModel(GV: N->getGlobal());
9709
9710 if (DAG.getMachineFunction().getFunction().getCallingConv() ==
9711 CallingConv::GHC)
9712 reportFatalUsageError(reason: "In GHC calling convention TLS is not supported");
9713
9714 SDValue Addr;
9715 switch (Model) {
9716 case TLSModel::LocalExec:
9717 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/false);
9718 break;
9719 case TLSModel::InitialExec:
9720 Addr = getStaticTLSAddr(N, DAG, /*UseGOT=*/true);
9721 break;
9722 case TLSModel::LocalDynamic:
9723 case TLSModel::GeneralDynamic:
9724 Addr = DAG.getTarget().useTLSDESC() ? getTLSDescAddr(N, DAG)
9725 : getDynamicTLSAddr(N, DAG);
9726 break;
9727 }
9728
9729 return Addr;
9730}
9731
9732// Return true if Val is equal to (setcc LHS, RHS, CC).
9733// Return false if Val is the inverse of (setcc LHS, RHS, CC).
9734// Otherwise, return std::nullopt.
9735static std::optional<bool> matchSetCC(SDValue LHS, SDValue RHS,
9736 ISD::CondCode CC, SDValue Val) {
9737 assert(Val->getOpcode() == ISD::SETCC);
9738 SDValue LHS2 = Val.getOperand(i: 0);
9739 SDValue RHS2 = Val.getOperand(i: 1);
9740 ISD::CondCode CC2 = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
9741
9742 if (LHS == LHS2 && RHS == RHS2) {
9743 if (CC == CC2)
9744 return true;
9745 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
9746 return false;
9747 } else if (LHS == RHS2 && RHS == LHS2) {
9748 CC2 = ISD::getSetCCSwappedOperands(Operation: CC2);
9749 if (CC == CC2)
9750 return true;
9751 if (CC == ISD::getSetCCInverse(Operation: CC2, Type: LHS2.getValueType()))
9752 return false;
9753 }
9754
9755 return std::nullopt;
9756}
9757
9758static bool isSimm12Constant(SDValue V) {
9759 return isa<ConstantSDNode>(Val: V) && V->getAsAPIntVal().isSignedIntN(N: 12);
9760}
9761
9762static SDValue lowerSelectToBinOp(SDNode *N, SelectionDAG &DAG,
9763 const RISCVSubtarget &Subtarget) {
9764 SDValue CondV = N->getOperand(Num: 0);
9765 SDValue TrueV = N->getOperand(Num: 1);
9766 SDValue FalseV = N->getOperand(Num: 2);
9767 MVT VT = N->getSimpleValueType(ResNo: 0);
9768 SDLoc DL(N);
9769
9770 if (!Subtarget.hasConditionalMoveFusion()) {
9771 // (select c, -1, y) -> -c | y
9772 if (isAllOnesConstant(V: TrueV)) {
9773 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
9774 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
9775 }
9776 // (select c, y, -1) -> (c-1) | y
9777 if (isAllOnesConstant(V: FalseV)) {
9778 SDValue Neg = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV,
9779 N2: DAG.getAllOnesConstant(DL, VT));
9780 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
9781 }
9782
9783 const bool HasCZero = VT.isScalarInteger() && Subtarget.hasCZEROLike();
9784
9785 // (select c, 0, y) -> (c-1) & y
9786 if (isNullConstant(V: TrueV) && (!HasCZero || isSimm12Constant(V: FalseV))) {
9787 SDValue Neg =
9788 DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: DAG.getAllOnesConstant(DL, VT));
9789 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: FalseV));
9790 }
9791 if (isNullConstant(V: FalseV)) {
9792 // (select c, (1 << ShAmount) + 1, 0) -> (c << ShAmount) + c
9793 if (auto *TrueC = dyn_cast<ConstantSDNode>(Val&: TrueV)) {
9794 uint64_t TrueM1 = TrueC->getZExtValue() - 1;
9795 if (isPowerOf2_64(Value: TrueM1)) {
9796 unsigned ShAmount = Log2_64(Value: TrueM1);
9797 if (Subtarget.hasShlAdd(ShAmt: ShAmount))
9798 return DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: CondV,
9799 N2: DAG.getTargetConstant(Val: ShAmount, DL, VT), N3: CondV);
9800 }
9801 }
9802 // (select c, y, 0) -> -c & y
9803 if (!HasCZero || isSimm12Constant(V: TrueV)) {
9804 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
9805 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: DAG.getFreeze(V: TrueV));
9806 }
9807 }
9808 }
9809
9810 // select c, ~x, x --> xor -c, x
9811 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
9812 const APInt &TrueVal = TrueV->getAsAPIntVal();
9813 const APInt &FalseVal = FalseV->getAsAPIntVal();
9814 if (~TrueVal == FalseVal) {
9815 SDValue Neg = DAG.getNegative(Val: CondV, DL, VT);
9816 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Neg, N2: FalseV);
9817 }
9818 }
9819
9820 // Try to fold (select (setcc lhs, rhs, cc), truev, falsev) into bitwise ops
9821 // when both truev and falsev are also setcc.
9822 if (CondV.getOpcode() == ISD::SETCC && TrueV.getOpcode() == ISD::SETCC &&
9823 FalseV.getOpcode() == ISD::SETCC) {
9824 SDValue LHS = CondV.getOperand(i: 0);
9825 SDValue RHS = CondV.getOperand(i: 1);
9826 ISD::CondCode CC = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
9827
9828 // (select x, x, y) -> x | y
9829 // (select !x, x, y) -> x & y
9830 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: TrueV)) {
9831 return DAG.getNode(Opcode: *MatchResult ? ISD::OR : ISD::AND, DL, VT, N1: TrueV,
9832 N2: DAG.getFreeze(V: FalseV));
9833 }
9834 // (select x, y, x) -> x & y
9835 // (select !x, y, x) -> x | y
9836 if (std::optional<bool> MatchResult = matchSetCC(LHS, RHS, CC, Val: FalseV)) {
9837 return DAG.getNode(Opcode: *MatchResult ? ISD::AND : ISD::OR, DL, VT,
9838 N1: DAG.getFreeze(V: TrueV), N2: FalseV);
9839 }
9840 }
9841
9842 return SDValue();
9843}
9844
9845// Transform `binOp (select cond, x, c0), c1` where `c0` and `c1` are constants
9846// into `select cond, binOp(x, c1), binOp(c0, c1)` if profitable.
9847// For now we only consider transformation profitable if `binOp(c0, c1)` ends up
9848// being `0` or `-1`. In such cases we can replace `select` with `and`.
9849// TODO: Should we also do this if `binOp(c0, c1)` is cheaper to materialize
9850// than `c0`?
9851static SDValue
9852foldBinOpIntoSelectIfProfitable(SDNode *BO, SelectionDAG &DAG,
9853 const RISCVSubtarget &Subtarget) {
9854 if (Subtarget.hasShortForwardBranchIALU())
9855 return SDValue();
9856
9857 unsigned SelOpNo = 0;
9858 SDValue Sel = BO->getOperand(Num: 0);
9859 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse()) {
9860 SelOpNo = 1;
9861 Sel = BO->getOperand(Num: 1);
9862 }
9863
9864 if (Sel.getOpcode() != ISD::SELECT || !Sel.hasOneUse())
9865 return SDValue();
9866
9867 unsigned ConstSelOpNo = 1;
9868 unsigned OtherSelOpNo = 2;
9869 if (!isa<ConstantSDNode>(Val: Sel->getOperand(Num: ConstSelOpNo))) {
9870 ConstSelOpNo = 2;
9871 OtherSelOpNo = 1;
9872 }
9873 SDValue ConstSelOp = Sel->getOperand(Num: ConstSelOpNo);
9874 ConstantSDNode *ConstSelOpNode = dyn_cast<ConstantSDNode>(Val&: ConstSelOp);
9875 if (!ConstSelOpNode || ConstSelOpNode->isOpaque())
9876 return SDValue();
9877
9878 SDValue ConstBinOp = BO->getOperand(Num: SelOpNo ^ 1);
9879 ConstantSDNode *ConstBinOpNode = dyn_cast<ConstantSDNode>(Val&: ConstBinOp);
9880 if (!ConstBinOpNode || ConstBinOpNode->isOpaque())
9881 return SDValue();
9882
9883 SDLoc DL(Sel);
9884 EVT VT = BO->getValueType(ResNo: 0);
9885
9886 SDValue NewConstOps[2] = {ConstSelOp, ConstBinOp};
9887 if (SelOpNo == 1)
9888 std::swap(a&: NewConstOps[0], b&: NewConstOps[1]);
9889
9890 SDValue NewConstOp =
9891 DAG.FoldConstantArithmetic(Opcode: BO->getOpcode(), DL, VT, Ops: NewConstOps);
9892 if (!NewConstOp)
9893 return SDValue();
9894
9895 const APInt &NewConstAPInt = NewConstOp->getAsAPIntVal();
9896 if (!NewConstAPInt.isZero() && !NewConstAPInt.isAllOnes())
9897 return SDValue();
9898
9899 SDValue OtherSelOp = Sel->getOperand(Num: OtherSelOpNo);
9900 SDValue NewNonConstOps[2] = {OtherSelOp, ConstBinOp};
9901 if (SelOpNo == 1)
9902 std::swap(a&: NewNonConstOps[0], b&: NewNonConstOps[1]);
9903 SDValue NewNonConstOp = DAG.getNode(Opcode: BO->getOpcode(), DL, VT, Ops: NewNonConstOps);
9904
9905 SDValue NewT = (ConstSelOpNo == 1) ? NewConstOp : NewNonConstOp;
9906 SDValue NewF = (ConstSelOpNo == 1) ? NewNonConstOp : NewConstOp;
9907 return DAG.getSelect(DL, VT, Cond: Sel.getOperand(i: 0), LHS: NewT, RHS: NewF);
9908}
9909
9910SDValue RISCVTargetLowering::lowerSELECT(SDValue Op, SelectionDAG &DAG) const {
9911 SDValue CondV = Op.getOperand(i: 0);
9912 SDValue TrueV = Op.getOperand(i: 1);
9913 SDValue FalseV = Op.getOperand(i: 2);
9914 SDLoc DL(Op);
9915 MVT VT = Op.getSimpleValueType();
9916 MVT XLenVT = Subtarget.getXLenVT();
9917
9918 // Handle P extension packed types by bitcasting to XLenVT for selection,
9919 // e.g. select i1 %cond, <2 x i16> %TrueV, <2 x i16> %FalseV
9920 // These types fit in a single GPR so can use the same selection mechanism
9921 // as scalars.
9922 if (Subtarget.isPExtPackedType(VT)) {
9923 SDValue TrueVInt = DAG.getBitcast(VT: XLenVT, V: TrueV);
9924 SDValue FalseVInt = DAG.getBitcast(VT: XLenVT, V: FalseV);
9925 SDValue ResultInt =
9926 DAG.getNode(Opcode: ISD::SELECT, DL, VT: XLenVT, N1: CondV, N2: TrueVInt, N3: FalseVInt);
9927 return DAG.getBitcast(VT, V: ResultInt);
9928 }
9929
9930 // Lower vector SELECTs to VSELECTs by splatting the condition.
9931 if (VT.isVector()) {
9932 MVT SplatCondVT = VT.changeVectorElementType(EltVT: MVT::i1);
9933 SDValue CondSplat = DAG.getSplat(VT: SplatCondVT, DL, Op: CondV);
9934 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CondSplat, N2: TrueV, N3: FalseV);
9935 }
9936
9937 // Try some other optimizations before falling back to generic lowering.
9938 if (SDValue V = lowerSelectToBinOp(N: Op.getNode(), DAG, Subtarget))
9939 return V;
9940
9941 // When there is no cost for GPR <-> FPR, we can use zicond select for
9942 // floating value when CondV is int type
9943 bool FPinGPR = Subtarget.hasStdExtZfinx();
9944
9945 // We can handle FGPR without spliting into hi/lo parts
9946 bool FitsInGPR = TypeSize::isKnownLE(LHS: VT.getSizeInBits(),
9947 RHS: Subtarget.getXLenVT().getSizeInBits());
9948
9949 bool UseZicondForFPSel = Subtarget.hasStdExtZicond() && FPinGPR &&
9950 VT.isFloatingPoint() && FitsInGPR;
9951
9952 if (UseZicondForFPSel) {
9953
9954 auto CastToInt = [&](SDValue V) -> SDValue {
9955 // Treat +0.0 as int 0 to enable single 'czero' instruction generation.
9956 if (isNullFPConstant(V))
9957 return DAG.getConstant(Val: 0, DL, VT: XLenVT);
9958
9959 if (VT == MVT::f16)
9960 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: V);
9961
9962 if (VT == MVT::f32 && Subtarget.is64Bit())
9963 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: XLenVT, Operand: V);
9964
9965 return DAG.getBitcast(VT: XLenVT, V);
9966 };
9967
9968 SDValue TrueVInt = CastToInt(TrueV);
9969 SDValue FalseVInt = CastToInt(FalseV);
9970
9971 // Emit integer SELECT (lowers to Zicond)
9972 SDValue ResultInt =
9973 DAG.getNode(Opcode: ISD::SELECT, DL, VT: XLenVT, N1: CondV, N2: TrueVInt, N3: FalseVInt);
9974
9975 // Convert back to floating VT
9976 if (VT == MVT::f32 && Subtarget.is64Bit())
9977 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT, Operand: ResultInt);
9978
9979 if (VT == MVT::f16)
9980 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT, Operand: ResultInt);
9981
9982 return DAG.getBitcast(VT, V: ResultInt);
9983 }
9984
9985 // When Zicond or XVentanaCondOps is present, emit CZERO_EQZ and CZERO_NEZ
9986 // nodes to implement the SELECT. Performing the lowering here allows for
9987 // greater control over when CZERO_{EQZ/NEZ} are used vs another branchless
9988 // sequence or RISCVISD::SELECT_CC node (branch-based select).
9989 if (Subtarget.hasCZEROLike() && VT.isScalarInteger()) {
9990
9991 // (select c, t, 0) -> (czero_eqz t, c)
9992 if (isNullConstant(V: FalseV))
9993 return DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV);
9994 // (select c, 0, f) -> (czero_nez f, c)
9995 if (isNullConstant(V: TrueV))
9996 return DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV);
9997
9998 // Check to see if a given operation is a 'NOT', if so return the negated
9999 // operand
10000 auto getNotOperand = [](const SDValue &Op) -> std::optional<const SDValue> {
10001 using namespace llvm::SDPatternMatch;
10002 SDValue Xor;
10003 if (sd_match(N: Op, P: m_OneUse(P: m_Not(V: m_Value(N&: Xor))))) {
10004 return Xor;
10005 }
10006 return std::nullopt;
10007 };
10008 // (select c, (and f, x), f) -> (or (and f, x), (czero_nez f, c))
10009 // (select c, (and f, ~x), f) -> (andn f, (czero_eqz x, c))
10010 if (TrueV.getOpcode() == ISD::AND &&
10011 (TrueV.getOperand(i: 0) == FalseV || TrueV.getOperand(i: 1) == FalseV)) {
10012 auto NotOperand = (TrueV.getOperand(i: 0) == FalseV)
10013 ? getNotOperand(TrueV.getOperand(i: 1))
10014 : getNotOperand(TrueV.getOperand(i: 0));
10015 if (NotOperand) {
10016 SDValue CMOV =
10017 DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: *NotOperand, N2: CondV);
10018 SDValue NOT = DAG.getNOT(DL, Val: CMOV, VT);
10019 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: FalseV, N2: NOT);
10020 }
10021 return DAG.getNode(
10022 Opcode: ISD::OR, DL, VT, N1: TrueV,
10023 N2: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV));
10024 }
10025
10026 // (select c, t, (and t, x)) -> (or (czero_eqz t, c), (and t, x))
10027 // (select c, t, (and t, ~x)) -> (andn t, (czero_nez x, c))
10028 if (FalseV.getOpcode() == ISD::AND &&
10029 (FalseV.getOperand(i: 0) == TrueV || FalseV.getOperand(i: 1) == TrueV)) {
10030 auto NotOperand = (FalseV.getOperand(i: 0) == TrueV)
10031 ? getNotOperand(FalseV.getOperand(i: 1))
10032 : getNotOperand(FalseV.getOperand(i: 0));
10033 if (NotOperand) {
10034 SDValue CMOV =
10035 DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: *NotOperand, N2: CondV);
10036 SDValue NOT = DAG.getNOT(DL, Val: CMOV, VT);
10037 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: TrueV, N2: NOT);
10038 }
10039 return DAG.getNode(
10040 Opcode: ISD::OR, DL, VT, N1: FalseV,
10041 N2: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV));
10042 }
10043
10044 // (select c, c1, c2) -> (add (czero_nez c2 - c1, c), c1)
10045 // (select c, c1, c2) -> (add (czero_eqz c1 - c2, c), c2)
10046 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV)) {
10047 const APInt &TrueVal = TrueV->getAsAPIntVal();
10048 const APInt &FalseVal = FalseV->getAsAPIntVal();
10049
10050 // Prefer these over Zicond to avoid materializing an immediate:
10051 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
10052 // (select (x > -1), z, y) -> x >> (XLEN - 1) & (y - z) + z
10053 if (CondV.getOpcode() == ISD::SETCC &&
10054 CondV.getOperand(i: 0).getValueType() == VT && CondV.hasOneUse()) {
10055 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10056 if ((CCVal == ISD::SETLT && isNullConstant(V: CondV.getOperand(i: 1))) ||
10057 (CCVal == ISD::SETGT && isAllOnesConstant(V: CondV.getOperand(i: 1)))) {
10058 int64_t TrueImm = TrueVal.getSExtValue();
10059 int64_t FalseImm = FalseVal.getSExtValue();
10060 if (CCVal == ISD::SETGT)
10061 std::swap(a&: TrueImm, b&: FalseImm);
10062 if (isInt<12>(x: TrueImm) && isInt<12>(x: FalseImm) &&
10063 isInt<12>(x: TrueImm - FalseImm)) {
10064 SDValue SRA =
10065 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: CondV.getOperand(i: 0),
10066 N2: DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT));
10067 SDValue AND =
10068 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
10069 N2: DAG.getSignedConstant(Val: TrueImm - FalseImm, DL, VT));
10070 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND,
10071 N2: DAG.getSignedConstant(Val: FalseImm, DL, VT));
10072 }
10073 }
10074 }
10075
10076 // Use SHL/ADDI (and possible XORI) to avoid having to materialize
10077 // a constant in register
10078 if ((TrueVal - FalseVal).isPowerOf2() && FalseVal.isSignedIntN(N: 12)) {
10079 SDValue Log2 = DAG.getConstant(Val: (TrueVal - FalseVal).logBase2(), DL, VT);
10080 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10081 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FalseV, N2: BitDiff);
10082 }
10083 if ((FalseVal - TrueVal).isPowerOf2() && TrueVal.isSignedIntN(N: 12)) {
10084 SDValue Log2 = DAG.getConstant(Val: (FalseVal - TrueVal).logBase2(), DL, VT);
10085 CondV = DAG.getLogicalNOT(DL, Val: CondV, VT: CondV->getValueType(ResNo: 0));
10086 SDValue BitDiff = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: CondV, N2: Log2);
10087 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: TrueV, N2: BitDiff);
10088 }
10089
10090 auto getCost = [&](const APInt &Delta, const APInt &Addend) {
10091 const int DeltaCost = RISCVMatInt::getIntMatCost(
10092 Val: Delta, Size: Subtarget.getXLen(), STI: Subtarget, /*CompressionCost=*/true);
10093 // Does the addend fold into an ADDI
10094 if (Addend.isSignedIntN(N: 12))
10095 return DeltaCost;
10096 const int AddendCost = RISCVMatInt::getIntMatCost(
10097 Val: Addend, Size: Subtarget.getXLen(), STI: Subtarget, /*CompressionCost=*/true);
10098 return AddendCost + DeltaCost;
10099 };
10100 bool IsCZERO_NEZ = getCost(FalseVal - TrueVal, TrueVal) <=
10101 getCost(TrueVal - FalseVal, FalseVal);
10102 SDValue LHSVal = DAG.getConstant(
10103 Val: IsCZERO_NEZ ? FalseVal - TrueVal : TrueVal - FalseVal, DL, VT);
10104 SDValue CMOV =
10105 DAG.getNode(Opcode: IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10106 DL, VT, N1: LHSVal, N2: CondV);
10107 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CMOV, N2: IsCZERO_NEZ ? TrueV : FalseV);
10108 }
10109
10110 // (select c, c1, t) -> (add (czero_nez t - c1, c), c1)
10111 // (select c, t, c1) -> (add (czero_eqz t - c1, c), c1)
10112 if (isa<ConstantSDNode>(Val: TrueV) != isa<ConstantSDNode>(Val: FalseV)) {
10113 bool IsCZERO_NEZ = isa<ConstantSDNode>(Val: TrueV);
10114 SDValue ConstVal = IsCZERO_NEZ ? TrueV : FalseV;
10115 SDValue RegV = IsCZERO_NEZ ? FalseV : TrueV;
10116 int64_t RawConstVal = cast<ConstantSDNode>(Val&: ConstVal)->getSExtValue();
10117 // Efficient only if the constant and its negation fit into `ADDI`
10118 // Prefer Add/Sub over Xor since can be compressed for small immediates
10119 if (isInt<12>(x: RawConstVal)) {
10120 // Fall back to XORI if Const == -0x800 since we don't have SUBI.
10121 unsigned SubOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::SUB;
10122 unsigned AddOpc = (RawConstVal == -0x800) ? ISD::XOR : ISD::ADD;
10123 SDValue SubOp = DAG.getNode(Opcode: SubOpc, DL, VT, N1: RegV, N2: ConstVal);
10124 SDValue CZERO =
10125 DAG.getNode(Opcode: IsCZERO_NEZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ,
10126 DL, VT, N1: SubOp, N2: CondV);
10127 return DAG.getNode(Opcode: AddOpc, DL, VT, N1: CZERO, N2: ConstVal);
10128 }
10129 }
10130
10131 // (select c, t, f) -> (or (czero_eqz t, c), (czero_nez f, c))
10132 // Unless we have the short forward branch optimization.
10133 if (!Subtarget.hasConditionalMoveFusion())
10134 return DAG.getNode(
10135 Opcode: ISD::OR, DL, VT,
10136 N1: DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV, N2: CondV),
10137 N2: DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV, N2: CondV),
10138 Flags: SDNodeFlags::Disjoint);
10139 }
10140
10141 if (Op.hasOneUse()) {
10142 unsigned UseOpc = Op->user_begin()->getOpcode();
10143 if (isBinOp(Opcode: UseOpc) && DAG.isSafeToSpeculativelyExecute(Opcode: UseOpc)) {
10144 SDNode *BinOp = *Op->user_begin();
10145 if (SDValue NewSel = foldBinOpIntoSelectIfProfitable(BO: *Op->user_begin(),
10146 DAG, Subtarget)) {
10147 DAG.ReplaceAllUsesWith(From: BinOp, To: &NewSel);
10148 // Opcode check is necessary because foldBinOpIntoSelectIfProfitable
10149 // may return a constant node and cause crash in lowerSELECT.
10150 if (NewSel.getOpcode() == ISD::SELECT)
10151 return lowerSELECT(Op: NewSel, DAG);
10152 return NewSel;
10153 }
10154 }
10155 }
10156
10157 // (select cc, 1.0, 0.0) -> (sint_to_fp (zext cc))
10158 // (select cc, 0.0, 1.0) -> (sint_to_fp (zext (xor cc, 1)))
10159 const ConstantFPSDNode *FPTV = dyn_cast<ConstantFPSDNode>(Val&: TrueV);
10160 const ConstantFPSDNode *FPFV = dyn_cast<ConstantFPSDNode>(Val&: FalseV);
10161 if (FPTV && FPFV) {
10162 if (FPTV->isExactlyValue(V: 1.0) && FPFV->isExactlyValue(V: 0.0))
10163 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: CondV);
10164 if (FPTV->isExactlyValue(V: 0.0) && FPFV->isExactlyValue(V: 1.0)) {
10165 SDValue XOR = DAG.getNode(Opcode: ISD::XOR, DL, VT: XLenVT, N1: CondV,
10166 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
10167 return DAG.getNode(Opcode: ISD::SINT_TO_FP, DL, VT, Operand: XOR);
10168 }
10169 }
10170
10171 // If the condition is not an integer SETCC which operates on XLenVT, we need
10172 // to emit a RISCVISD::SELECT_CC comparing the condition to zero. i.e.:
10173 // (select condv, truev, falsev)
10174 // -> (riscvisd::select_cc condv, zero, setne, truev, falsev)
10175 if (CondV.getOpcode() != ISD::SETCC ||
10176 CondV.getOperand(i: 0).getSimpleValueType() != XLenVT) {
10177 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
10178 SDValue SetNE = DAG.getCondCode(Cond: ISD::SETNE);
10179
10180 SDValue Ops[] = {CondV, Zero, SetNE, TrueV, FalseV};
10181
10182 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, Ops);
10183 }
10184
10185 // If the CondV is the output of a SETCC node which operates on XLenVT inputs,
10186 // then merge the SETCC node into the lowered RISCVISD::SELECT_CC to take
10187 // advantage of the integer compare+branch instructions. i.e.:
10188 // (select (setcc lhs, rhs, cc), truev, falsev)
10189 // -> (riscvisd::select_cc lhs, rhs, cc, truev, falsev)
10190 SDValue LHS = CondV.getOperand(i: 0);
10191 SDValue RHS = CondV.getOperand(i: 1);
10192 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10193
10194 // Special case for a select of 2 constants that have a difference of 1.
10195 // Normally this is done by DAGCombine, but if the select is introduced by
10196 // type legalization or op legalization, we miss it. Restricting to SETLT
10197 // case for now because that is what signed saturating add/sub need.
10198 // FIXME: We don't need the condition to be SETLT or even a SETCC,
10199 // but we would probably want to swap the true/false values if the condition
10200 // is SETGE/SETLE to avoid an XORI.
10201 if (isa<ConstantSDNode>(Val: TrueV) && isa<ConstantSDNode>(Val: FalseV) &&
10202 CCVal == ISD::SETLT) {
10203 const APInt &TrueVal = TrueV->getAsAPIntVal();
10204 const APInt &FalseVal = FalseV->getAsAPIntVal();
10205 if (TrueVal - 1 == FalseVal)
10206 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: CondV, N2: FalseV);
10207 if (TrueVal + 1 == FalseVal)
10208 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: FalseV, N2: CondV);
10209 }
10210
10211 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
10212 // 1 < x ? x : 1 -> 0 < x ? x : 1
10213 if (isOneConstant(V: LHS) && (CCVal == ISD::SETLT || CCVal == ISD::SETULT) &&
10214 RHS == TrueV && LHS == FalseV) {
10215 LHS = DAG.getConstant(Val: 0, DL, VT);
10216 // 0 <u x is the same as x != 0.
10217 if (CCVal == ISD::SETULT) {
10218 std::swap(a&: LHS, b&: RHS);
10219 CCVal = ISD::SETNE;
10220 }
10221 }
10222
10223 // x <s -1 ? x : -1 -> x <s 0 ? x : -1
10224 if (isAllOnesConstant(V: RHS) && CCVal == ISD::SETLT && LHS == TrueV &&
10225 RHS == FalseV) {
10226 RHS = DAG.getConstant(Val: 0, DL, VT);
10227 }
10228
10229 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
10230
10231 if (isa<ConstantSDNode>(Val: TrueV) && !isa<ConstantSDNode>(Val: FalseV)) {
10232 // (select (setcc lhs, rhs, CC), constant, falsev)
10233 // -> (select (setcc lhs, rhs, InverseCC), falsev, constant)
10234 std::swap(a&: TrueV, b&: FalseV);
10235 TargetCC = DAG.getCondCode(Cond: ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType()));
10236 }
10237
10238 SDValue Ops[] = {LHS, RHS, TargetCC, TrueV, FalseV};
10239 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, Ops);
10240}
10241
10242SDValue RISCVTargetLowering::lowerBRCOND(SDValue Op, SelectionDAG &DAG) const {
10243 SDValue CondV = Op.getOperand(i: 1);
10244 SDLoc DL(Op);
10245 MVT XLenVT = Subtarget.getXLenVT();
10246
10247 if (CondV.getOpcode() == ISD::SETCC &&
10248 CondV.getOperand(i: 0).getValueType() == XLenVT) {
10249 SDValue LHS = CondV.getOperand(i: 0);
10250 SDValue RHS = CondV.getOperand(i: 1);
10251 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: CondV.getOperand(i: 2))->get();
10252
10253 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
10254
10255 SDValue TargetCC = DAG.getCondCode(Cond: CCVal);
10256 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
10257 N2: LHS, N3: RHS, N4: TargetCC, N5: Op.getOperand(i: 2));
10258 }
10259
10260 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: Op.getValueType(), N1: Op.getOperand(i: 0),
10261 N2: CondV, N3: DAG.getConstant(Val: 0, DL, VT: XLenVT),
10262 N4: DAG.getCondCode(Cond: ISD::SETNE), N5: Op.getOperand(i: 2));
10263}
10264
10265SDValue RISCVTargetLowering::lowerVASTART(SDValue Op, SelectionDAG &DAG) const {
10266 MachineFunction &MF = DAG.getMachineFunction();
10267 RISCVMachineFunctionInfo *FuncInfo = MF.getInfo<RISCVMachineFunctionInfo>();
10268
10269 SDLoc DL(Op);
10270 SDValue FI = DAG.getFrameIndex(FI: FuncInfo->getVarArgsFrameIndex(),
10271 VT: getPointerTy(DL: MF.getDataLayout()));
10272
10273 // vastart just stores the address of the VarArgsFrameIndex slot into the
10274 // memory location argument.
10275 const Value *SV = cast<SrcValueSDNode>(Val: Op.getOperand(i: 2))->getValue();
10276 return DAG.getStore(Chain: Op.getOperand(i: 0), dl: DL, Val: FI, Ptr: Op.getOperand(i: 1),
10277 PtrInfo: MachinePointerInfo(SV));
10278}
10279
10280SDValue RISCVTargetLowering::lowerFRAMEADDR(SDValue Op,
10281 SelectionDAG &DAG) const {
10282 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10283 MachineFunction &MF = DAG.getMachineFunction();
10284 MachineFrameInfo &MFI = MF.getFrameInfo();
10285 MFI.setFrameAddressIsTaken(true);
10286 Register FrameReg = RI.getFrameRegister(MF);
10287 int XLenInBytes = Subtarget.getXLen() / 8;
10288
10289 EVT VT = Op.getValueType();
10290 SDLoc DL(Op);
10291 SDValue FrameAddr = DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg: FrameReg, VT);
10292 unsigned Depth = Op.getConstantOperandVal(i: 0);
10293 while (Depth--) {
10294 int Offset = -(XLenInBytes * 2);
10295 SDValue Ptr = DAG.getNode(
10296 Opcode: ISD::ADD, DL, VT, N1: FrameAddr,
10297 N2: DAG.getSignedConstant(Val: Offset, DL, VT: getPointerTy(DL: DAG.getDataLayout())));
10298 FrameAddr =
10299 DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(), Ptr, PtrInfo: MachinePointerInfo());
10300 }
10301 return FrameAddr;
10302}
10303
10304SDValue RISCVTargetLowering::lowerRETURNADDR(SDValue Op,
10305 SelectionDAG &DAG) const {
10306 const RISCVRegisterInfo &RI = *Subtarget.getRegisterInfo();
10307 MachineFunction &MF = DAG.getMachineFunction();
10308 MachineFrameInfo &MFI = MF.getFrameInfo();
10309 MFI.setReturnAddressIsTaken(true);
10310 MVT XLenVT = Subtarget.getXLenVT();
10311 int XLenInBytes = Subtarget.getXLen() / 8;
10312
10313 EVT VT = Op.getValueType();
10314 SDLoc DL(Op);
10315 unsigned Depth = Op.getConstantOperandVal(i: 0);
10316 if (Depth) {
10317 int Off = -XLenInBytes;
10318 SDValue FrameAddr = lowerFRAMEADDR(Op, DAG);
10319 SDValue Offset = DAG.getSignedConstant(Val: Off, DL, VT);
10320 return DAG.getLoad(VT, dl: DL, Chain: DAG.getEntryNode(),
10321 Ptr: DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: FrameAddr, N2: Offset),
10322 PtrInfo: MachinePointerInfo());
10323 }
10324
10325 // Return the value of the return address register, marking it an implicit
10326 // live-in.
10327 Register Reg = MF.addLiveIn(PReg: RI.getRARegister(), RC: getRegClassFor(VT: XLenVT));
10328 return DAG.getCopyFromReg(Chain: DAG.getEntryNode(), dl: DL, Reg, VT: XLenVT);
10329}
10330
10331SDValue RISCVTargetLowering::lowerShiftLeftParts(SDValue Op,
10332 SelectionDAG &DAG) const {
10333 SDLoc DL(Op);
10334 SDValue Lo = Op.getOperand(i: 0);
10335 SDValue Hi = Op.getOperand(i: 1);
10336 SDValue Shamt = Op.getOperand(i: 2);
10337 EVT VT = Lo.getValueType();
10338 unsigned XLen = Subtarget.getXLen();
10339
10340 // With P extension, use SLX (FSHL) for the high part.
10341 if (Subtarget.hasStdExtP()) {
10342 // HiRes = fshl(Hi, Lo, Shamt) - correct when Shamt < XLen
10343 SDValue HiRes = DAG.getNode(Opcode: ISD::FSHL, DL, VT, N1: Hi, N2: Lo, N3: Shamt);
10344 // LoRes = Lo << Shamt - correct Lo when Shamt < XLen,
10345 // Mask shift amount to avoid UB when Shamt >= XLen.
10346 SDValue ShamtMasked =
10347 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shamt, N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10348 SDValue LoRes = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMasked);
10349
10350 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
10351 // FIXME: We should use a select and let LowerSelect make the
10352 // optimizations.
10353 SDValue ShAmtExt =
10354 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Shamt,
10355 N2: DAG.getConstant(Val: XLen - Log2_32(Value: XLen) - 1, DL, VT));
10356 SDValue Mask = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: ShAmtExt,
10357 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10358
10359 // When Shamt >= XLen: HiRes = LoRes, LoRes = 0
10360 // HiRes = (HiRes & ~Mask) | (LoRes & Mask)
10361 SDValue HiMasked =
10362 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10363 SDValue LoMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: Mask);
10364 HiRes =
10365 DAG.getNode(Opcode: ISD::OR, DL, VT, N1: HiMasked, N2: LoMasked, Flags: SDNodeFlags::Disjoint);
10366
10367 // LoRes = LoRes & ~Mask (clear when Shamt >= XLen)
10368 LoRes = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10369
10370 return DAG.getMergeValues(Ops: {LoRes, HiRes}, dl: DL);
10371 }
10372
10373 // if Shamt-XLEN < 0: // Shamt < XLEN
10374 // Lo = Lo << Shamt
10375 // Hi = (Hi << Shamt) | ((Lo >>u 1) >>u (XLEN-1 - Shamt))
10376 // else:
10377 // Lo = 0
10378 // Hi = Lo << (Shamt-XLEN)
10379
10380 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
10381 SDValue One = DAG.getConstant(Val: 1, DL, VT);
10382 SDValue MinusXLen = DAG.getSignedConstant(Val: -(int)XLen, DL, VT);
10383 SDValue XLenMinus1 = DAG.getConstant(Val: XLen - 1, DL, VT);
10384 SDValue ShamtMinusXLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusXLen);
10385 SDValue XLenMinus1Shamt = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: XLenMinus1, N2: Shamt);
10386
10387 SDValue LoTrue = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: Shamt);
10388 SDValue ShiftRight1Lo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: One);
10389 SDValue ShiftRightLo =
10390 DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: ShiftRight1Lo, N2: XLenMinus1Shamt);
10391 SDValue ShiftLeftHi = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: Shamt);
10392 SDValue HiTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftLeftHi, N2: ShiftRightLo);
10393 SDValue HiFalse = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Lo, N2: ShamtMinusXLen);
10394
10395 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusXLen, RHS: Zero, Cond: ISD::SETLT);
10396
10397 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: Zero);
10398 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
10399
10400 SDValue Parts[2] = {Lo, Hi};
10401 return DAG.getMergeValues(Ops: Parts, dl: DL);
10402}
10403
10404SDValue RISCVTargetLowering::lowerShiftRightParts(SDValue Op, SelectionDAG &DAG,
10405 bool IsSRA) const {
10406 SDLoc DL(Op);
10407 SDValue Lo = Op.getOperand(i: 0);
10408 SDValue Hi = Op.getOperand(i: 1);
10409 SDValue Shamt = Op.getOperand(i: 2);
10410 EVT VT = Lo.getValueType();
10411
10412 // With P extension, use NSRL/NSRA for RV32 or FSHR (SRX) for RV64.
10413 if (Subtarget.hasStdExtP()) {
10414 unsigned XLen = Subtarget.getXLen();
10415
10416 SDValue LoRes;
10417 if (Subtarget.is64Bit()) {
10418 // On RV64, use FSHR (SRX instruction) for the low part. We will need
10419 // to fix this later if ShAmt >= 64.
10420 LoRes = DAG.getNode(Opcode: ISD::FSHR, DL, VT, N1: Hi, N2: Lo, N3: Shamt);
10421 } else {
10422 // On RV32, use NSRL/NSRA for the low part.
10423 // NSRL/NSRA read 6 bits of shift amount, so they handle Shamt >= 32
10424 // correctly.
10425 LoRes = DAG.getNode(Opcode: IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL, VT, N1: Lo,
10426 N2: Hi, N3: Shamt);
10427 }
10428
10429 // Mask shift amount to avoid UB when Shamt >= XLen.
10430 SDValue ShamtMasked =
10431 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Shamt, N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10432 SDValue HiRes =
10433 DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL, VT, N1: Hi, N2: ShamtMasked);
10434
10435 // Create a mask that is -1 when Shamt >= XLen, 0 otherwise.
10436 // FIXME: We should use a select and let LowerSelect make the
10437 // optimizations.
10438 SDValue ShAmtExt =
10439 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Shamt,
10440 N2: DAG.getConstant(Val: XLen - Log2_32(Value: XLen) - 1, DL, VT));
10441 SDValue Mask = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: ShAmtExt,
10442 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10443
10444 if (Subtarget.is64Bit()) {
10445 // On RV64, FSHR masks shift amount to 63. We need to replace LoRes
10446 // with HiRes when Shamt >= 64.
10447 // LoRes = (LoRes & ~Mask) | (HiRes & Mask)
10448 SDValue LoMasked =
10449 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10450 SDValue HiMasked = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: Mask);
10451 LoRes = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: LoMasked, N2: HiMasked,
10452 Flags: SDNodeFlags::Disjoint);
10453 }
10454
10455 // If ShAmt >= XLen, we need to replace HiRes with 0 or sign bits.
10456 if (IsSRA) {
10457 // sra hi, hi, (mask & (XLen-1)) - shifts by XLen-1 when shamt >= XLen
10458 SDValue MaskAmt = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Mask,
10459 N2: DAG.getConstant(Val: XLen - 1, DL, VT));
10460 HiRes = DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: HiRes, N2: MaskAmt);
10461 } else {
10462 // andn hi, hi, mask - clears hi when shamt >= XLen
10463 HiRes = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: HiRes, N2: DAG.getNOT(DL, Val: Mask, VT));
10464 }
10465
10466 return DAG.getMergeValues(Ops: {LoRes, HiRes}, dl: DL);
10467 }
10468
10469 // SRA expansion:
10470 // if Shamt-XLEN < 0: // Shamt < XLEN
10471 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
10472 // Hi = Hi >>s Shamt
10473 // else:
10474 // Lo = Hi >>s (Shamt-XLEN);
10475 // Hi = Hi >>s (XLEN-1)
10476 //
10477 // SRL expansion:
10478 // if Shamt-XLEN < 0: // Shamt < XLEN
10479 // Lo = (Lo >>u Shamt) | ((Hi << 1) << (XLEN-1 - ShAmt))
10480 // Hi = Hi >>u Shamt
10481 // else:
10482 // Lo = Hi >>u (Shamt-XLEN);
10483 // Hi = 0;
10484
10485 unsigned ShiftRightOp = IsSRA ? ISD::SRA : ISD::SRL;
10486
10487 SDValue Zero = DAG.getConstant(Val: 0, DL, VT);
10488 SDValue One = DAG.getConstant(Val: 1, DL, VT);
10489 SDValue MinusXLen = DAG.getSignedConstant(Val: -(int)Subtarget.getXLen(), DL, VT);
10490 SDValue XLenMinus1 = DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT);
10491 SDValue ShamtMinusXLen = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Shamt, N2: MinusXLen);
10492 SDValue XLenMinus1Shamt = DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: XLenMinus1, N2: Shamt);
10493
10494 SDValue ShiftRightLo = DAG.getNode(Opcode: ISD::SRL, DL, VT, N1: Lo, N2: Shamt);
10495 SDValue ShiftLeftHi1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: Hi, N2: One);
10496 SDValue ShiftLeftHi =
10497 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShiftLeftHi1, N2: XLenMinus1Shamt);
10498 SDValue LoTrue = DAG.getNode(Opcode: ISD::OR, DL, VT, N1: ShiftRightLo, N2: ShiftLeftHi);
10499 SDValue HiTrue = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: Shamt);
10500 SDValue LoFalse = DAG.getNode(Opcode: ShiftRightOp, DL, VT, N1: Hi, N2: ShamtMinusXLen);
10501 SDValue HiFalse =
10502 IsSRA ? DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Hi, N2: XLenMinus1) : Zero;
10503
10504 SDValue CC = DAG.getSetCC(DL, VT, LHS: ShamtMinusXLen, RHS: Zero, Cond: ISD::SETLT);
10505
10506 Lo = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: LoTrue, N3: LoFalse);
10507 Hi = DAG.getNode(Opcode: ISD::SELECT, DL, VT, N1: CC, N2: HiTrue, N3: HiFalse);
10508
10509 SDValue Parts[2] = {Lo, Hi};
10510 return DAG.getMergeValues(Ops: Parts, dl: DL);
10511}
10512
10513// Lower splats of i1 types to SETCC. For each mask vector type, we have a
10514// legal equivalently-sized i8 type, so we can use that as a go-between.
10515SDValue RISCVTargetLowering::lowerVectorMaskSplat(SDValue Op,
10516 SelectionDAG &DAG) const {
10517 SDLoc DL(Op);
10518 MVT VT = Op.getSimpleValueType();
10519 SDValue SplatVal = Op.getOperand(i: 0);
10520 // All-zeros or all-ones splats are handled specially.
10521 if (ISD::isConstantSplatVectorAllOnes(N: Op.getNode())) {
10522 SDValue VL = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget).second;
10523 return DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT, Operand: VL);
10524 }
10525 if (ISD::isConstantSplatVectorAllZeros(N: Op.getNode())) {
10526 SDValue VL = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget).second;
10527 return DAG.getNode(Opcode: RISCVISD::VMCLR_VL, DL, VT, Operand: VL);
10528 }
10529 MVT InterVT = VT.changeVectorElementType(EltVT: MVT::i8);
10530 SplatVal = DAG.getNode(Opcode: ISD::AND, DL, VT: SplatVal.getValueType(), N1: SplatVal,
10531 N2: DAG.getConstant(Val: 1, DL, VT: SplatVal.getValueType()));
10532 SDValue LHS = DAG.getSplatVector(VT: InterVT, DL, Op: SplatVal);
10533 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: InterVT);
10534 return DAG.getSetCC(DL, VT, LHS, RHS: Zero, Cond: ISD::SETNE);
10535}
10536
10537// Custom-lower a SPLAT_VECTOR_PARTS where XLEN<SEW, as the SEW element type is
10538// illegal (currently only vXi64 RV32).
10539// FIXME: We could also catch non-constant sign-extended i32 values and lower
10540// them to VMV_V_X_VL.
10541SDValue RISCVTargetLowering::lowerSPLAT_VECTOR_PARTS(SDValue Op,
10542 SelectionDAG &DAG) const {
10543 SDLoc DL(Op);
10544 MVT VecVT = Op.getSimpleValueType();
10545 assert(!Subtarget.is64Bit() && VecVT.getVectorElementType() == MVT::i64 &&
10546 "Unexpected SPLAT_VECTOR_PARTS lowering");
10547
10548 assert(Op.getNumOperands() == 2 && "Unexpected number of operands!");
10549 SDValue Lo = Op.getOperand(i: 0);
10550 SDValue Hi = Op.getOperand(i: 1);
10551
10552 MVT ContainerVT = VecVT;
10553 if (VecVT.isFixedLengthVector())
10554 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
10555
10556 auto VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
10557
10558 SDValue Res =
10559 splatPartsI64WithVL(DL, VT: ContainerVT, Passthru: SDValue(), Lo, Hi, VL, DAG);
10560
10561 if (VecVT.isFixedLengthVector())
10562 Res = convertFromScalableVector(VT: VecVT, V: Res, DAG, Subtarget);
10563
10564 return Res;
10565}
10566
10567// Custom-lower extensions from mask vectors by using a vselect either with 1
10568// for zero/any-extension or -1 for sign-extension:
10569// (vXiN = (s|z)ext vXi1:vmask) -> (vXiN = vselect vmask, (-1 or 1), 0)
10570// Note that any-extension is lowered identically to zero-extension.
10571SDValue RISCVTargetLowering::lowerVectorMaskExt(SDValue Op, SelectionDAG &DAG,
10572 int64_t ExtTrueVal) const {
10573 SDLoc DL(Op);
10574 MVT VecVT = Op.getSimpleValueType();
10575 SDValue Src = Op.getOperand(i: 0);
10576 // Only custom-lower extensions from mask types
10577 assert(Src.getValueType().isVector() &&
10578 Src.getValueType().getVectorElementType() == MVT::i1);
10579
10580 if (VecVT.isScalableVector()) {
10581 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: VecVT);
10582 SDValue SplatTrueVal = DAG.getSignedConstant(Val: ExtTrueVal, DL, VT: VecVT);
10583 if (Src.getOpcode() == ISD::XOR &&
10584 ISD::isConstantSplatVectorAllOnes(N: Src.getOperand(i: 1).getNode()))
10585 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Src.getOperand(i: 0), N2: SplatZero,
10586 N3: SplatTrueVal);
10587 return DAG.getNode(Opcode: ISD::VSELECT, DL, VT: VecVT, N1: Src, N2: SplatTrueVal, N3: SplatZero);
10588 }
10589
10590 MVT ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
10591 MVT I1ContainerVT =
10592 MVT::getVectorVT(VT: MVT::i1, EC: ContainerVT.getVectorElementCount());
10593
10594 SDValue CC = convertToScalableVector(VT: I1ContainerVT, V: Src, DAG, Subtarget);
10595
10596 SDValue VL = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).second;
10597
10598 MVT XLenVT = Subtarget.getXLenVT();
10599 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
10600 SDValue SplatTrueVal = DAG.getSignedConstant(Val: ExtTrueVal, DL, VT: XLenVT);
10601
10602 if (Src.getOpcode() == ISD::EXTRACT_SUBVECTOR) {
10603 SDValue Xor = Src.getOperand(i: 0);
10604 if (Xor.getOpcode() == RISCVISD::VMXOR_VL) {
10605 SDValue ScalableOnes = Xor.getOperand(i: 1);
10606 if (ScalableOnes.getOpcode() == ISD::INSERT_SUBVECTOR &&
10607 ScalableOnes.getOperand(i: 0).isUndef() &&
10608 ISD::isConstantSplatVectorAllOnes(
10609 N: ScalableOnes.getOperand(i: 1).getNode())) {
10610 CC = Xor.getOperand(i: 0);
10611 std::swap(a&: SplatZero, b&: SplatTrueVal);
10612 }
10613 }
10614 }
10615
10616 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
10617 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatZero, N3: VL);
10618 SplatTrueVal = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
10619 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatTrueVal, N3: VL);
10620 SDValue Select =
10621 DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: CC, N2: SplatTrueVal,
10622 N3: SplatZero, N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
10623
10624 return convertFromScalableVector(VT: VecVT, V: Select, DAG, Subtarget);
10625}
10626
10627// Custom-lower truncations from vectors to mask vectors by using a mask and a
10628// setcc operation:
10629// (vXi1 = trunc vXiN vec) -> (vXi1 = setcc (and vec, 1), 0, ne)
10630SDValue RISCVTargetLowering::lowerVectorMaskTruncLike(SDValue Op,
10631 SelectionDAG &DAG) const {
10632 bool IsVPTrunc = Op.getOpcode() == ISD::VP_TRUNCATE;
10633 SDLoc DL(Op);
10634 EVT MaskVT = Op.getValueType();
10635 // Only expect to custom-lower truncations to mask types
10636 assert(MaskVT.isVector() && MaskVT.getVectorElementType() == MVT::i1 &&
10637 "Unexpected type for vector mask lowering");
10638 SDValue Src = Op.getOperand(i: 0);
10639 MVT VecVT = Src.getSimpleValueType();
10640 SDValue Mask, VL;
10641 if (IsVPTrunc) {
10642 Mask = Op.getOperand(i: 1);
10643 VL = Op.getOperand(i: 2);
10644 }
10645 // If this is a fixed vector, we need to convert it to a scalable vector.
10646 MVT ContainerVT = VecVT;
10647
10648 if (VecVT.isFixedLengthVector()) {
10649 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
10650 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
10651 if (IsVPTrunc) {
10652 MVT MaskContainerVT =
10653 getContainerForFixedLengthVector(VT: Mask.getSimpleValueType());
10654 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
10655 }
10656 }
10657
10658 if (!IsVPTrunc) {
10659 std::tie(args&: Mask, args&: VL) =
10660 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
10661 }
10662
10663 SDValue SplatOne = DAG.getConstant(Val: 1, DL, VT: Subtarget.getXLenVT());
10664 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
10665
10666 SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
10667 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatOne, N3: VL);
10668 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
10669 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatZero, N3: VL);
10670
10671 MVT MaskContainerVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
10672 SDValue Trunc = DAG.getNode(Opcode: RISCVISD::AND_VL, DL, VT: ContainerVT, N1: Src, N2: SplatOne,
10673 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
10674 Trunc = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: MaskContainerVT,
10675 Ops: {Trunc, SplatZero, DAG.getCondCode(Cond: ISD::SETNE),
10676 DAG.getUNDEF(VT: MaskContainerVT), Mask, VL});
10677 if (MaskVT.isFixedLengthVector())
10678 Trunc = convertFromScalableVector(VT: MaskVT, V: Trunc, DAG, Subtarget);
10679 return Trunc;
10680}
10681
10682SDValue RISCVTargetLowering::lowerVectorTruncLike(SDValue Op,
10683 SelectionDAG &DAG) const {
10684 unsigned Opc = Op.getOpcode();
10685 bool IsVPTrunc = Opc == ISD::VP_TRUNCATE;
10686 SDLoc DL(Op);
10687
10688 MVT VT = Op.getSimpleValueType();
10689 // Only custom-lower vector truncates
10690 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
10691
10692 // Truncates to mask types are handled differently
10693 if (VT.getVectorElementType() == MVT::i1)
10694 return lowerVectorMaskTruncLike(Op, DAG);
10695
10696 // RVV only has truncates which operate from SEW*2->SEW, so lower arbitrary
10697 // truncates as a series of "RISCVISD::TRUNCATE_VECTOR_VL" nodes which
10698 // truncate by one power of two at a time.
10699 MVT DstEltVT = VT.getVectorElementType();
10700
10701 SDValue Src = Op.getOperand(i: 0);
10702 MVT SrcVT = Src.getSimpleValueType();
10703 MVT SrcEltVT = SrcVT.getVectorElementType();
10704
10705 assert(DstEltVT.bitsLT(SrcEltVT) && isPowerOf2_64(DstEltVT.getSizeInBits()) &&
10706 isPowerOf2_64(SrcEltVT.getSizeInBits()) &&
10707 "Unexpected vector truncate lowering");
10708
10709 MVT ContainerVT = SrcVT;
10710 SDValue Mask, VL;
10711 if (IsVPTrunc) {
10712 Mask = Op.getOperand(i: 1);
10713 VL = Op.getOperand(i: 2);
10714 }
10715 if (SrcVT.isFixedLengthVector()) {
10716 ContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
10717 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
10718 if (IsVPTrunc) {
10719 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
10720 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
10721 }
10722 }
10723
10724 SDValue Result = Src;
10725 if (!IsVPTrunc) {
10726 std::tie(args&: Mask, args&: VL) =
10727 getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
10728 }
10729
10730 unsigned NewOpc;
10731 if (Opc == ISD::TRUNCATE_SSAT_S)
10732 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
10733 else if (Opc == ISD::TRUNCATE_USAT_U)
10734 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
10735 else
10736 NewOpc = RISCVISD::TRUNCATE_VECTOR_VL;
10737
10738 do {
10739 SrcEltVT = MVT::getIntegerVT(BitWidth: SrcEltVT.getSizeInBits() / 2);
10740 MVT ResultVT = ContainerVT.changeVectorElementType(EltVT: SrcEltVT);
10741 Result = DAG.getNode(Opcode: NewOpc, DL, VT: ResultVT, N1: Result, N2: Mask, N3: VL);
10742 } while (SrcEltVT != DstEltVT);
10743
10744 if (SrcVT.isFixedLengthVector())
10745 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
10746
10747 return Result;
10748}
10749
10750SDValue
10751RISCVTargetLowering::lowerStrictFPExtendOrRoundLike(SDValue Op,
10752 SelectionDAG &DAG) const {
10753 SDLoc DL(Op);
10754 SDValue Chain = Op.getOperand(i: 0);
10755 SDValue Src = Op.getOperand(i: 1);
10756 MVT VT = Op.getSimpleValueType();
10757 MVT SrcVT = Src.getSimpleValueType();
10758 MVT ContainerVT = VT;
10759 if (VT.isFixedLengthVector()) {
10760 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
10761 ContainerVT =
10762 SrcContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType());
10763 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
10764 }
10765
10766 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
10767
10768 // RVV can only widen/truncate fp to types double/half the size as the source.
10769 if ((VT.getVectorElementType() == MVT::f64 &&
10770 (SrcVT.getVectorElementType() == MVT::f16 ||
10771 SrcVT.getVectorElementType() == MVT::bf16)) ||
10772 ((VT.getVectorElementType() == MVT::f16 ||
10773 VT.getVectorElementType() == MVT::bf16) &&
10774 SrcVT.getVectorElementType() == MVT::f64)) {
10775 // For double rounding, the intermediate rounding should be round-to-odd.
10776 unsigned InterConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
10777 ? RISCVISD::STRICT_FP_EXTEND_VL
10778 : RISCVISD::STRICT_VFNCVT_ROD_VL;
10779 MVT InterVT = ContainerVT.changeVectorElementType(EltVT: MVT::f32);
10780 Src = DAG.getNode(Opcode: InterConvOpc, DL, VTList: DAG.getVTList(VT1: InterVT, VT2: MVT::Other),
10781 N1: Chain, N2: Src, N3: Mask, N4: VL);
10782 Chain = Src.getValue(R: 1);
10783 }
10784
10785 unsigned ConvOpc = Op.getOpcode() == ISD::STRICT_FP_EXTEND
10786 ? RISCVISD::STRICT_FP_EXTEND_VL
10787 : RISCVISD::STRICT_FP_ROUND_VL;
10788 SDValue Res = DAG.getNode(Opcode: ConvOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other),
10789 N1: Chain, N2: Src, N3: Mask, N4: VL);
10790 if (VT.isFixedLengthVector()) {
10791 // StrictFP operations have two result values. Their lowered result should
10792 // have same result count.
10793 SDValue SubVec = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
10794 Res = DAG.getMergeValues(Ops: {SubVec, Res.getValue(R: 1)}, dl: DL);
10795 }
10796 return Res;
10797}
10798
10799SDValue
10800RISCVTargetLowering::lowerVectorFPExtendOrRoundLike(SDValue Op,
10801 SelectionDAG &DAG) const {
10802 bool IsVP =
10803 Op.getOpcode() == ISD::VP_FP_ROUND || Op.getOpcode() == ISD::VP_FP_EXTEND;
10804 bool IsExtend =
10805 Op.getOpcode() == ISD::VP_FP_EXTEND || Op.getOpcode() == ISD::FP_EXTEND;
10806 // RVV can only do truncate fp to types half the size as the source. We
10807 // custom-lower f64->f16 rounds via RVV's round-to-odd float
10808 // conversion instruction.
10809 SDLoc DL(Op);
10810 MVT VT = Op.getSimpleValueType();
10811
10812 assert(VT.isVector() && "Unexpected type for vector truncate lowering");
10813
10814 SDValue Src = Op.getOperand(i: 0);
10815 MVT SrcVT = Src.getSimpleValueType();
10816
10817 bool IsDirectExtend =
10818 IsExtend && (VT.getVectorElementType() != MVT::f64 ||
10819 (SrcVT.getVectorElementType() != MVT::f16 &&
10820 SrcVT.getVectorElementType() != MVT::bf16));
10821 bool IsDirectTrunc = !IsExtend && ((VT.getVectorElementType() != MVT::f16 &&
10822 VT.getVectorElementType() != MVT::bf16) ||
10823 SrcVT.getVectorElementType() != MVT::f64);
10824
10825 bool IsDirectConv = IsDirectExtend || IsDirectTrunc;
10826
10827 // We have regular SD node patterns for direct non-VL extends.
10828 if (VT.isScalableVector() && IsDirectConv && !IsVP)
10829 return Op;
10830
10831 // Prepare any fixed-length vector operands.
10832 MVT ContainerVT = VT;
10833 SDValue Mask, VL;
10834 if (IsVP) {
10835 Mask = Op.getOperand(i: 1);
10836 VL = Op.getOperand(i: 2);
10837 }
10838 if (VT.isFixedLengthVector()) {
10839 MVT SrcContainerVT = getContainerForFixedLengthVector(VT: SrcVT);
10840 ContainerVT =
10841 SrcContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType());
10842 Src = convertToScalableVector(VT: SrcContainerVT, V: Src, DAG, Subtarget);
10843 if (IsVP) {
10844 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
10845 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
10846 }
10847 }
10848
10849 if (!IsVP)
10850 std::tie(args&: Mask, args&: VL) =
10851 getDefaultVLOps(VecVT: SrcVT, ContainerVT, DL, DAG, Subtarget);
10852
10853 unsigned ConvOpc = IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::FP_ROUND_VL;
10854
10855 if (IsDirectConv) {
10856 Src = DAG.getNode(Opcode: ConvOpc, DL, VT: ContainerVT, N1: Src, N2: Mask, N3: VL);
10857 if (VT.isFixedLengthVector())
10858 Src = convertFromScalableVector(VT, V: Src, DAG, Subtarget);
10859 return Src;
10860 }
10861
10862 unsigned InterConvOpc =
10863 IsExtend ? RISCVISD::FP_EXTEND_VL : RISCVISD::VFNCVT_ROD_VL;
10864
10865 MVT InterVT = ContainerVT.changeVectorElementType(EltVT: MVT::f32);
10866 SDValue IntermediateConv =
10867 DAG.getNode(Opcode: InterConvOpc, DL, VT: InterVT, N1: Src, N2: Mask, N3: VL);
10868 SDValue Result =
10869 DAG.getNode(Opcode: ConvOpc, DL, VT: ContainerVT, N1: IntermediateConv, N2: Mask, N3: VL);
10870 if (VT.isFixedLengthVector())
10871 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
10872 return Result;
10873}
10874
10875// Given a scalable vector type and an index into it, returns the type for the
10876// smallest subvector that the index fits in. This can be used to reduce LMUL
10877// for operations like vslidedown.
10878//
10879// E.g. With Zvl128b, index 3 in a nxv4i32 fits within the first nxv2i32.
10880static std::optional<MVT>
10881getSmallestVTForIndex(MVT VecVT, unsigned MaxIdx, SDLoc DL, SelectionDAG &DAG,
10882 const RISCVSubtarget &Subtarget) {
10883 assert(VecVT.isScalableVector());
10884 const unsigned EltSize = VecVT.getScalarSizeInBits();
10885 const unsigned VectorBitsMin = Subtarget.getRealMinVLen();
10886 const unsigned MinVLMAX = VectorBitsMin / EltSize;
10887 MVT SmallerVT;
10888 if (MaxIdx < MinVLMAX)
10889 SmallerVT = RISCVTargetLowering::getM1VT(VT: VecVT);
10890 else if (MaxIdx < MinVLMAX * 2)
10891 SmallerVT =
10892 RISCVTargetLowering::getM1VT(VT: VecVT).getDoubleNumVectorElementsVT();
10893 else if (MaxIdx < MinVLMAX * 4)
10894 SmallerVT = RISCVTargetLowering::getM1VT(VT: VecVT)
10895 .getDoubleNumVectorElementsVT()
10896 .getDoubleNumVectorElementsVT();
10897 if (!SmallerVT.isValid() || !VecVT.bitsGT(VT: SmallerVT))
10898 return std::nullopt;
10899 return SmallerVT;
10900}
10901
10902static bool isValidVisniInsertExtractIndex(SDValue Idx) {
10903 auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
10904 if (!IdxC || isNullConstant(V: Idx))
10905 return false;
10906 return isUInt<5>(x: IdxC->getZExtValue());
10907}
10908
10909// Custom-legalize INSERT_VECTOR_ELT so that the value is inserted into the
10910// first position of a vector, and that vector is slid up to the insert index.
10911// By limiting the active vector length to index+1 and merging with the
10912// original vector (with an undisturbed tail policy for elements >= VL), we
10913// achieve the desired result of leaving all elements untouched except the one
10914// at VL-1, which is replaced with the desired value.
10915SDValue RISCVTargetLowering::lowerINSERT_VECTOR_ELT(SDValue Op,
10916 SelectionDAG &DAG) const {
10917 SDLoc DL(Op);
10918 MVT VecVT = Op.getSimpleValueType();
10919 MVT XLenVT = Subtarget.getXLenVT();
10920 SDValue Vec = Op.getOperand(i: 0);
10921 SDValue Val = Op.getOperand(i: 1);
10922 MVT ValVT = Val.getSimpleValueType();
10923 SDValue Idx = Op.getOperand(i: 2);
10924
10925 if (VecVT.getVectorElementType() == MVT::i1) {
10926 // FIXME: For now we just promote to an i8 vector and insert into that,
10927 // but this is probably not optimal.
10928 MVT WideVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
10929 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Vec);
10930 Vec = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: WideVT, N1: Vec, N2: Val, N3: Idx);
10931 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: VecVT, Operand: Vec);
10932 }
10933
10934 if ((ValVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
10935 (ValVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
10936 // If we don't have vfmv.s.f for f16/bf16, use fmv.x.h first.
10937 MVT IntVT = VecVT.changeTypeToInteger();
10938 SDValue IntInsert = DAG.getNode(
10939 Opcode: ISD::INSERT_VECTOR_ELT, DL, VT: IntVT, N1: DAG.getBitcast(VT: IntVT, V: Vec),
10940 N2: DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Val), N3: Idx);
10941 return DAG.getBitcast(VT: VecVT, V: IntInsert);
10942 }
10943
10944 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
10945 auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
10946 if (!IdxC)
10947 return SDValue();
10948
10949 unsigned IdxVal = IdxC->getZExtValue();
10950 unsigned NumElts = VecVT.getVectorNumElements();
10951 MVT EltVT = VecVT.getVectorElementType();
10952 Vec = DAG.getBitcast(VT: XLenVT, V: Vec);
10953 SDValue ExtVal = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Val);
10954
10955 // For 2-element vectors, BUILD_VECTOR is more efficient since it only needs
10956 // at most 2 instructions.
10957 if (NumElts == 2) {
10958 unsigned EltBits = EltVT.getSizeInBits();
10959 SDValue Elt0, Elt1;
10960 if (IdxVal == 0) {
10961 Elt0 = ExtVal;
10962 Elt1 = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Vec,
10963 N2: DAG.getConstant(Val: EltBits, DL, VT: XLenVT));
10964 } else {
10965 Elt0 = Vec;
10966 Elt1 = ExtVal;
10967 }
10968 return DAG.getNode(Opcode: ISD::BUILD_VECTOR, DL, VT: VecVT, N1: Elt0, N2: Elt1);
10969 }
10970
10971 // For 4/8-element vectors, use MVM(or MERGE) instruction which does bitwise
10972 // select: rd = (~mask & rd) | (mask & rs1).
10973 // This generates: slli + lui/li + mvm
10974 if (NumElts == 4 || NumElts == 8) {
10975 unsigned EltBits = EltVT.getSizeInBits();
10976 unsigned ShiftAmt = IdxVal * EltBits;
10977 uint64_t PosMask = ((1ULL << EltBits) - 1) << ShiftAmt;
10978
10979 SDValue ShiftedVal = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: ExtVal,
10980 N2: DAG.getConstant(Val: ShiftAmt, DL, VT: XLenVT));
10981 SDValue Mask = DAG.getConstant(Val: PosMask, DL, VT: XLenVT);
10982 SDValue Result =
10983 DAG.getNode(Opcode: RISCVISD::MERGE, DL, VT: XLenVT, N1: Mask, N2: Vec, N3: ShiftedVal);
10984 return DAG.getBitcast(VT: VecVT, V: Result);
10985 }
10986
10987 return SDValue();
10988 }
10989
10990 MVT ContainerVT = VecVT;
10991 // If the operand is a fixed-length vector, convert to a scalable one.
10992 if (VecVT.isFixedLengthVector()) {
10993 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
10994 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
10995 }
10996
10997 // If we know the index we're going to insert at, we can shrink Vec so that
10998 // we're performing the scalar inserts and slideup on a smaller LMUL.
10999 SDValue OrigVec = Vec;
11000 std::optional<unsigned> AlignedIdx;
11001 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx)) {
11002 const unsigned OrigIdx = IdxC->getZExtValue();
11003 // Do we know an upper bound on LMUL?
11004 if (auto ShrunkVT = getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: OrigIdx,
11005 DL, DAG, Subtarget)) {
11006 ContainerVT = *ShrunkVT;
11007 AlignedIdx = 0;
11008 }
11009
11010 // If we're compiling for an exact VLEN value, we can always perform
11011 // the insert in m1 as we can determine the register corresponding to
11012 // the index in the register group.
11013 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
11014 if (auto VLEN = Subtarget.getRealVLen(); VLEN && ContainerVT.bitsGT(VT: M1VT)) {
11015 EVT ElemVT = VecVT.getVectorElementType();
11016 unsigned ElemsPerVReg = *VLEN / ElemVT.getFixedSizeInBits();
11017 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11018 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11019 AlignedIdx = SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11020 Idx = DAG.getVectorIdxConstant(Val: RemIdx, DL);
11021 ContainerVT = M1VT;
11022 }
11023
11024 if (AlignedIdx)
11025 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: *AlignedIdx);
11026 }
11027
11028 bool IsLegalInsert = Subtarget.is64Bit() || Val.getValueType() != MVT::i64;
11029 // Even i64-element vectors on RV32 can be lowered without scalar
11030 // legalization if the most-significant 32 bits of the value are not affected
11031 // by the sign-extension of the lower 32 bits. This applies to i32 constants
11032 // and sign_extend of i32 values.
11033 if (!IsLegalInsert) {
11034 if (isa<ConstantSDNode>(Val)) {
11035 const auto *CVal = cast<ConstantSDNode>(Val);
11036 if (isInt<32>(x: CVal->getSExtValue())) {
11037 IsLegalInsert = true;
11038 Val = DAG.getSignedConstant(Val: CVal->getSExtValue(), DL, VT: MVT::i32);
11039 }
11040 } else if (Val.getOpcode() == ISD::SIGN_EXTEND &&
11041 Val.getOperand(i: 0).getValueType() == MVT::i32) {
11042 IsLegalInsert = true;
11043 Val = Val.getOperand(i: 0);
11044 }
11045 }
11046
11047 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11048
11049 SDValue ValInVec;
11050
11051 if (IsLegalInsert) {
11052 unsigned Opc =
11053 VecVT.isFloatingPoint() ? RISCVISD::VFMV_S_F_VL : RISCVISD::VMV_S_X_VL;
11054 if (isNullConstant(V: Idx)) {
11055 if (!VecVT.isFloatingPoint())
11056 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Val);
11057 Vec = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: Vec, N2: Val, N3: VL);
11058
11059 if (AlignedIdx)
11060 Vec = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: Vec, Idx: *AlignedIdx);
11061 if (!VecVT.isFixedLengthVector())
11062 return Vec;
11063 return convertFromScalableVector(VT: VecVT, V: Vec, DAG, Subtarget);
11064 }
11065
11066 // Use ri.vinsert.v.x if available.
11067 if (Subtarget.hasVendorXRivosVisni() && VecVT.isInteger() &&
11068 isValidVisniInsertExtractIndex(Idx)) {
11069 // Tail policy applies to elements past VLMAX (by assumption Idx < VLMAX)
11070 SDValue PolicyOp =
11071 DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
11072 Vec = DAG.getNode(Opcode: RISCVISD::RI_VINSERT_VL, DL, VT: ContainerVT, N1: Vec, N2: Val, N3: Idx,
11073 N4: VL, N5: PolicyOp);
11074 if (AlignedIdx)
11075 Vec = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: Vec, Idx: *AlignedIdx);
11076 if (!VecVT.isFixedLengthVector())
11077 return Vec;
11078 return convertFromScalableVector(VT: VecVT, V: Vec, DAG, Subtarget);
11079 }
11080
11081 ValInVec = lowerScalarInsert(Scalar: Val, VL, VT: ContainerVT, DL, DAG, Subtarget);
11082 } else {
11083 // On RV32, i64-element vectors must be specially handled to place the
11084 // value at element 0, by using two vslide1down instructions in sequence on
11085 // the i32 split lo/hi value. Use an equivalently-sized i32 vector for
11086 // this.
11087 SDValue ValLo, ValHi;
11088 std::tie(args&: ValLo, args&: ValHi) = DAG.SplitScalar(N: Val, DL, LoVT: MVT::i32, HiVT: MVT::i32);
11089 MVT I32ContainerVT =
11090 MVT::getVectorVT(VT: MVT::i32, EC: ContainerVT.getVectorElementCount() * 2);
11091 SDValue I32Mask =
11092 getDefaultScalableVLOps(VecVT: I32ContainerVT, DL, DAG, Subtarget).first;
11093 // Limit the active VL to two.
11094 SDValue InsertI64VL = DAG.getConstant(Val: 2, DL, VT: XLenVT);
11095 // If the Idx is 0 we can insert directly into the vector.
11096 if (isNullConstant(V: Idx)) {
11097 // First slide in the lo value, then the hi in above it. We use slide1down
11098 // to avoid the register group overlap constraint of vslide1up.
11099 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11100 N1: Vec, N2: Vec, N3: ValLo, N4: I32Mask, N5: InsertI64VL);
11101 // If the source vector is undef don't pass along the tail elements from
11102 // the previous slide1down.
11103 SDValue Tail = Vec.isUndef() ? Vec : ValInVec;
11104 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11105 N1: Tail, N2: ValInVec, N3: ValHi, N4: I32Mask, N5: InsertI64VL);
11106 // Bitcast back to the right container type.
11107 ValInVec = DAG.getBitcast(VT: ContainerVT, V: ValInVec);
11108
11109 if (AlignedIdx)
11110 ValInVec = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: ValInVec, Idx: *AlignedIdx);
11111 if (!VecVT.isFixedLengthVector())
11112 return ValInVec;
11113 return convertFromScalableVector(VT: VecVT, V: ValInVec, DAG, Subtarget);
11114 }
11115
11116 // First slide in the lo value, then the hi in above it. We use slide1down
11117 // to avoid the register group overlap constraint of vslide1up.
11118 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11119 N1: DAG.getUNDEF(VT: I32ContainerVT),
11120 N2: DAG.getUNDEF(VT: I32ContainerVT), N3: ValLo,
11121 N4: I32Mask, N5: InsertI64VL);
11122 ValInVec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32ContainerVT,
11123 N1: DAG.getUNDEF(VT: I32ContainerVT), N2: ValInVec, N3: ValHi,
11124 N4: I32Mask, N5: InsertI64VL);
11125 // Bitcast back to the right container type.
11126 ValInVec = DAG.getBitcast(VT: ContainerVT, V: ValInVec);
11127 }
11128
11129 // Now that the value is in a vector, slide it into position.
11130 SDValue InsertVL =
11131 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: Idx, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11132
11133 // Use tail agnostic policy if Idx is the last index of Vec.
11134 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
11135 if (VecVT.isFixedLengthVector() && isa<ConstantSDNode>(Val: Idx) &&
11136 Idx->getAsZExtVal() + 1 == VecVT.getVectorNumElements())
11137 Policy = RISCVVType::TAIL_AGNOSTIC;
11138 SDValue Slideup = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Vec, Op: ValInVec,
11139 Offset: Idx, Mask, VL: InsertVL, Policy);
11140
11141 if (AlignedIdx)
11142 Slideup = DAG.getInsertSubvector(DL, Vec: OrigVec, SubVec: Slideup, Idx: *AlignedIdx);
11143 if (!VecVT.isFixedLengthVector())
11144 return Slideup;
11145 return convertFromScalableVector(VT: VecVT, V: Slideup, DAG, Subtarget);
11146}
11147
11148// Custom-lower EXTRACT_VECTOR_ELT operations to slide the vector down, then
11149// extract the first element: (extractelt (slidedown vec, idx), 0). For integer
11150// types this is done using VMV_X_S to allow us to glean information about the
11151// sign bits of the result.
11152SDValue RISCVTargetLowering::lowerEXTRACT_VECTOR_ELT(SDValue Op,
11153 SelectionDAG &DAG) const {
11154 SDLoc DL(Op);
11155 SDValue Idx = Op.getOperand(i: 1);
11156 SDValue Vec = Op.getOperand(i: 0);
11157 EVT EltVT = Op.getValueType();
11158 MVT VecVT = Vec.getSimpleValueType();
11159 MVT XLenVT = Subtarget.getXLenVT();
11160
11161 if (VecVT.getVectorElementType() == MVT::i1) {
11162 // Use vfirst.m to extract the first bit.
11163 if (isNullConstant(V: Idx)) {
11164 MVT ContainerVT = VecVT;
11165 if (VecVT.isFixedLengthVector()) {
11166 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11167 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11168 }
11169 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
11170 SDValue Vfirst =
11171 DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
11172 SDValue Res = DAG.getSetCC(DL, VT: XLenVT, LHS: Vfirst,
11173 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
11174 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
11175 }
11176 if (VecVT.isFixedLengthVector()) {
11177 unsigned NumElts = VecVT.getVectorNumElements();
11178 if (NumElts >= 8) {
11179 MVT WideEltVT;
11180 unsigned WidenVecLen;
11181 SDValue ExtractElementIdx;
11182 SDValue ExtractBitIdx;
11183 unsigned MaxEEW = Subtarget.getELen();
11184 MVT LargestEltVT = MVT::getIntegerVT(
11185 BitWidth: std::min(a: MaxEEW, b: unsigned(XLenVT.getSizeInBits())));
11186 if (NumElts <= LargestEltVT.getSizeInBits()) {
11187 assert(isPowerOf2_32(NumElts) &&
11188 "the number of elements should be power of 2");
11189 WideEltVT = MVT::getIntegerVT(BitWidth: NumElts);
11190 WidenVecLen = 1;
11191 ExtractElementIdx = DAG.getConstant(Val: 0, DL, VT: XLenVT);
11192 ExtractBitIdx = Idx;
11193 } else {
11194 WideEltVT = LargestEltVT;
11195 WidenVecLen = NumElts / WideEltVT.getSizeInBits();
11196 // extract element index = index / element width
11197 ExtractElementIdx = DAG.getNode(
11198 Opcode: ISD::SRL, DL, VT: XLenVT, N1: Idx,
11199 N2: DAG.getConstant(Val: Log2_64(Value: WideEltVT.getSizeInBits()), DL, VT: XLenVT));
11200 // mask bit index = index % element width
11201 ExtractBitIdx = DAG.getNode(
11202 Opcode: ISD::AND, DL, VT: XLenVT, N1: Idx,
11203 N2: DAG.getConstant(Val: WideEltVT.getSizeInBits() - 1, DL, VT: XLenVT));
11204 }
11205 MVT WideVT = MVT::getVectorVT(VT: WideEltVT, NumElements: WidenVecLen);
11206 Vec = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: WideVT, Operand: Vec);
11207 SDValue ExtractElt = DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: XLenVT,
11208 N1: Vec, N2: ExtractElementIdx);
11209 // Extract the bit from GPR.
11210 SDValue ShiftRight =
11211 DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: ExtractElt, N2: ExtractBitIdx);
11212 SDValue Res = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: ShiftRight,
11213 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11214 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Res);
11215 }
11216 }
11217 // Otherwise, promote to an i8 vector and extract from that.
11218 MVT WideVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
11219 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Vec);
11220 return DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: EltVT, N1: Vec, N2: Idx);
11221 }
11222
11223 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
11224 (EltVT == MVT::bf16 && !Subtarget.hasVInstructionsBF16())) {
11225 // If we don't have vfmv.f.s for f16/bf16, extract to a gpr then use fmv.h.x
11226 MVT IntVT = VecVT.changeTypeToInteger();
11227 SDValue IntVec = DAG.getBitcast(VT: IntVT, V: Vec);
11228 SDValue IntExtract =
11229 DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: XLenVT, N1: IntVec, N2: Idx);
11230 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT: EltVT, Operand: IntExtract);
11231 }
11232
11233 if (Subtarget.hasStdExtP() && VecVT.isFixedLengthVector()) {
11234 if (VecVT != MVT::v4i16 && VecVT != MVT::v2i16 && VecVT != MVT::v8i8 &&
11235 VecVT != MVT::v4i8 && VecVT != MVT::v2i32)
11236 return SDValue();
11237 SDValue Extracted = DAG.getBitcast(VT: XLenVT, V: Vec);
11238 unsigned ElemWidth = VecVT.getVectorElementType().getSizeInBits();
11239 SDValue Shamt = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Idx,
11240 N2: DAG.getConstant(Val: ElemWidth, DL, VT: XLenVT));
11241 return DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT, N1: Extracted, N2: Shamt);
11242 }
11243
11244 // If this is a fixed vector, we need to convert it to a scalable vector.
11245 MVT ContainerVT = VecVT;
11246 if (VecVT.isFixedLengthVector()) {
11247 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
11248 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
11249 }
11250
11251 // If we're compiling for an exact VLEN value and we have a known
11252 // constant index, we can always perform the extract in m1 (or
11253 // smaller) as we can determine the register corresponding to
11254 // the index in the register group.
11255 const auto VLen = Subtarget.getRealVLen();
11256 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx);
11257 IdxC && VLen && VecVT.getSizeInBits().getKnownMinValue() > *VLen) {
11258 MVT M1VT = RISCVTargetLowering::getM1VT(VT: ContainerVT);
11259 unsigned OrigIdx = IdxC->getZExtValue();
11260 EVT ElemVT = VecVT.getVectorElementType();
11261 unsigned ElemsPerVReg = *VLen / ElemVT.getFixedSizeInBits();
11262 unsigned RemIdx = OrigIdx % ElemsPerVReg;
11263 unsigned SubRegIdx = OrigIdx / ElemsPerVReg;
11264 unsigned ExtractIdx =
11265 SubRegIdx * M1VT.getVectorElementCount().getKnownMinValue();
11266 Vec = DAG.getExtractSubvector(DL, VT: M1VT, Vec, Idx: ExtractIdx);
11267 Idx = DAG.getVectorIdxConstant(Val: RemIdx, DL);
11268 ContainerVT = M1VT;
11269 }
11270
11271 // Reduce the LMUL of our slidedown and vmv.x.s to the smallest LMUL which
11272 // contains our index.
11273 std::optional<uint64_t> MaxIdx;
11274 if (VecVT.isFixedLengthVector())
11275 MaxIdx = VecVT.getVectorNumElements() - 1;
11276 if (auto *IdxC = dyn_cast<ConstantSDNode>(Val&: Idx))
11277 MaxIdx = IdxC->getZExtValue();
11278 if (MaxIdx) {
11279 if (auto SmallerVT =
11280 getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: *MaxIdx, DL, DAG, Subtarget)) {
11281 ContainerVT = *SmallerVT;
11282 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: 0);
11283 }
11284 }
11285
11286 // Use ri.vextract.x.v if available.
11287 // TODO: Avoid index 0 and just use the vmv.x.s
11288 if (Subtarget.hasVendorXRivosVisni() && EltVT.isInteger() &&
11289 isValidVisniInsertExtractIndex(Idx)) {
11290 SDValue Elt = DAG.getNode(Opcode: RISCVISD::RI_VEXTRACT, DL, VT: XLenVT, N1: Vec, N2: Idx);
11291 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Elt);
11292 }
11293
11294 // If after narrowing, the required slide is still greater than LMUL2,
11295 // fallback to generic expansion and go through the stack. This is done
11296 // for a subtle reason: extracting *all* elements out of a vector is
11297 // widely expected to be linear in vector size, but because vslidedown
11298 // is linear in LMUL, performing N extracts using vslidedown becomes
11299 // O(n^2) / (VLEN/ETYPE) work. On the surface, going through the stack
11300 // seems to have the same problem (the store is linear in LMUL), but the
11301 // generic expansion *memoizes* the store, and thus for many extracts of
11302 // the same vector we end up with one store and a bunch of loads.
11303 // TODO: We don't have the same code for insert_vector_elt because we
11304 // have BUILD_VECTOR and handle the degenerate case there. Should we
11305 // consider adding an inverse BUILD_VECTOR node?
11306 MVT LMUL2VT =
11307 RISCVTargetLowering::getM1VT(VT: ContainerVT).getDoubleNumVectorElementsVT();
11308 if (ContainerVT.bitsGT(VT: LMUL2VT) && VecVT.isFixedLengthVector())
11309 return SDValue();
11310
11311 // If the index is 0, the vector is already in the right position.
11312 if (!isNullConstant(V: Idx)) {
11313 // Use a VL of 1 to avoid processing more elements than we need.
11314 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT, DL, DAG, Subtarget);
11315 Vec = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
11316 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: Idx, Mask, VL);
11317 }
11318
11319 if (!EltVT.isInteger()) {
11320 // Floating-point extracts are handled in TableGen.
11321 return DAG.getExtractVectorElt(DL, VT: EltVT, Vec, Idx: 0);
11322 }
11323
11324 SDValue Elt0 = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
11325 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: EltVT, Operand: Elt0);
11326}
11327
11328// Some RVV intrinsics may claim that they want an integer operand to be
11329// promoted or expanded.
11330static SDValue lowerVectorIntrinsicScalars(SDValue Op, SelectionDAG &DAG,
11331 const RISCVSubtarget &Subtarget) {
11332 assert((Op.getOpcode() == ISD::INTRINSIC_VOID ||
11333 Op.getOpcode() == ISD::INTRINSIC_WO_CHAIN ||
11334 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN) &&
11335 "Unexpected opcode");
11336
11337 if (!Subtarget.hasVInstructions())
11338 return SDValue();
11339
11340 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
11341 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
11342 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
11343
11344 SDLoc DL(Op);
11345
11346 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
11347 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
11348 if (!II || !II->hasScalarOperand())
11349 return SDValue();
11350
11351 unsigned SplatOp = II->ScalarOperand + 1 + HasChain;
11352 assert(SplatOp < Op.getNumOperands());
11353
11354 SmallVector<SDValue, 8> Operands(Op->ops());
11355 SDValue &ScalarOp = Operands[SplatOp];
11356 MVT OpVT = ScalarOp.getSimpleValueType();
11357 MVT XLenVT = Subtarget.getXLenVT();
11358
11359 // If this isn't a scalar, or its type is XLenVT we're done.
11360 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
11361 return SDValue();
11362
11363 // Simplest case is that the operand needs to be promoted to XLenVT.
11364 if (OpVT.bitsLT(VT: XLenVT)) {
11365 // If the operand is a constant, sign extend to increase our chances
11366 // of being able to use a .vi instruction. ANY_EXTEND would become a
11367 // a zero extend and the simm5 check in isel would fail.
11368 // FIXME: Should we ignore the upper bits in isel instead?
11369 unsigned ExtOpc =
11370 isa<ConstantSDNode>(Val: ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
11371 ScalarOp = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: ScalarOp);
11372 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
11373 }
11374
11375 // Use the previous operand to get the vXi64 VT. The result might be a mask
11376 // VT for compares. Using the previous operand assumes that the previous
11377 // operand will never have a smaller element size than a scalar operand and
11378 // that a widening operation never uses SEW=64.
11379 // NOTE: If this fails the below assert, we can probably just find the
11380 // element count from any operand or result and use it to construct the VT.
11381 assert(II->ScalarOperand > 0 && "Unexpected splat operand!");
11382 MVT VT = Op.getOperand(i: SplatOp - 1).getSimpleValueType();
11383
11384 // The more complex case is when the scalar is larger than XLenVT.
11385 assert(XLenVT == MVT::i32 && OpVT == MVT::i64 &&
11386 VT.getVectorElementType() == MVT::i64 && "Unexpected VTs!");
11387
11388 // If this is a sign-extended 32-bit value, we can truncate it and rely on the
11389 // instruction to sign-extend since SEW>XLEN.
11390 if (DAG.ComputeNumSignBits(Op: ScalarOp) > 32) {
11391 ScalarOp = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: ScalarOp);
11392 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
11393 }
11394
11395 switch (IntNo) {
11396 case Intrinsic::riscv_vslide1up:
11397 case Intrinsic::riscv_vslide1down:
11398 case Intrinsic::riscv_vslide1up_mask:
11399 case Intrinsic::riscv_vslide1down_mask: {
11400 // We need to special case these when the scalar is larger than XLen.
11401 unsigned NumOps = Op.getNumOperands();
11402 bool IsMasked = NumOps == 7;
11403
11404 // Convert the vector source to the equivalent nxvXi32 vector.
11405 MVT I32VT = MVT::getVectorVT(VT: MVT::i32, EC: VT.getVectorElementCount() * 2);
11406 SDValue Vec = DAG.getBitcast(VT: I32VT, V: Operands[2]);
11407 SDValue ScalarLo, ScalarHi;
11408 std::tie(args&: ScalarLo, args&: ScalarHi) =
11409 DAG.SplitScalar(N: ScalarOp, DL, LoVT: MVT::i32, HiVT: MVT::i32);
11410
11411 // Double the VL since we halved SEW.
11412 SDValue AVL = getVLOperand(Op);
11413 SDValue I32VL;
11414
11415 // Optimize for constant AVL
11416 if (isa<ConstantSDNode>(Val: AVL)) {
11417 const auto [MinVLMAX, MaxVLMAX] =
11418 RISCVTargetLowering::computeVLMAXBounds(VecVT: VT, Subtarget);
11419
11420 uint64_t AVLInt = AVL->getAsZExtVal();
11421 if (AVLInt <= MinVLMAX) {
11422 I32VL = DAG.getConstant(Val: 2 * AVLInt, DL, VT: XLenVT);
11423 } else if (AVLInt >= 2 * MaxVLMAX) {
11424 // Just set vl to VLMAX in this situation
11425 I32VL = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
11426 } else {
11427 // For AVL between (MinVLMAX, 2 * MaxVLMAX), the actual working vl
11428 // is related to the hardware implementation.
11429 // So let the following code handle
11430 }
11431 }
11432 if (!I32VL) {
11433 RISCVVType::VLMUL Lmul = RISCVTargetLowering::getLMUL(VT);
11434 SDValue LMUL = DAG.getConstant(Val: Lmul, DL, VT: XLenVT);
11435 unsigned Sew = RISCVVType::encodeSEW(SEW: VT.getScalarSizeInBits());
11436 SDValue SEW = DAG.getConstant(Val: Sew, DL, VT: XLenVT);
11437 SDValue SETVL =
11438 DAG.getTargetConstant(Val: Intrinsic::riscv_vsetvli, DL, VT: MVT::i32);
11439 // Using vsetvli instruction to get actually used length which related to
11440 // the hardware implementation
11441 SDValue VL = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: XLenVT, N1: SETVL, N2: AVL,
11442 N3: SEW, N4: LMUL);
11443 I32VL =
11444 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: VL, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
11445 }
11446
11447 SDValue I32Mask = getAllOnesMask(VecVT: I32VT, VL: I32VL, DL, DAG);
11448
11449 // Shift the two scalar parts in using SEW=32 slide1up/slide1down
11450 // instructions.
11451 SDValue Passthru;
11452 if (IsMasked)
11453 Passthru = DAG.getUNDEF(VT: I32VT);
11454 else
11455 Passthru = DAG.getBitcast(VT: I32VT, V: Operands[1]);
11456
11457 if (IntNo == Intrinsic::riscv_vslide1up ||
11458 IntNo == Intrinsic::riscv_vslide1up_mask) {
11459 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1UP_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11460 N3: ScalarHi, N4: I32Mask, N5: I32VL);
11461 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1UP_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11462 N3: ScalarLo, N4: I32Mask, N5: I32VL);
11463 } else {
11464 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11465 N3: ScalarLo, N4: I32Mask, N5: I32VL);
11466 Vec = DAG.getNode(Opcode: RISCVISD::VSLIDE1DOWN_VL, DL, VT: I32VT, N1: Passthru, N2: Vec,
11467 N3: ScalarHi, N4: I32Mask, N5: I32VL);
11468 }
11469
11470 // Convert back to nxvXi64.
11471 Vec = DAG.getBitcast(VT, V: Vec);
11472
11473 if (!IsMasked)
11474 return Vec;
11475 // Apply mask after the operation.
11476 SDValue Mask = Operands[NumOps - 3];
11477 SDValue MaskedOff = Operands[1];
11478 // Assume Policy operand is the last operand.
11479 uint64_t Policy = Operands[NumOps - 1]->getAsZExtVal();
11480 // We don't need to select maskedoff if it's undef.
11481 if (MaskedOff.isUndef())
11482 return Vec;
11483 // TAMU
11484 if (Policy == RISCVVType::TAIL_AGNOSTIC)
11485 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: Mask, N2: Vec, N3: MaskedOff,
11486 N4: DAG.getUNDEF(VT), N5: AVL);
11487 // TUMA or TUMU: Currently we always emit tumu policy regardless of tuma.
11488 // It's fine because vmerge does not care mask policy.
11489 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: Mask, N2: Vec, N3: MaskedOff,
11490 N4: MaskedOff, N5: AVL);
11491 }
11492 }
11493
11494 // We need to convert the scalar to a splat vector.
11495 SDValue VL = getVLOperand(Op);
11496 assert(VL.getValueType() == XLenVT);
11497 ScalarOp = splatSplitI64WithVL(DL, VT, Passthru: SDValue(), Scalar: ScalarOp, VL, DAG);
11498 return DAG.getNode(Opcode: Op->getOpcode(), DL, VTList: Op->getVTList(), Ops: Operands);
11499}
11500
11501// Lower the llvm.get.vector.length intrinsic to vsetvli. We only support
11502// scalable vector llvm.get.vector.length for now.
11503//
11504// We need to convert from a scalable VF to a vsetvli with VLMax equal to
11505// (vscale * VF). The vscale and VF are independent of element width. We use
11506// SEW=8 for the vsetvli because it is the only element width that supports all
11507// fractional LMULs. The LMUL is chosen so that with SEW=8 the VLMax is
11508// (vscale * VF). Where vscale is defined as VLEN/RVVBitsPerBlock. The
11509// InsertVSETVLI pass can fix up the vtype of the vsetvli if a different
11510// SEW and LMUL are better for the surrounding vector instructions.
11511static SDValue lowerGetVectorLength(SDNode *N, SelectionDAG &DAG,
11512 const RISCVSubtarget &Subtarget) {
11513 MVT XLenVT = Subtarget.getXLenVT();
11514
11515 // The smallest LMUL is only valid for the smallest element width.
11516 const unsigned ElementWidth = 8;
11517
11518 // Determine the VF that corresponds to LMUL 1 for ElementWidth.
11519 unsigned LMul1VF = RISCV::RVVBitsPerBlock / ElementWidth;
11520 // We don't support VF==1 with ELEN==32.
11521 [[maybe_unused]] unsigned MinVF =
11522 RISCV::RVVBitsPerBlock / Subtarget.getELen();
11523
11524 [[maybe_unused]] unsigned VF = N->getConstantOperandVal(Num: 2);
11525 assert(VF >= MinVF && VF <= (LMul1VF * 8) && isPowerOf2_32(VF) &&
11526 "Unexpected VF");
11527
11528 bool Fractional = VF < LMul1VF;
11529 unsigned LMulVal = Fractional ? LMul1VF / VF : VF / LMul1VF;
11530 unsigned VLMUL = (unsigned)RISCVVType::encodeLMUL(LMUL: LMulVal, Fractional);
11531 unsigned VSEW = RISCVVType::encodeSEW(SEW: ElementWidth);
11532
11533 SDLoc DL(N);
11534
11535 SDValue LMul = DAG.getTargetConstant(Val: VLMUL, DL, VT: XLenVT);
11536 SDValue Sew = DAG.getTargetConstant(Val: VSEW, DL, VT: XLenVT);
11537
11538 SDValue AVL = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 1));
11539
11540 SDValue ID = DAG.getTargetConstant(Val: Intrinsic::riscv_vsetvli, DL, VT: XLenVT);
11541 SDValue Res =
11542 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: XLenVT, N1: ID, N2: AVL, N3: Sew, N4: LMul);
11543 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: Res);
11544}
11545
11546static SDValue lowerCttzElts(SDValue Op, SelectionDAG &DAG,
11547 const RISCVSubtarget &Subtarget) {
11548 SDValue Op0 = Op.getOperand(i: 0);
11549 MVT OpVT = Op0.getSimpleValueType();
11550 MVT ContainerVT = OpVT;
11551 if (OpVT.isFixedLengthVector()) {
11552 ContainerVT = getContainerForFixedLengthVector(DAG, VT: OpVT, Subtarget);
11553 Op0 = convertToScalableVector(VT: ContainerVT, V: Op0, DAG, Subtarget);
11554 }
11555 MVT XLenVT = Subtarget.getXLenVT();
11556 SDLoc DL(Op);
11557 auto [Mask, VL] = getDefaultVLOps(VecVT: OpVT, ContainerVT, DL, DAG, Subtarget);
11558 SDValue Res = DAG.getNode(Opcode: RISCVISD::VFIRST_VL, DL, VT: XLenVT, N1: Op0, N2: Mask, N3: VL);
11559 if (Op.getOpcode() == ISD::CTTZ_ELTS_ZERO_POISON)
11560 return Res;
11561
11562 // Convert -1 to VL.
11563 SDValue Setcc =
11564 DAG.getSetCC(DL, VT: XLenVT, LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETLT);
11565 VL = DAG.getElementCount(DL, VT: XLenVT, EC: OpVT.getVectorElementCount());
11566 return DAG.getSelect(DL, VT: XLenVT, Cond: Setcc, LHS: VL, RHS: Res);
11567}
11568
11569static inline void promoteVCIXScalar(SDValue Op,
11570 MutableArrayRef<SDValue> Operands,
11571 SelectionDAG &DAG) {
11572 const RISCVSubtarget &Subtarget =
11573 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
11574
11575 bool HasChain = Op.getOpcode() == ISD::INTRINSIC_VOID ||
11576 Op.getOpcode() == ISD::INTRINSIC_W_CHAIN;
11577 unsigned IntNo = Op.getConstantOperandVal(i: HasChain ? 1 : 0);
11578 SDLoc DL(Op);
11579
11580 const RISCVVIntrinsicsTable::RISCVVIntrinsicInfo *II =
11581 RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: IntNo);
11582 if (!II || !II->hasScalarOperand())
11583 return;
11584
11585 unsigned SplatOp = II->ScalarOperand + 1;
11586 assert(SplatOp < Op.getNumOperands());
11587
11588 SDValue &ScalarOp = Operands[SplatOp];
11589 MVT OpVT = ScalarOp.getSimpleValueType();
11590 MVT XLenVT = Subtarget.getXLenVT();
11591
11592 // The code below is partially copied from lowerVectorIntrinsicScalars.
11593 // If this isn't a scalar, or its type is XLenVT we're done.
11594 if (!OpVT.isScalarInteger() || OpVT == XLenVT)
11595 return;
11596
11597 // Manually emit promote operation for scalar operation.
11598 if (OpVT.bitsLT(VT: XLenVT)) {
11599 unsigned ExtOpc =
11600 isa<ConstantSDNode>(Val: ScalarOp) ? ISD::SIGN_EXTEND : ISD::ANY_EXTEND;
11601 ScalarOp = DAG.getNode(Opcode: ExtOpc, DL, VT: XLenVT, Operand: ScalarOp);
11602 }
11603}
11604
11605static void processVCIXOperands(SDValue OrigOp,
11606 MutableArrayRef<SDValue> Operands,
11607 SelectionDAG &DAG) {
11608 promoteVCIXScalar(Op: OrigOp, Operands, DAG);
11609 const RISCVSubtarget &Subtarget =
11610 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
11611 for (SDValue &V : Operands) {
11612 EVT ValType = V.getValueType();
11613 if (ValType.isVector() && ValType.isFloatingPoint()) {
11614 MVT InterimIVT =
11615 MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: ValType.getScalarSizeInBits()),
11616 EC: ValType.getVectorElementCount());
11617 V = DAG.getBitcast(VT: InterimIVT, V);
11618 }
11619 if (ValType.isFixedLengthVector()) {
11620 MVT OpContainerVT = getContainerForFixedLengthVector(
11621 DAG, VT: V.getSimpleValueType(), Subtarget);
11622 V = convertToScalableVector(VT: OpContainerVT, V, DAG, Subtarget);
11623 }
11624 }
11625}
11626
11627// LMUL * VLEN should be greater than or equal to EGS * SEW
11628static inline bool isValidEGW(int EGS, EVT VT,
11629 const RISCVSubtarget &Subtarget) {
11630 return (Subtarget.getRealMinVLen() *
11631 VT.getSizeInBits().getKnownMinValue()) / RISCV::RVVBitsPerBlock >=
11632 EGS * VT.getScalarSizeInBits();
11633}
11634
11635SDValue RISCVTargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op,
11636 SelectionDAG &DAG) const {
11637 unsigned IntNo = Op.getConstantOperandVal(i: 0);
11638 SDLoc DL(Op);
11639 MVT XLenVT = Subtarget.getXLenVT();
11640
11641 switch (IntNo) {
11642 default:
11643 break; // Don't custom lower most intrinsics.
11644 case Intrinsic::riscv_tuple_insert: {
11645 SDValue Vec = Op.getOperand(i: 1);
11646 SDValue SubVec = Op.getOperand(i: 2);
11647 SDValue Index = Op.getOperand(i: 3);
11648
11649 return DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: Op.getValueType(), N1: Vec,
11650 N2: SubVec, N3: Index);
11651 }
11652 case Intrinsic::riscv_tuple_extract: {
11653 SDValue Vec = Op.getOperand(i: 1);
11654 SDValue Index = Op.getOperand(i: 2);
11655
11656 return DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: Op.getValueType(), N1: Vec,
11657 N2: Index);
11658 }
11659 case Intrinsic::thread_pointer: {
11660 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
11661 return DAG.getRegister(Reg: RISCV::X4, VT: PtrVT);
11662 }
11663 case Intrinsic::riscv_orc_b:
11664 case Intrinsic::riscv_brev8:
11665 case Intrinsic::riscv_sha256sig0:
11666 case Intrinsic::riscv_sha256sig1:
11667 case Intrinsic::riscv_sha256sum0:
11668 case Intrinsic::riscv_sha256sum1:
11669 case Intrinsic::riscv_sm3p0:
11670 case Intrinsic::riscv_sm3p1: {
11671 unsigned Opc;
11672 switch (IntNo) {
11673 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
11674 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
11675 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
11676 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
11677 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
11678 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
11679 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
11680 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
11681 }
11682
11683 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
11684 }
11685 case Intrinsic::riscv_sm4ks:
11686 case Intrinsic::riscv_sm4ed: {
11687 unsigned Opc =
11688 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
11689
11690 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2),
11691 N3: Op.getOperand(i: 3));
11692 }
11693 case Intrinsic::riscv_zip:
11694 case Intrinsic::riscv_unzip: {
11695 unsigned Opc =
11696 IntNo == Intrinsic::riscv_zip ? RISCVISD::ZIP : RISCVISD::UNZIP;
11697 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
11698 }
11699 case Intrinsic::riscv_mopr:
11700 return DAG.getNode(Opcode: RISCVISD::MOP_R, DL, VT: XLenVT, N1: Op.getOperand(i: 1),
11701 N2: Op.getOperand(i: 2));
11702
11703 case Intrinsic::riscv_moprr: {
11704 return DAG.getNode(Opcode: RISCVISD::MOP_RR, DL, VT: XLenVT, N1: Op.getOperand(i: 1),
11705 N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11706 }
11707 case Intrinsic::riscv_clmulh:
11708 case Intrinsic::riscv_clmulr: {
11709 unsigned Opc = IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
11710 return DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2));
11711 }
11712 case Intrinsic::experimental_get_vector_length:
11713 return lowerGetVectorLength(N: Op.getNode(), DAG, Subtarget);
11714 case Intrinsic::riscv_vmv_x_s: {
11715 SDValue Res = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Op.getOperand(i: 1));
11716 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: Res);
11717 }
11718 case Intrinsic::riscv_vfmv_f_s:
11719 return DAG.getExtractVectorElt(DL, VT: Op.getValueType(), Vec: Op.getOperand(i: 1), Idx: 0);
11720 case Intrinsic::riscv_vmv_v_x:
11721 return lowerScalarSplat(Passthru: Op.getOperand(i: 1), Scalar: Op.getOperand(i: 2),
11722 VL: Op.getOperand(i: 3), VT: Op.getSimpleValueType(), DL, DAG,
11723 Subtarget);
11724 case Intrinsic::riscv_vfmv_v_f:
11725 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: Op.getValueType(),
11726 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11727 case Intrinsic::riscv_vmv_s_x: {
11728 SDValue Scalar = Op.getOperand(i: 2);
11729
11730 if (Scalar.getValueType().bitsLE(VT: XLenVT)) {
11731 Scalar = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: Scalar);
11732 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT: Op.getValueType(),
11733 N1: Op.getOperand(i: 1), N2: Scalar, N3: Op.getOperand(i: 3));
11734 }
11735
11736 assert(Scalar.getValueType() == MVT::i64 && "Unexpected scalar VT!");
11737
11738 // This is an i64 value that lives in two scalar registers. We have to
11739 // insert this in a convoluted way. First we build vXi64 splat containing
11740 // the two values that we assemble using some bit math. Next we'll use
11741 // vid.v and vmseq to build a mask with bit 0 set. Then we'll use that mask
11742 // to merge element 0 from our splat into the source vector.
11743 // FIXME: This is probably not the best way to do this, but it is
11744 // consistent with INSERT_VECTOR_ELT lowering so it is a good starting
11745 // point.
11746 // sw lo, (a0)
11747 // sw hi, 4(a0)
11748 // vlse vX, (a0)
11749 //
11750 // vid.v vVid
11751 // vmseq.vx mMask, vVid, 0
11752 // vmerge.vvm vDest, vSrc, vVal, mMask
11753 MVT VT = Op.getSimpleValueType();
11754 SDValue Vec = Op.getOperand(i: 1);
11755 SDValue VL = getVLOperand(Op);
11756
11757 SDValue SplattedVal = splatSplitI64WithVL(DL, VT, Passthru: SDValue(), Scalar, VL, DAG);
11758 if (Op.getOperand(i: 1).isUndef())
11759 return SplattedVal;
11760 SDValue SplattedIdx =
11761 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
11762 N2: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N3: VL);
11763
11764 MVT MaskVT = getMaskTypeFor(VecVT: VT);
11765 SDValue Mask = getAllOnesMask(VecVT: VT, VL, DL, DAG);
11766 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT, N1: Mask, N2: VL);
11767 SDValue SelectCond =
11768 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: MaskVT,
11769 Ops: {VID, SplattedIdx, DAG.getCondCode(Cond: ISD::SETEQ),
11770 DAG.getUNDEF(VT: MaskVT), Mask, VL});
11771 return DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT, N1: SelectCond, N2: SplattedVal,
11772 N3: Vec, N4: DAG.getUNDEF(VT), N5: VL);
11773 }
11774 case Intrinsic::riscv_vfmv_s_f:
11775 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT: Op.getValueType(),
11776 N1: Op.getOperand(i: 1), N2: Op.getOperand(i: 2), N3: Op.getOperand(i: 3));
11777 // EGS * EEW >= 128 bits
11778 case Intrinsic::riscv_vaesdf_vv:
11779 case Intrinsic::riscv_vaesdf_vs:
11780 case Intrinsic::riscv_vaesdm_vv:
11781 case Intrinsic::riscv_vaesdm_vs:
11782 case Intrinsic::riscv_vaesef_vv:
11783 case Intrinsic::riscv_vaesef_vs:
11784 case Intrinsic::riscv_vaesem_vv:
11785 case Intrinsic::riscv_vaesem_vs:
11786 case Intrinsic::riscv_vaeskf1:
11787 case Intrinsic::riscv_vaeskf2:
11788 case Intrinsic::riscv_vaesz_vs:
11789 case Intrinsic::riscv_vsm4k:
11790 case Intrinsic::riscv_vsm4r_vv:
11791 case Intrinsic::riscv_vsm4r_vs: {
11792 if (!isValidEGW(EGS: 4, VT: Op.getSimpleValueType(), Subtarget) ||
11793 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget) ||
11794 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 2).getSimpleValueType(), Subtarget))
11795 reportFatalUsageError(reason: "EGW should be greater than or equal to 4 * SEW.");
11796 return Op;
11797 }
11798 // EGS * EEW >= 256 bits
11799 case Intrinsic::riscv_vsm3c:
11800 case Intrinsic::riscv_vsm3me: {
11801 if (!isValidEGW(EGS: 8, VT: Op.getSimpleValueType(), Subtarget) ||
11802 !isValidEGW(EGS: 8, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget))
11803 reportFatalUsageError(reason: "EGW should be greater than or equal to 8 * SEW.");
11804 return Op;
11805 }
11806 // zvknha(SEW=32)/zvknhb(SEW=[32|64])
11807 case Intrinsic::riscv_vsha2ch:
11808 case Intrinsic::riscv_vsha2cl:
11809 case Intrinsic::riscv_vsha2ms: {
11810 if (Op->getSimpleValueType(ResNo: 0).getScalarSizeInBits() == 64 &&
11811 !Subtarget.hasStdExtZvknhb())
11812 reportFatalUsageError(reason: "SEW=64 needs Zvknhb to be enabled.");
11813 if (!isValidEGW(EGS: 4, VT: Op.getSimpleValueType(), Subtarget) ||
11814 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 1).getSimpleValueType(), Subtarget) ||
11815 !isValidEGW(EGS: 4, VT: Op->getOperand(Num: 2).getSimpleValueType(), Subtarget))
11816 reportFatalUsageError(reason: "EGW should be greater than or equal to 4 * SEW.");
11817 return Op;
11818 }
11819 case Intrinsic::riscv_sf_vc_v_x:
11820 case Intrinsic::riscv_sf_vc_v_i:
11821 case Intrinsic::riscv_sf_vc_v_xv:
11822 case Intrinsic::riscv_sf_vc_v_iv:
11823 case Intrinsic::riscv_sf_vc_v_vv:
11824 case Intrinsic::riscv_sf_vc_v_fv:
11825 case Intrinsic::riscv_sf_vc_v_xvv:
11826 case Intrinsic::riscv_sf_vc_v_ivv:
11827 case Intrinsic::riscv_sf_vc_v_vvv:
11828 case Intrinsic::riscv_sf_vc_v_fvv:
11829 case Intrinsic::riscv_sf_vc_v_xvw:
11830 case Intrinsic::riscv_sf_vc_v_ivw:
11831 case Intrinsic::riscv_sf_vc_v_vvw:
11832 case Intrinsic::riscv_sf_vc_v_fvw: {
11833 MVT VT = Op.getSimpleValueType();
11834
11835 SmallVector<SDValue> Operands{Op->op_values()};
11836 processVCIXOperands(OrigOp: Op, Operands, DAG);
11837
11838 MVT RetVT = VT;
11839 if (VT.isFixedLengthVector())
11840 RetVT = getContainerForFixedLengthVector(VT);
11841 else if (VT.isFloatingPoint())
11842 RetVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits()),
11843 EC: VT.getVectorElementCount());
11844
11845 SDValue NewNode = DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: RetVT, Ops: Operands);
11846
11847 if (VT.isFixedLengthVector())
11848 NewNode = convertFromScalableVector(VT, V: NewNode, DAG, Subtarget);
11849 else if (VT.isFloatingPoint())
11850 NewNode = DAG.getBitcast(VT, V: NewNode);
11851
11852 if (Op == NewNode)
11853 break;
11854
11855 return NewNode;
11856 }
11857 }
11858
11859 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
11860}
11861
11862static inline SDValue getVCIXISDNodeWCHAIN(SDValue Op, SelectionDAG &DAG,
11863 unsigned Type) {
11864 SDLoc DL(Op);
11865 SmallVector<SDValue> Operands{Op->op_values()};
11866 Operands.erase(CI: Operands.begin() + 1);
11867
11868 const RISCVSubtarget &Subtarget =
11869 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>();
11870 MVT VT = Op.getSimpleValueType();
11871 MVT RetVT = VT;
11872 MVT FloatVT = VT;
11873
11874 if (VT.isFloatingPoint()) {
11875 RetVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: VT.getScalarSizeInBits()),
11876 EC: VT.getVectorElementCount());
11877 FloatVT = RetVT;
11878 }
11879 if (VT.isFixedLengthVector())
11880 RetVT = getContainerForFixedLengthVector(TLI: DAG.getTargetLoweringInfo(), VT: RetVT,
11881 Subtarget);
11882
11883 processVCIXOperands(OrigOp: Op, Operands, DAG);
11884
11885 SDVTList VTs = DAG.getVTList(VTs: {RetVT, MVT::Other});
11886 SDValue NewNode = DAG.getNode(Opcode: Type, DL, VTList: VTs, Ops: Operands);
11887 SDValue Chain = NewNode.getValue(R: 1);
11888
11889 if (VT.isFixedLengthVector())
11890 NewNode = convertFromScalableVector(VT: FloatVT, V: NewNode, DAG, Subtarget);
11891 if (VT.isFloatingPoint())
11892 NewNode = DAG.getBitcast(VT, V: NewNode);
11893
11894 NewNode = DAG.getMergeValues(Ops: {NewNode, Chain}, dl: DL);
11895
11896 return NewNode;
11897}
11898
11899static inline SDValue getVCIXISDNodeVOID(SDValue Op, SelectionDAG &DAG,
11900 unsigned Type) {
11901 SmallVector<SDValue> Operands{Op->op_values()};
11902 Operands.erase(CI: Operands.begin() + 1);
11903 processVCIXOperands(OrigOp: Op, Operands, DAG);
11904
11905 return DAG.getNode(Opcode: Type, DL: SDLoc(Op), VT: Op.getValueType(), Ops: Operands);
11906}
11907
11908static SDValue
11909lowerFixedVectorSegLoadIntrinsics(unsigned IntNo, SDValue Op,
11910 const RISCVSubtarget &Subtarget,
11911 SelectionDAG &DAG) {
11912 bool IsStrided;
11913 switch (IntNo) {
11914 case Intrinsic::riscv_seg2_load_mask:
11915 case Intrinsic::riscv_seg3_load_mask:
11916 case Intrinsic::riscv_seg4_load_mask:
11917 case Intrinsic::riscv_seg5_load_mask:
11918 case Intrinsic::riscv_seg6_load_mask:
11919 case Intrinsic::riscv_seg7_load_mask:
11920 case Intrinsic::riscv_seg8_load_mask:
11921 IsStrided = false;
11922 break;
11923 case Intrinsic::riscv_sseg2_load_mask:
11924 case Intrinsic::riscv_sseg3_load_mask:
11925 case Intrinsic::riscv_sseg4_load_mask:
11926 case Intrinsic::riscv_sseg5_load_mask:
11927 case Intrinsic::riscv_sseg6_load_mask:
11928 case Intrinsic::riscv_sseg7_load_mask:
11929 case Intrinsic::riscv_sseg8_load_mask:
11930 IsStrided = true;
11931 break;
11932 default:
11933 llvm_unreachable("unexpected intrinsic ID");
11934 };
11935
11936 static const Intrinsic::ID VlsegInts[7] = {
11937 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
11938 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
11939 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
11940 Intrinsic::riscv_vlseg8_mask};
11941 static const Intrinsic::ID VlssegInts[7] = {
11942 Intrinsic::riscv_vlsseg2_mask, Intrinsic::riscv_vlsseg3_mask,
11943 Intrinsic::riscv_vlsseg4_mask, Intrinsic::riscv_vlsseg5_mask,
11944 Intrinsic::riscv_vlsseg6_mask, Intrinsic::riscv_vlsseg7_mask,
11945 Intrinsic::riscv_vlsseg8_mask};
11946
11947 SDLoc DL(Op);
11948 unsigned NF = Op->getNumValues() - 1;
11949 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
11950 MVT XLenVT = Subtarget.getXLenVT();
11951 MVT VT = Op->getSimpleValueType(ResNo: 0);
11952 MVT ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
11953 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
11954 ContainerVT.getScalarSizeInBits();
11955 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: NF);
11956
11957 // Operands: (chain, int_id, pointer, mask, vl) or
11958 // (chain, int_id, pointer, offset, mask, vl)
11959 SDValue VL = Op.getOperand(i: Op.getNumOperands() - 1);
11960 SDValue Mask = Op.getOperand(i: Op.getNumOperands() - 2);
11961 MVT MaskVT = Mask.getSimpleValueType();
11962 MVT MaskContainerVT =
11963 ::getContainerForFixedLengthVector(DAG, VT: MaskVT, Subtarget);
11964 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
11965
11966 SDValue IntID = DAG.getTargetConstant(
11967 Val: IsStrided ? VlssegInts[NF - 2] : VlsegInts[NF - 2], DL, VT: XLenVT);
11968 auto *Load = cast<MemIntrinsicSDNode>(Val&: Op);
11969
11970 SDVTList VTs = DAG.getVTList(VTs: {VecTupTy, MVT::Other});
11971 SmallVector<SDValue, 9> Ops = {
11972 Load->getChain(),
11973 IntID,
11974 DAG.getUNDEF(VT: VecTupTy),
11975 Op.getOperand(i: 2),
11976 Mask,
11977 VL,
11978 DAG.getTargetConstant(
11979 Val: RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, VT: XLenVT),
11980 DAG.getTargetConstant(Val: Log2_64(Value: VT.getScalarSizeInBits()), DL, VT: XLenVT)};
11981 // Insert the stride operand.
11982 if (IsStrided)
11983 Ops.insert(I: std::next(x: Ops.begin(), n: 4), Elt: Op.getOperand(i: 3));
11984
11985 SDValue Result =
11986 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
11987 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
11988 SmallVector<SDValue, 9> Results;
11989 for (unsigned int RetIdx = 0; RetIdx < NF; RetIdx++) {
11990 SDValue SubVec = DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: ContainerVT,
11991 N1: Result.getValue(R: 0),
11992 N2: DAG.getTargetConstant(Val: RetIdx, DL, VT: MVT::i32));
11993 Results.push_back(Elt: convertFromScalableVector(VT, V: SubVec, DAG, Subtarget));
11994 }
11995 Results.push_back(Elt: Result.getValue(R: 1));
11996 return DAG.getMergeValues(Ops: Results, dl: DL);
11997}
11998
11999SDValue RISCVTargetLowering::LowerINTRINSIC_W_CHAIN(SDValue Op,
12000 SelectionDAG &DAG) const {
12001 unsigned IntNo = Op.getConstantOperandVal(i: 1);
12002 switch (IntNo) {
12003 default:
12004 break;
12005 case Intrinsic::riscv_seg2_load_mask:
12006 case Intrinsic::riscv_seg3_load_mask:
12007 case Intrinsic::riscv_seg4_load_mask:
12008 case Intrinsic::riscv_seg5_load_mask:
12009 case Intrinsic::riscv_seg6_load_mask:
12010 case Intrinsic::riscv_seg7_load_mask:
12011 case Intrinsic::riscv_seg8_load_mask:
12012 case Intrinsic::riscv_sseg2_load_mask:
12013 case Intrinsic::riscv_sseg3_load_mask:
12014 case Intrinsic::riscv_sseg4_load_mask:
12015 case Intrinsic::riscv_sseg5_load_mask:
12016 case Intrinsic::riscv_sseg6_load_mask:
12017 case Intrinsic::riscv_sseg7_load_mask:
12018 case Intrinsic::riscv_sseg8_load_mask:
12019 return lowerFixedVectorSegLoadIntrinsics(IntNo, Op, Subtarget, DAG);
12020
12021 case Intrinsic::riscv_sf_vc_v_x_se:
12022 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_X_SE);
12023 case Intrinsic::riscv_sf_vc_v_i_se:
12024 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_I_SE);
12025 case Intrinsic::riscv_sf_vc_v_xv_se:
12026 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XV_SE);
12027 case Intrinsic::riscv_sf_vc_v_iv_se:
12028 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IV_SE);
12029 case Intrinsic::riscv_sf_vc_v_vv_se:
12030 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VV_SE);
12031 case Intrinsic::riscv_sf_vc_v_fv_se:
12032 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FV_SE);
12033 case Intrinsic::riscv_sf_vc_v_xvv_se:
12034 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XVV_SE);
12035 case Intrinsic::riscv_sf_vc_v_ivv_se:
12036 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IVV_SE);
12037 case Intrinsic::riscv_sf_vc_v_vvv_se:
12038 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VVV_SE);
12039 case Intrinsic::riscv_sf_vc_v_fvv_se:
12040 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FVV_SE);
12041 case Intrinsic::riscv_sf_vc_v_xvw_se:
12042 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_XVW_SE);
12043 case Intrinsic::riscv_sf_vc_v_ivw_se:
12044 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_IVW_SE);
12045 case Intrinsic::riscv_sf_vc_v_vvw_se:
12046 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_VVW_SE);
12047 case Intrinsic::riscv_sf_vc_v_fvw_se:
12048 return getVCIXISDNodeWCHAIN(Op, DAG, Type: RISCVISD::SF_VC_V_FVW_SE);
12049 }
12050
12051 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12052}
12053
12054static SDValue
12055lowerFixedVectorSegStoreIntrinsics(unsigned IntNo, SDValue Op,
12056 const RISCVSubtarget &Subtarget,
12057 SelectionDAG &DAG) {
12058 bool IsStrided;
12059 switch (IntNo) {
12060 case Intrinsic::riscv_seg2_store_mask:
12061 case Intrinsic::riscv_seg3_store_mask:
12062 case Intrinsic::riscv_seg4_store_mask:
12063 case Intrinsic::riscv_seg5_store_mask:
12064 case Intrinsic::riscv_seg6_store_mask:
12065 case Intrinsic::riscv_seg7_store_mask:
12066 case Intrinsic::riscv_seg8_store_mask:
12067 IsStrided = false;
12068 break;
12069 case Intrinsic::riscv_sseg2_store_mask:
12070 case Intrinsic::riscv_sseg3_store_mask:
12071 case Intrinsic::riscv_sseg4_store_mask:
12072 case Intrinsic::riscv_sseg5_store_mask:
12073 case Intrinsic::riscv_sseg6_store_mask:
12074 case Intrinsic::riscv_sseg7_store_mask:
12075 case Intrinsic::riscv_sseg8_store_mask:
12076 IsStrided = true;
12077 break;
12078 default:
12079 llvm_unreachable("unexpected intrinsic ID");
12080 }
12081
12082 SDLoc DL(Op);
12083 static const Intrinsic::ID VssegInts[] = {
12084 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
12085 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
12086 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
12087 Intrinsic::riscv_vsseg8_mask};
12088 static const Intrinsic::ID VsssegInts[] = {
12089 Intrinsic::riscv_vssseg2_mask, Intrinsic::riscv_vssseg3_mask,
12090 Intrinsic::riscv_vssseg4_mask, Intrinsic::riscv_vssseg5_mask,
12091 Intrinsic::riscv_vssseg6_mask, Intrinsic::riscv_vssseg7_mask,
12092 Intrinsic::riscv_vssseg8_mask};
12093
12094 // Operands: (chain, int_id, vec*, ptr, mask, vl) or
12095 // (chain, int_id, vec*, ptr, stride, mask, vl)
12096 unsigned NF = Op->getNumOperands() - (IsStrided ? 6 : 5);
12097 assert(NF >= 2 && NF <= 8 && "Unexpected seg number");
12098 MVT XLenVT = Subtarget.getXLenVT();
12099 MVT VT = Op->getOperand(Num: 2).getSimpleValueType();
12100 MVT ContainerVT = ::getContainerForFixedLengthVector(DAG, VT, Subtarget);
12101 unsigned Sz = NF * ContainerVT.getVectorMinNumElements() *
12102 ContainerVT.getScalarSizeInBits();
12103 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: NF);
12104
12105 SDValue VL = Op.getOperand(i: Op.getNumOperands() - 1);
12106 SDValue Mask = Op.getOperand(i: Op.getNumOperands() - 2);
12107 MVT MaskVT = Mask.getSimpleValueType();
12108 MVT MaskContainerVT =
12109 ::getContainerForFixedLengthVector(DAG, VT: MaskVT, Subtarget);
12110 Mask = convertToScalableVector(VT: MaskContainerVT, V: Mask, DAG, Subtarget);
12111
12112 SDValue IntID = DAG.getTargetConstant(
12113 Val: IsStrided ? VsssegInts[NF - 2] : VssegInts[NF - 2], DL, VT: XLenVT);
12114 SDValue Ptr = Op->getOperand(Num: NF + 2);
12115
12116 auto *FixedIntrinsic = cast<MemIntrinsicSDNode>(Val&: Op);
12117
12118 SDValue StoredVal = DAG.getUNDEF(VT: VecTupTy);
12119 for (unsigned i = 0; i < NF; i++)
12120 StoredVal = DAG.getNode(
12121 Opcode: RISCVISD::TUPLE_INSERT, DL, VT: VecTupTy, N1: StoredVal,
12122 N2: convertToScalableVector(VT: ContainerVT, V: FixedIntrinsic->getOperand(Num: 2 + i),
12123 DAG, Subtarget),
12124 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
12125
12126 SmallVector<SDValue, 10> Ops = {
12127 FixedIntrinsic->getChain(),
12128 IntID,
12129 StoredVal,
12130 Ptr,
12131 Mask,
12132 VL,
12133 DAG.getTargetConstant(Val: Log2_64(Value: VT.getScalarSizeInBits()), DL, VT: XLenVT)};
12134 // Insert the stride operand.
12135 if (IsStrided)
12136 Ops.insert(I: std::next(x: Ops.begin(), n: 4),
12137 Elt: Op.getOperand(i: Op.getNumOperands() - 3));
12138
12139 return DAG.getMemIntrinsicNode(
12140 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
12141 MemVT: FixedIntrinsic->getMemoryVT(), MMO: FixedIntrinsic->getMemOperand());
12142}
12143
12144SDValue RISCVTargetLowering::LowerINTRINSIC_VOID(SDValue Op,
12145 SelectionDAG &DAG) const {
12146 unsigned IntNo = Op.getConstantOperandVal(i: 1);
12147 switch (IntNo) {
12148 default:
12149 break;
12150 case Intrinsic::riscv_seg2_store_mask:
12151 case Intrinsic::riscv_seg3_store_mask:
12152 case Intrinsic::riscv_seg4_store_mask:
12153 case Intrinsic::riscv_seg5_store_mask:
12154 case Intrinsic::riscv_seg6_store_mask:
12155 case Intrinsic::riscv_seg7_store_mask:
12156 case Intrinsic::riscv_seg8_store_mask:
12157 case Intrinsic::riscv_sseg2_store_mask:
12158 case Intrinsic::riscv_sseg3_store_mask:
12159 case Intrinsic::riscv_sseg4_store_mask:
12160 case Intrinsic::riscv_sseg5_store_mask:
12161 case Intrinsic::riscv_sseg6_store_mask:
12162 case Intrinsic::riscv_sseg7_store_mask:
12163 case Intrinsic::riscv_sseg8_store_mask:
12164 return lowerFixedVectorSegStoreIntrinsics(IntNo, Op, Subtarget, DAG);
12165
12166 case Intrinsic::riscv_sf_vc_xv_se:
12167 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XV_SE);
12168 case Intrinsic::riscv_sf_vc_iv_se:
12169 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IV_SE);
12170 case Intrinsic::riscv_sf_vc_vv_se:
12171 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VV_SE);
12172 case Intrinsic::riscv_sf_vc_fv_se:
12173 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FV_SE);
12174 case Intrinsic::riscv_sf_vc_xvv_se:
12175 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XVV_SE);
12176 case Intrinsic::riscv_sf_vc_ivv_se:
12177 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IVV_SE);
12178 case Intrinsic::riscv_sf_vc_vvv_se:
12179 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VVV_SE);
12180 case Intrinsic::riscv_sf_vc_fvv_se:
12181 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FVV_SE);
12182 case Intrinsic::riscv_sf_vc_xvw_se:
12183 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_XVW_SE);
12184 case Intrinsic::riscv_sf_vc_ivw_se:
12185 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_IVW_SE);
12186 case Intrinsic::riscv_sf_vc_vvw_se:
12187 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_VVW_SE);
12188 case Intrinsic::riscv_sf_vc_fvw_se:
12189 return getVCIXISDNodeVOID(Op, DAG, Type: RISCVISD::SF_VC_FVW_SE);
12190 }
12191
12192 return lowerVectorIntrinsicScalars(Op, DAG, Subtarget);
12193}
12194
12195static unsigned getRVVReductionOp(unsigned ISDOpcode) {
12196 switch (ISDOpcode) {
12197 default:
12198 llvm_unreachable("Unhandled reduction");
12199 case ISD::VP_REDUCE_ADD:
12200 case ISD::VECREDUCE_ADD:
12201 return RISCVISD::VECREDUCE_ADD_VL;
12202 case ISD::VP_REDUCE_UMAX:
12203 case ISD::VECREDUCE_UMAX:
12204 return RISCVISD::VECREDUCE_UMAX_VL;
12205 case ISD::VP_REDUCE_SMAX:
12206 case ISD::VECREDUCE_SMAX:
12207 return RISCVISD::VECREDUCE_SMAX_VL;
12208 case ISD::VP_REDUCE_UMIN:
12209 case ISD::VECREDUCE_UMIN:
12210 return RISCVISD::VECREDUCE_UMIN_VL;
12211 case ISD::VP_REDUCE_SMIN:
12212 case ISD::VECREDUCE_SMIN:
12213 return RISCVISD::VECREDUCE_SMIN_VL;
12214 case ISD::VP_REDUCE_AND:
12215 case ISD::VECREDUCE_AND:
12216 return RISCVISD::VECREDUCE_AND_VL;
12217 case ISD::VP_REDUCE_OR:
12218 case ISD::VECREDUCE_OR:
12219 return RISCVISD::VECREDUCE_OR_VL;
12220 case ISD::VP_REDUCE_XOR:
12221 case ISD::VECREDUCE_XOR:
12222 return RISCVISD::VECREDUCE_XOR_VL;
12223 case ISD::VP_REDUCE_FADD:
12224 return RISCVISD::VECREDUCE_FADD_VL;
12225 case ISD::VP_REDUCE_SEQ_FADD:
12226 return RISCVISD::VECREDUCE_SEQ_FADD_VL;
12227 case ISD::VP_REDUCE_FMAX:
12228 case ISD::VP_REDUCE_FMAXIMUM:
12229 return RISCVISD::VECREDUCE_FMAX_VL;
12230 case ISD::VP_REDUCE_FMIN:
12231 case ISD::VP_REDUCE_FMINIMUM:
12232 return RISCVISD::VECREDUCE_FMIN_VL;
12233 }
12234
12235}
12236
12237SDValue RISCVTargetLowering::lowerVectorMaskVecReduction(SDValue Op,
12238 SelectionDAG &DAG,
12239 bool IsVP) const {
12240 SDLoc DL(Op);
12241 SDValue Vec = Op.getOperand(i: IsVP ? 1 : 0);
12242 MVT VecVT = Vec.getSimpleValueType();
12243 assert((Op.getOpcode() == ISD::VECREDUCE_AND ||
12244 Op.getOpcode() == ISD::VECREDUCE_OR ||
12245 Op.getOpcode() == ISD::VECREDUCE_XOR ||
12246 Op.getOpcode() == ISD::VP_REDUCE_AND ||
12247 Op.getOpcode() == ISD::VP_REDUCE_OR ||
12248 Op.getOpcode() == ISD::VP_REDUCE_XOR) &&
12249 "Unexpected reduction lowering");
12250
12251 MVT XLenVT = Subtarget.getXLenVT();
12252
12253 MVT ContainerVT = VecVT;
12254 if (VecVT.isFixedLengthVector()) {
12255 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12256 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12257 }
12258
12259 SDValue Mask, VL;
12260 if (IsVP) {
12261 Mask = Op.getOperand(i: 2);
12262 VL = Op.getOperand(i: 3);
12263 } else {
12264 std::tie(args&: Mask, args&: VL) =
12265 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12266 }
12267
12268 ISD::CondCode CC;
12269 switch (Op.getOpcode()) {
12270 default:
12271 llvm_unreachable("Unhandled reduction");
12272 case ISD::VECREDUCE_AND:
12273 case ISD::VP_REDUCE_AND: {
12274 // vcpop ~x == 0
12275 SDValue TrueMask = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
12276 if (IsVP || VecVT.isFixedLengthVector())
12277 Vec = DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Vec, N2: TrueMask, N3: VL);
12278 else
12279 Vec = DAG.getNode(Opcode: ISD::XOR, DL, VT: ContainerVT, N1: Vec, N2: TrueMask);
12280 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
12281 CC = ISD::SETEQ;
12282 break;
12283 }
12284 case ISD::VECREDUCE_OR:
12285 case ISD::VP_REDUCE_OR:
12286 // vcpop x != 0
12287 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
12288 CC = ISD::SETNE;
12289 break;
12290 case ISD::VECREDUCE_XOR:
12291 case ISD::VP_REDUCE_XOR: {
12292 // ((vcpop x) & 1) != 0
12293 SDValue One = DAG.getConstant(Val: 1, DL, VT: XLenVT);
12294 Vec = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Vec, N2: Mask, N3: VL);
12295 Vec = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Vec, N2: One);
12296 CC = ISD::SETNE;
12297 break;
12298 }
12299 }
12300
12301 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
12302 SDValue SetCC = DAG.getSetCC(DL, VT: XLenVT, LHS: Vec, RHS: Zero, Cond: CC);
12303 SetCC = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: Op.getValueType(), Operand: SetCC);
12304
12305 if (!IsVP)
12306 return SetCC;
12307
12308 // Now include the start value in the operation.
12309 // Note that we must return the start value when no elements are operated
12310 // upon. The vcpop instructions we've emitted in each case above will return
12311 // 0 for an inactive vector, and so we've already received the neutral value:
12312 // AND gives us (0 == 0) -> 1 and OR/XOR give us (0 != 0) -> 0. Therefore we
12313 // can simply include the start value.
12314 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
12315 return DAG.getNode(Opcode: BaseOpc, DL, VT: Op.getValueType(), N1: SetCC, N2: Op.getOperand(i: 0));
12316}
12317
12318static bool isNonZeroAVL(SDValue AVL) {
12319 auto *RegisterAVL = dyn_cast<RegisterSDNode>(Val&: AVL);
12320 auto *ImmAVL = dyn_cast<ConstantSDNode>(Val&: AVL);
12321 return (RegisterAVL && RegisterAVL->getReg() == RISCV::X0) ||
12322 (ImmAVL && ImmAVL->getZExtValue() >= 1);
12323}
12324
12325/// Helper to lower a reduction sequence of the form:
12326/// scalar = reduce_op vec, scalar_start
12327static SDValue lowerReductionSeq(unsigned RVVOpcode, MVT ResVT,
12328 SDValue StartValue, SDValue Vec, SDValue Mask,
12329 SDValue VL, const SDLoc &DL, SelectionDAG &DAG,
12330 const RISCVSubtarget &Subtarget) {
12331 const MVT VecVT = Vec.getSimpleValueType();
12332 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: VecVT);
12333 const MVT XLenVT = Subtarget.getXLenVT();
12334 const bool NonZeroAVL = isNonZeroAVL(AVL: VL);
12335
12336 // The reduction needs an LMUL1 input; do the splat at either LMUL1
12337 // or the original VT if fractional.
12338 auto InnerVT = VecVT.bitsLE(VT: M1VT) ? VecVT : M1VT;
12339 // We reuse the VL of the reduction to reduce vsetvli toggles if we can
12340 // prove it is non-zero. For the AVL=0 case, we need the scalar to
12341 // be the result of the reduction operation.
12342 auto InnerVL = NonZeroAVL ? VL : DAG.getConstant(Val: 1, DL, VT: XLenVT);
12343 SDValue InitialValue =
12344 lowerScalarInsert(Scalar: StartValue, VL: InnerVL, VT: InnerVT, DL, DAG, Subtarget);
12345 if (M1VT != InnerVT)
12346 InitialValue =
12347 DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: M1VT), SubVec: InitialValue, Idx: 0);
12348 SDValue PassThru = NonZeroAVL ? DAG.getUNDEF(VT: M1VT) : InitialValue;
12349 SDValue Policy = DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
12350 SDValue Ops[] = {PassThru, Vec, InitialValue, Mask, VL, Policy};
12351 SDValue Reduction = DAG.getNode(Opcode: RVVOpcode, DL, VT: M1VT, Ops);
12352 return DAG.getExtractVectorElt(DL, VT: ResVT, Vec: Reduction, Idx: 0);
12353}
12354
12355SDValue RISCVTargetLowering::lowerVECREDUCE(SDValue Op,
12356 SelectionDAG &DAG) const {
12357 SDLoc DL(Op);
12358 SDValue Vec = Op.getOperand(i: 0);
12359 EVT VecEVT = Vec.getValueType();
12360
12361 unsigned BaseOpc = ISD::getVecReduceBaseOpcode(VecReduceOpcode: Op.getOpcode());
12362
12363 // Due to ordering in legalize types we may have a vector type that needs to
12364 // be split. Do that manually so we can get down to a legal type.
12365 while (getTypeAction(Context&: *DAG.getContext(), VT: VecEVT) ==
12366 TargetLowering::TypeSplitVector) {
12367 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
12368 VecEVT = Lo.getValueType();
12369 Vec = DAG.getNode(Opcode: BaseOpc, DL, VT: VecEVT, N1: Lo, N2: Hi);
12370 }
12371
12372 // TODO: The type may need to be widened rather than split. Or widened before
12373 // it can be split.
12374 if (!isTypeLegal(VT: VecEVT))
12375 return SDValue();
12376
12377 MVT VecVT = VecEVT.getSimpleVT();
12378 MVT VecEltVT = VecVT.getVectorElementType();
12379 unsigned RVVOpcode = getRVVReductionOp(ISDOpcode: Op.getOpcode());
12380
12381 MVT ContainerVT = VecVT;
12382 if (VecVT.isFixedLengthVector()) {
12383 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12384 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12385 }
12386
12387 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12388
12389 SDValue StartV = DAG.getNeutralElement(Opcode: BaseOpc, DL, VT: VecEltVT, Flags: SDNodeFlags());
12390 switch (BaseOpc) {
12391 case ISD::AND:
12392 case ISD::OR:
12393 case ISD::UMAX:
12394 case ISD::UMIN:
12395 case ISD::SMAX:
12396 case ISD::SMIN:
12397 StartV = DAG.getExtractVectorElt(DL, VT: VecEltVT, Vec, Idx: 0);
12398 }
12399 return lowerReductionSeq(RVVOpcode, ResVT: Op.getSimpleValueType(), StartValue: StartV, Vec,
12400 Mask, VL, DL, DAG, Subtarget);
12401}
12402
12403// Given a reduction op, this function returns the matching reduction opcode,
12404// the vector SDValue and the scalar SDValue required to lower this to a
12405// RISCVISD node.
12406static std::tuple<unsigned, SDValue, SDValue>
12407getRVVFPReductionOpAndOperands(SDValue Op, SelectionDAG &DAG, EVT EltVT,
12408 const RISCVSubtarget &Subtarget) {
12409 SDLoc DL(Op);
12410 auto Flags = Op->getFlags();
12411 unsigned Opcode = Op.getOpcode();
12412 switch (Opcode) {
12413 default:
12414 llvm_unreachable("Unhandled reduction");
12415 case ISD::VECREDUCE_FADD: {
12416 // Use positive zero if we can. It is cheaper to materialize.
12417 SDValue Zero =
12418 DAG.getConstantFP(Val: Flags.hasNoSignedZeros() ? 0.0 : -0.0, DL, VT: EltVT);
12419 return std::make_tuple(args: RISCVISD::VECREDUCE_FADD_VL, args: Op.getOperand(i: 0), args&: Zero);
12420 }
12421 case ISD::VECREDUCE_SEQ_FADD:
12422 return std::make_tuple(args: RISCVISD::VECREDUCE_SEQ_FADD_VL, args: Op.getOperand(i: 1),
12423 args: Op.getOperand(i: 0));
12424 case ISD::VECREDUCE_FMINIMUM:
12425 case ISD::VECREDUCE_FMAXIMUM:
12426 case ISD::VECREDUCE_FMIN:
12427 case ISD::VECREDUCE_FMAX: {
12428 SDValue Front = DAG.getExtractVectorElt(DL, VT: EltVT, Vec: Op.getOperand(i: 0), Idx: 0);
12429 unsigned RVVOpc =
12430 (Opcode == ISD::VECREDUCE_FMIN || Opcode == ISD::VECREDUCE_FMINIMUM)
12431 ? RISCVISD::VECREDUCE_FMIN_VL
12432 : RISCVISD::VECREDUCE_FMAX_VL;
12433 return std::make_tuple(args&: RVVOpc, args: Op.getOperand(i: 0), args&: Front);
12434 }
12435 }
12436}
12437
12438SDValue RISCVTargetLowering::lowerFPVECREDUCE(SDValue Op,
12439 SelectionDAG &DAG) const {
12440 SDLoc DL(Op);
12441 MVT VecEltVT = Op.getSimpleValueType();
12442
12443 unsigned RVVOpcode;
12444 SDValue VectorVal, ScalarVal;
12445 std::tie(args&: RVVOpcode, args&: VectorVal, args&: ScalarVal) =
12446 getRVVFPReductionOpAndOperands(Op, DAG, EltVT: VecEltVT, Subtarget);
12447 MVT VecVT = VectorVal.getSimpleValueType();
12448
12449 MVT ContainerVT = VecVT;
12450 if (VecVT.isFixedLengthVector()) {
12451 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12452 VectorVal = convertToScalableVector(VT: ContainerVT, V: VectorVal, DAG, Subtarget);
12453 }
12454
12455 MVT ResVT = Op.getSimpleValueType();
12456 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
12457 SDValue Res = lowerReductionSeq(RVVOpcode, ResVT, StartValue: ScalarVal, Vec: VectorVal, Mask,
12458 VL, DL, DAG, Subtarget);
12459 if (Op.getOpcode() != ISD::VECREDUCE_FMINIMUM &&
12460 Op.getOpcode() != ISD::VECREDUCE_FMAXIMUM)
12461 return Res;
12462
12463 if (Op->getFlags().hasNoNaNs())
12464 return Res;
12465
12466 // Force output to NaN if any element is Nan.
12467 SDValue IsNan =
12468 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: Mask.getValueType(),
12469 Ops: {VectorVal, VectorVal, DAG.getCondCode(Cond: ISD::SETNE),
12470 DAG.getUNDEF(VT: Mask.getValueType()), Mask, VL});
12471 MVT XLenVT = Subtarget.getXLenVT();
12472 SDValue CPop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: IsNan, N2: Mask, N3: VL);
12473 SDValue NoNaNs = DAG.getSetCC(DL, VT: XLenVT, LHS: CPop,
12474 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
12475 return DAG.getSelect(
12476 DL, VT: ResVT, Cond: NoNaNs, LHS: Res,
12477 RHS: DAG.getConstantFP(Val: APFloat::getNaN(Sem: ResVT.getFltSemantics()), DL, VT: ResVT));
12478}
12479
12480SDValue RISCVTargetLowering::lowerVPREDUCE(SDValue Op,
12481 SelectionDAG &DAG) const {
12482 SDLoc DL(Op);
12483 unsigned Opc = Op.getOpcode();
12484 SDValue Start = Op.getOperand(i: 0);
12485 SDValue Vec = Op.getOperand(i: 1);
12486 EVT VecEVT = Vec.getValueType();
12487 MVT XLenVT = Subtarget.getXLenVT();
12488
12489 // TODO: The type may need to be widened rather than split. Or widened before
12490 // it can be split.
12491 if (!isTypeLegal(VT: VecEVT))
12492 return SDValue();
12493
12494 MVT VecVT = VecEVT.getSimpleVT();
12495 unsigned RVVOpcode = getRVVReductionOp(ISDOpcode: Opc);
12496
12497 if (VecVT.isFixedLengthVector()) {
12498 auto ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12499 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12500 }
12501
12502 SDValue VL = Op.getOperand(i: 3);
12503 SDValue Mask = Op.getOperand(i: 2);
12504 SDValue Res =
12505 lowerReductionSeq(RVVOpcode, ResVT: Op.getSimpleValueType(), StartValue: Op.getOperand(i: 0),
12506 Vec, Mask, VL, DL, DAG, Subtarget);
12507 if ((Opc != ISD::VP_REDUCE_FMINIMUM && Opc != ISD::VP_REDUCE_FMAXIMUM) ||
12508 Op->getFlags().hasNoNaNs())
12509 return Res;
12510
12511 // Propagate NaNs.
12512 MVT PredVT = getMaskTypeFor(VecVT: Vec.getSimpleValueType());
12513 // Check if any of the elements in Vec is NaN.
12514 SDValue IsNaN = DAG.getNode(
12515 Opcode: RISCVISD::SETCC_VL, DL, VT: PredVT,
12516 Ops: {Vec, Vec, DAG.getCondCode(Cond: ISD::SETNE), DAG.getUNDEF(VT: PredVT), Mask, VL});
12517 SDValue VCPop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: IsNaN, N2: Mask, N3: VL);
12518 // Check if the start value is NaN.
12519 SDValue StartIsNaN = DAG.getSetCC(DL, VT: XLenVT, LHS: Start, RHS: Start, Cond: ISD::SETUO);
12520 VCPop = DAG.getNode(Opcode: ISD::OR, DL, VT: XLenVT, N1: VCPop, N2: StartIsNaN);
12521 SDValue NoNaNs = DAG.getSetCC(DL, VT: XLenVT, LHS: VCPop,
12522 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: ISD::SETEQ);
12523 MVT ResVT = Res.getSimpleValueType();
12524 return DAG.getSelect(
12525 DL, VT: ResVT, Cond: NoNaNs, LHS: Res,
12526 RHS: DAG.getConstantFP(Val: APFloat::getNaN(Sem: ResVT.getFltSemantics()), DL, VT: ResVT));
12527}
12528
12529SDValue RISCVTargetLowering::lowerINSERT_SUBVECTOR(SDValue Op,
12530 SelectionDAG &DAG) const {
12531 SDValue Vec = Op.getOperand(i: 0);
12532 SDValue SubVec = Op.getOperand(i: 1);
12533 MVT VecVT = Vec.getSimpleValueType();
12534 MVT SubVecVT = SubVec.getSimpleValueType();
12535
12536 SDLoc DL(Op);
12537 MVT XLenVT = Subtarget.getXLenVT();
12538 unsigned OrigIdx = Op.getConstantOperandVal(i: 2);
12539 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
12540
12541 if (OrigIdx == 0 && Vec.isUndef())
12542 return Op;
12543
12544 // We don't have the ability to slide mask vectors up indexed by their i1
12545 // elements; the smallest we can do is i8. Often we are able to bitcast to
12546 // equivalent i8 vectors. Note that when inserting a fixed-length vector
12547 // into a scalable one, we might not necessarily have enough scalable
12548 // elements to safely divide by 8: nxv1i1 = insert nxv1i1, v4i1 is valid.
12549 if (SubVecVT.getVectorElementType() == MVT::i1) {
12550 if (VecVT.getVectorMinNumElements() >= 8 &&
12551 SubVecVT.getVectorMinNumElements() >= 8) {
12552 assert(OrigIdx % 8 == 0 && "Invalid index");
12553 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
12554 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
12555 "Unexpected mask vector lowering");
12556 OrigIdx /= 8;
12557 SubVecVT =
12558 MVT::getVectorVT(VT: MVT::i8, NumElements: SubVecVT.getVectorMinNumElements() / 8,
12559 IsScalable: SubVecVT.isScalableVector());
12560 VecVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VecVT.getVectorMinNumElements() / 8,
12561 IsScalable: VecVT.isScalableVector());
12562 Vec = DAG.getBitcast(VT: VecVT, V: Vec);
12563 SubVec = DAG.getBitcast(VT: SubVecVT, V: SubVec);
12564 } else {
12565 // We can't slide this mask vector up indexed by its i1 elements.
12566 // This poses a problem when we wish to insert a scalable vector which
12567 // can't be re-expressed as a larger type. Just choose the slow path and
12568 // extend to a larger type, then truncate back down.
12569 MVT ExtVecVT = VecVT.changeVectorElementType(EltVT: MVT::i8);
12570 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(EltVT: MVT::i8);
12571 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVecVT, Operand: Vec);
12572 SubVec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtSubVecVT, Operand: SubVec);
12573 Vec = DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT: ExtVecVT, N1: Vec, N2: SubVec,
12574 N3: Op.getOperand(i: 2));
12575 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: ExtVecVT);
12576 return DAG.getSetCC(DL, VT: VecVT, LHS: Vec, RHS: SplatZero, Cond: ISD::SETNE);
12577 }
12578 }
12579
12580 // If the subvector vector is a fixed-length type and we don't know VLEN
12581 // exactly, we cannot use subregister manipulation to simplify the codegen; we
12582 // don't know which register of a LMUL group contains the specific subvector
12583 // as we only know the minimum register size. Therefore we must slide the
12584 // vector group up the full amount.
12585 const auto VLen = Subtarget.getRealVLen();
12586 if (SubVecVT.isFixedLengthVector() && !VLen) {
12587 MVT ContainerVT = VecVT;
12588 if (VecVT.isFixedLengthVector()) {
12589 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12590 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12591 }
12592
12593 SubVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ContainerVT), SubVec, Idx: 0);
12594
12595 SDValue Mask =
12596 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
12597 // Set the vector length to only the number of elements we care about. Note
12598 // that for slideup this includes the offset.
12599 unsigned EndIndex = OrigIdx + SubVecVT.getVectorNumElements();
12600 SDValue VL = DAG.getConstant(Val: EndIndex, DL, VT: XLenVT);
12601
12602 // Use tail agnostic policy if we're inserting over Vec's tail.
12603 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
12604 if (VecVT.isFixedLengthVector() && EndIndex == VecVT.getVectorNumElements())
12605 Policy = RISCVVType::TAIL_AGNOSTIC;
12606
12607 // If we're inserting into the lowest elements, use a tail undisturbed
12608 // vmv.v.v.
12609 if (OrigIdx == 0) {
12610 SubVec =
12611 DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: ContainerVT, N1: Vec, N2: SubVec, N3: VL);
12612 } else {
12613 SDValue SlideupAmt = DAG.getConstant(Val: OrigIdx, DL, VT: XLenVT);
12614 SubVec = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Vec, Op: SubVec,
12615 Offset: SlideupAmt, Mask, VL, Policy);
12616 }
12617
12618 if (VecVT.isFixedLengthVector())
12619 SubVec = convertFromScalableVector(VT: VecVT, V: SubVec, DAG, Subtarget);
12620 return DAG.getBitcast(VT: Op.getValueType(), V: SubVec);
12621 }
12622
12623 MVT ContainerVecVT = VecVT;
12624 if (VecVT.isFixedLengthVector()) {
12625 ContainerVecVT = getContainerForFixedLengthVector(VT: VecVT);
12626 Vec = convertToScalableVector(VT: ContainerVecVT, V: Vec, DAG, Subtarget);
12627 }
12628
12629 MVT ContainerSubVecVT = SubVecVT;
12630 if (SubVecVT.isFixedLengthVector()) {
12631 ContainerSubVecVT = getContainerForFixedLengthVector(VT: SubVecVT);
12632 SubVec = convertToScalableVector(VT: ContainerSubVecVT, V: SubVec, DAG, Subtarget);
12633 }
12634
12635 unsigned SubRegIdx;
12636 ElementCount RemIdx;
12637 // insert_subvector scales the index by vscale if the subvector is scalable,
12638 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
12639 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
12640 if (SubVecVT.isFixedLengthVector()) {
12641 assert(VLen);
12642 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
12643 auto Decompose =
12644 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
12645 VecVT: ContainerVecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx / Vscale, TRI);
12646 SubRegIdx = Decompose.first;
12647 RemIdx = ElementCount::getFixed(MinVal: (Decompose.second * Vscale) +
12648 (OrigIdx % Vscale));
12649 } else {
12650 auto Decompose =
12651 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
12652 VecVT: ContainerVecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx, TRI);
12653 SubRegIdx = Decompose.first;
12654 RemIdx = ElementCount::getScalable(MinVal: Decompose.second);
12655 }
12656
12657 TypeSize VecRegSize = TypeSize::getScalable(MinimumSize: RISCV::RVVBitsPerBlock);
12658 assert(isPowerOf2_64(
12659 Subtarget.expandVScale(SubVecVT.getSizeInBits()).getKnownMinValue()));
12660 bool ExactlyVecRegSized =
12661 Subtarget.expandVScale(X: SubVecVT.getSizeInBits())
12662 .isKnownMultipleOf(RHS: Subtarget.expandVScale(X: VecRegSize));
12663
12664 // 1. If the Idx has been completely eliminated and this subvector's size is
12665 // a vector register or a multiple thereof, or the surrounding elements are
12666 // undef, then this is a subvector insert which naturally aligns to a vector
12667 // register. These can easily be handled using subregister manipulation.
12668 // 2. If the subvector isn't an exact multiple of a valid register group size,
12669 // then the insertion must preserve the undisturbed elements of the register.
12670 // We do this by lowering to an EXTRACT_SUBVECTOR grabbing the nearest LMUL=1
12671 // vector type (which resolves to a subregister copy), performing a VSLIDEUP
12672 // to place the subvector within the vector register, and an INSERT_SUBVECTOR
12673 // of that LMUL=1 type back into the larger vector (resolving to another
12674 // subregister operation). See below for how our VSLIDEUP works. We go via a
12675 // LMUL=1 type to avoid allocating a large register group to hold our
12676 // subvector.
12677 if (RemIdx.isZero() && (ExactlyVecRegSized || Vec.isUndef())) {
12678 if (SubVecVT.isFixedLengthVector()) {
12679 // We may get NoSubRegister if inserting at index 0 and the subvec
12680 // container is the same as the vector, e.g. vec=v4i32,subvec=v4i32,idx=0
12681 if (SubRegIdx == RISCV::NoSubRegister) {
12682 assert(OrigIdx == 0);
12683 return Op;
12684 }
12685
12686 // Use a insert_subvector that will resolve to an insert subreg.
12687 assert(VLen);
12688 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
12689 SDValue Insert =
12690 DAG.getInsertSubvector(DL, Vec, SubVec, Idx: OrigIdx / Vscale);
12691 if (VecVT.isFixedLengthVector())
12692 Insert = convertFromScalableVector(VT: VecVT, V: Insert, DAG, Subtarget);
12693 return Insert;
12694 }
12695 return Op;
12696 }
12697
12698 // VSLIDEUP works by leaving elements 0<i<OFFSET undisturbed, elements
12699 // OFFSET<=i<VL set to the "subvector" and vl<=i<VLMAX set to the tail policy
12700 // (in our case undisturbed). This means we can set up a subvector insertion
12701 // where OFFSET is the insertion offset, and the VL is the OFFSET plus the
12702 // size of the subvector.
12703 MVT InterSubVT = ContainerVecVT;
12704 SDValue AlignedExtract = Vec;
12705 unsigned AlignedIdx = OrigIdx - RemIdx.getKnownMinValue();
12706 if (SubVecVT.isFixedLengthVector()) {
12707 assert(VLen);
12708 AlignedIdx /= *VLen / RISCV::RVVBitsPerBlock;
12709 }
12710 if (ContainerVecVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: ContainerVecVT))) {
12711 InterSubVT = RISCVTargetLowering::getM1VT(VT: ContainerVecVT);
12712 // Extract a subvector equal to the nearest full vector register type. This
12713 // should resolve to a EXTRACT_SUBREG instruction.
12714 AlignedExtract = DAG.getExtractSubvector(DL, VT: InterSubVT, Vec, Idx: AlignedIdx);
12715 }
12716
12717 SubVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: InterSubVT), SubVec, Idx: 0);
12718
12719 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT: ContainerVecVT, DL, DAG, Subtarget);
12720
12721 ElementCount EndIndex = RemIdx + SubVecVT.getVectorElementCount();
12722 VL = DAG.getElementCount(DL, VT: XLenVT, EC: SubVecVT.getVectorElementCount());
12723
12724 // Use tail agnostic policy if we're inserting over InterSubVT's tail.
12725 unsigned Policy = RISCVVType::TAIL_UNDISTURBED_MASK_UNDISTURBED;
12726 if (Subtarget.expandVScale(X: EndIndex) ==
12727 Subtarget.expandVScale(X: InterSubVT.getVectorElementCount()))
12728 Policy = RISCVVType::TAIL_AGNOSTIC;
12729
12730 // If we're inserting into the lowest elements, use a tail undisturbed
12731 // vmv.v.v.
12732 if (RemIdx.isZero()) {
12733 SubVec = DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL, VT: InterSubVT, N1: AlignedExtract,
12734 N2: SubVec, N3: VL);
12735 } else {
12736 SDValue SlideupAmt = DAG.getElementCount(DL, VT: XLenVT, EC: RemIdx);
12737
12738 // Construct the vector length corresponding to RemIdx + length(SubVecVT).
12739 VL = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: SlideupAmt, N2: VL);
12740
12741 SubVec = getVSlideup(DAG, Subtarget, DL, VT: InterSubVT, Passthru: AlignedExtract, Op: SubVec,
12742 Offset: SlideupAmt, Mask, VL, Policy);
12743 }
12744
12745 // If required, insert this subvector back into the correct vector register.
12746 // This should resolve to an INSERT_SUBREG instruction.
12747 if (ContainerVecVT.bitsGT(VT: InterSubVT))
12748 SubVec = DAG.getInsertSubvector(DL, Vec, SubVec, Idx: AlignedIdx);
12749
12750 if (VecVT.isFixedLengthVector())
12751 SubVec = convertFromScalableVector(VT: VecVT, V: SubVec, DAG, Subtarget);
12752
12753 // We might have bitcast from a mask type: cast back to the original type if
12754 // required.
12755 return DAG.getBitcast(VT: Op.getSimpleValueType(), V: SubVec);
12756}
12757
12758SDValue RISCVTargetLowering::lowerEXTRACT_SUBVECTOR(SDValue Op,
12759 SelectionDAG &DAG) const {
12760 SDValue Vec = Op.getOperand(i: 0);
12761 MVT SubVecVT = Op.getSimpleValueType();
12762 MVT VecVT = Vec.getSimpleValueType();
12763
12764 SDLoc DL(Op);
12765 MVT XLenVT = Subtarget.getXLenVT();
12766 unsigned OrigIdx = Op.getConstantOperandVal(i: 1);
12767 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
12768
12769 // With an index of 0 this is a cast-like subvector, which can be performed
12770 // with subregister operations.
12771 if (OrigIdx == 0)
12772 return Op;
12773
12774 // We don't have the ability to slide mask vectors down indexed by their i1
12775 // elements; the smallest we can do is i8. Often we are able to bitcast to
12776 // equivalent i8 vectors. Note that when extracting a fixed-length vector
12777 // from a scalable one, we might not necessarily have enough scalable
12778 // elements to safely divide by 8: v8i1 = extract nxv1i1 is valid.
12779 if (SubVecVT.getVectorElementType() == MVT::i1) {
12780 if (VecVT.getVectorMinNumElements() >= 8 &&
12781 SubVecVT.getVectorMinNumElements() >= 8) {
12782 assert(OrigIdx % 8 == 0 && "Invalid index");
12783 assert(VecVT.getVectorMinNumElements() % 8 == 0 &&
12784 SubVecVT.getVectorMinNumElements() % 8 == 0 &&
12785 "Unexpected mask vector lowering");
12786 OrigIdx /= 8;
12787 SubVecVT =
12788 MVT::getVectorVT(VT: MVT::i8, NumElements: SubVecVT.getVectorMinNumElements() / 8,
12789 IsScalable: SubVecVT.isScalableVector());
12790 VecVT = MVT::getVectorVT(VT: MVT::i8, NumElements: VecVT.getVectorMinNumElements() / 8,
12791 IsScalable: VecVT.isScalableVector());
12792 Vec = DAG.getBitcast(VT: VecVT, V: Vec);
12793 } else {
12794 // We can't slide this mask vector down, indexed by its i1 elements.
12795 // This poses a problem when we wish to extract a scalable vector which
12796 // can't be re-expressed as a larger type. Just choose the slow path and
12797 // extend to a larger type, then truncate back down.
12798 // TODO: We could probably improve this when extracting certain fixed
12799 // from fixed, where we can extract as i8 and shift the correct element
12800 // right to reach the desired subvector?
12801 MVT ExtVecVT = VecVT.changeVectorElementType(EltVT: MVT::i8);
12802 MVT ExtSubVecVT = SubVecVT.changeVectorElementType(EltVT: MVT::i8);
12803 Vec = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: ExtVecVT, Operand: Vec);
12804 Vec = DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT: ExtSubVecVT, N1: Vec,
12805 N2: Op.getOperand(i: 1));
12806 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: ExtSubVecVT);
12807 return DAG.getSetCC(DL, VT: SubVecVT, LHS: Vec, RHS: SplatZero, Cond: ISD::SETNE);
12808 }
12809 }
12810
12811 const auto VLen = Subtarget.getRealVLen();
12812
12813 // If the subvector vector is a fixed-length type and we don't know VLEN
12814 // exactly, we cannot use subregister manipulation to simplify the codegen; we
12815 // don't know which register of a LMUL group contains the specific subvector
12816 // as we only know the minimum register size. Therefore we must slide the
12817 // vector group down the full amount.
12818 if (SubVecVT.isFixedLengthVector() && !VLen) {
12819 MVT ContainerVT = VecVT;
12820 if (VecVT.isFixedLengthVector()) {
12821 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12822 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
12823 }
12824
12825 // Shrink down Vec so we're performing the slidedown on a smaller LMUL.
12826 unsigned LastIdx = OrigIdx + SubVecVT.getVectorNumElements() - 1;
12827 if (auto ShrunkVT =
12828 getSmallestVTForIndex(VecVT: ContainerVT, MaxIdx: LastIdx, DL, DAG, Subtarget)) {
12829 ContainerVT = *ShrunkVT;
12830 Vec = DAG.getExtractSubvector(DL, VT: ContainerVT, Vec, Idx: 0);
12831 }
12832
12833 SDValue Mask =
12834 getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget).first;
12835 // Set the vector length to only the number of elements we care about. This
12836 // avoids sliding down elements we're going to discard straight away.
12837 SDValue VL = DAG.getConstant(Val: SubVecVT.getVectorNumElements(), DL, VT: XLenVT);
12838 SDValue SlidedownAmt = DAG.getConstant(Val: OrigIdx, DL, VT: XLenVT);
12839 SDValue Slidedown =
12840 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
12841 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: SlidedownAmt, Mask, VL);
12842 // Now we can use a cast-like subvector extract to get the result.
12843 Slidedown = DAG.getExtractSubvector(DL, VT: SubVecVT, Vec: Slidedown, Idx: 0);
12844 return DAG.getBitcast(VT: Op.getValueType(), V: Slidedown);
12845 }
12846
12847 if (VecVT.isFixedLengthVector()) {
12848 VecVT = getContainerForFixedLengthVector(VT: VecVT);
12849 Vec = convertToScalableVector(VT: VecVT, V: Vec, DAG, Subtarget);
12850 }
12851
12852 MVT ContainerSubVecVT = SubVecVT;
12853 if (SubVecVT.isFixedLengthVector())
12854 ContainerSubVecVT = getContainerForFixedLengthVector(VT: SubVecVT);
12855
12856 unsigned SubRegIdx;
12857 ElementCount RemIdx;
12858 // extract_subvector scales the index by vscale if the subvector is scalable,
12859 // and decomposeSubvectorInsertExtractToSubRegs takes this into account. So if
12860 // we have a fixed length subvector, we need to adjust the index by 1/vscale.
12861 if (SubVecVT.isFixedLengthVector()) {
12862 assert(VLen);
12863 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
12864 auto Decompose =
12865 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
12866 VecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx / Vscale, TRI);
12867 SubRegIdx = Decompose.first;
12868 RemIdx = ElementCount::getFixed(MinVal: (Decompose.second * Vscale) +
12869 (OrigIdx % Vscale));
12870 } else {
12871 auto Decompose =
12872 RISCVTargetLowering::decomposeSubvectorInsertExtractToSubRegs(
12873 VecVT, SubVecVT: ContainerSubVecVT, InsertExtractIdx: OrigIdx, TRI);
12874 SubRegIdx = Decompose.first;
12875 RemIdx = ElementCount::getScalable(MinVal: Decompose.second);
12876 }
12877
12878 // If the Idx has been completely eliminated then this is a subvector extract
12879 // which naturally aligns to a vector register. These can easily be handled
12880 // using subregister manipulation. We use an extract_subvector that will
12881 // resolve to an extract subreg.
12882 if (RemIdx.isZero()) {
12883 if (SubVecVT.isFixedLengthVector()) {
12884 assert(VLen);
12885 unsigned Vscale = *VLen / RISCV::RVVBitsPerBlock;
12886 Vec =
12887 DAG.getExtractSubvector(DL, VT: ContainerSubVecVT, Vec, Idx: OrigIdx / Vscale);
12888 return convertFromScalableVector(VT: SubVecVT, V: Vec, DAG, Subtarget);
12889 }
12890 return Op;
12891 }
12892
12893 // Else SubVecVT is M1 or smaller and may need to be slid down: if SubVecVT
12894 // was > M1 then the index would need to be a multiple of VLMAX, and so would
12895 // divide exactly.
12896 assert(RISCVVType::decodeVLMUL(getLMUL(ContainerSubVecVT)).second ||
12897 getLMUL(ContainerSubVecVT) == RISCVVType::LMUL_1);
12898
12899 // If the vector type is an LMUL-group type, extract a subvector equal to the
12900 // nearest full vector register type.
12901 MVT InterSubVT = VecVT;
12902 if (VecVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: VecVT))) {
12903 // If VecVT has an LMUL > 1, then SubVecVT should have a smaller LMUL, and
12904 // we should have successfully decomposed the extract into a subregister.
12905 // We use an extract_subvector that will resolve to a subreg extract.
12906 assert(SubRegIdx != RISCV::NoSubRegister);
12907 (void)SubRegIdx;
12908 unsigned Idx = OrigIdx - RemIdx.getKnownMinValue();
12909 if (SubVecVT.isFixedLengthVector()) {
12910 assert(VLen);
12911 Idx /= *VLen / RISCV::RVVBitsPerBlock;
12912 }
12913 InterSubVT = RISCVTargetLowering::getM1VT(VT: VecVT);
12914 Vec = DAG.getExtractSubvector(DL, VT: InterSubVT, Vec, Idx);
12915 }
12916
12917 // Slide this vector register down by the desired number of elements in order
12918 // to place the desired subvector starting at element 0.
12919 SDValue SlidedownAmt = DAG.getElementCount(DL, VT: XLenVT, EC: RemIdx);
12920 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: InterSubVT, DL, DAG, Subtarget);
12921 if (SubVecVT.isFixedLengthVector())
12922 VL = DAG.getConstant(Val: SubVecVT.getVectorNumElements(), DL, VT: XLenVT);
12923 SDValue Slidedown =
12924 getVSlidedown(DAG, Subtarget, DL, VT: InterSubVT, Passthru: DAG.getUNDEF(VT: InterSubVT),
12925 Op: Vec, Offset: SlidedownAmt, Mask, VL);
12926
12927 // Now the vector is in the right position, extract our final subvector. This
12928 // should resolve to a COPY.
12929 Slidedown = DAG.getExtractSubvector(DL, VT: SubVecVT, Vec: Slidedown, Idx: 0);
12930
12931 // We might have bitcast from a mask type: cast back to the original type if
12932 // required.
12933 return DAG.getBitcast(VT: Op.getSimpleValueType(), V: Slidedown);
12934}
12935
12936// Widen a vector's operands to i8, then truncate its results back to the
12937// original type, typically i1. All operand and result types must be the same.
12938static SDValue widenVectorOpsToi8(SDValue N, const SDLoc &DL,
12939 SelectionDAG &DAG) {
12940 MVT VT = N.getSimpleValueType();
12941 MVT WideVT = VT.changeVectorElementType(EltVT: MVT::i8);
12942 SmallVector<SDValue, 4> WideOps;
12943 for (SDValue Op : N->ops()) {
12944 assert(Op.getSimpleValueType() == VT &&
12945 "Operands and result must be same type");
12946 WideOps.push_back(Elt: DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WideVT, Operand: Op));
12947 }
12948
12949 unsigned NumVals = N->getNumValues();
12950
12951 SDVTList VTs = DAG.getVTList(VTs: SmallVector<EVT, 4>(
12952 NumVals,
12953 N.getValueType().changeVectorElementType(Context&: *DAG.getContext(), EltVT: MVT::i8)));
12954 SDValue WideN = DAG.getNode(Opcode: N.getOpcode(), DL, VTList: VTs, Ops: WideOps);
12955 SmallVector<SDValue, 4> TruncVals;
12956 for (unsigned I = 0; I < NumVals; I++) {
12957 TruncVals.push_back(
12958 Elt: DAG.getSetCC(DL, VT: N->getSimpleValueType(ResNo: I), LHS: WideN.getValue(R: I),
12959 RHS: DAG.getConstant(Val: 0, DL, VT: WideVT), Cond: ISD::SETNE));
12960 }
12961
12962 if (TruncVals.size() > 1)
12963 return DAG.getMergeValues(Ops: TruncVals, dl: DL);
12964 return TruncVals.front();
12965}
12966
12967SDValue RISCVTargetLowering::lowerVECTOR_DEINTERLEAVE(SDValue Op,
12968 SelectionDAG &DAG) const {
12969 SDLoc DL(Op);
12970 MVT VecVT = Op.getSimpleValueType();
12971
12972 const unsigned Factor = Op->getNumValues();
12973 assert(Factor <= 8);
12974
12975 // 1 bit element vectors need to be widened to e8
12976 if (VecVT.getVectorElementType() == MVT::i1)
12977 return widenVectorOpsToi8(N: Op, DL, DAG);
12978
12979 // Convert to scalable vectors first.
12980 if (VecVT.isFixedLengthVector()) {
12981 MVT ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
12982 SmallVector<SDValue, 8> Ops(Factor);
12983 for (unsigned i = 0U; i < Factor; ++i)
12984 Ops[i] = convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i), DAG,
12985 Subtarget);
12986
12987 SmallVector<EVT, 8> VTs(Factor, ContainerVT);
12988 SDValue NewDeinterleave =
12989 DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs, Ops);
12990
12991 SmallVector<SDValue, 8> Res(Factor);
12992 for (unsigned i = 0U; i < Factor; ++i)
12993 Res[i] = convertFromScalableVector(VT: VecVT, V: NewDeinterleave.getValue(R: i),
12994 DAG, Subtarget);
12995 return DAG.getMergeValues(Ops: Res, dl: DL);
12996 }
12997
12998 // If concatenating would exceed LMUL=8, we need to split.
12999 if ((VecVT.getSizeInBits().getKnownMinValue() * Factor) >
13000 (8 * RISCV::RVVBitsPerBlock)) {
13001 SmallVector<SDValue, 8> Ops(Factor * 2);
13002 for (unsigned i = 0; i != Factor; ++i) {
13003 auto [OpLo, OpHi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: i);
13004 Ops[i * 2] = OpLo;
13005 Ops[i * 2 + 1] = OpHi;
13006 }
13007
13008 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
13009
13010 SDValue Lo = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
13011 Ops: ArrayRef(Ops).slice(N: 0, M: Factor));
13012 SDValue Hi = DAG.getNode(Opcode: ISD::VECTOR_DEINTERLEAVE, DL, ResultTys: VTs,
13013 Ops: ArrayRef(Ops).slice(N: Factor, M: Factor));
13014
13015 SmallVector<SDValue, 8> Res(Factor);
13016 for (unsigned i = 0; i != Factor; ++i)
13017 Res[i] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Lo.getValue(R: i),
13018 N2: Hi.getValue(R: i));
13019
13020 return DAG.getMergeValues(Ops: Res, dl: DL);
13021 }
13022
13023 if (Subtarget.hasVendorXRivosVizip() && Factor == 2) {
13024 MVT VT = Op->getSimpleValueType(ResNo: 0);
13025 SDValue V1 = Op->getOperand(Num: 0);
13026 SDValue V2 = Op->getOperand(Num: 1);
13027
13028 // For fractional LMUL, check if we can use a higher LMUL
13029 // instruction to avoid a vslidedown.
13030 if (SDValue Src = foldConcatVector(V1, V2);
13031 Src && RISCVTargetLowering::getM1VT(VT).bitsGT(VT)) {
13032 EVT NewVT = VT.getDoubleNumVectorElementsVT();
13033 Src = DAG.getExtractSubvector(DL, VT: NewVT, Vec: Src, Idx: 0);
13034 // Freeze the source so we can increase its use count.
13035 Src = DAG.getFreeze(V: Src);
13036 SDValue Even = lowerVZIP(Opc: RISCVISD::RI_VUNZIP2A_VL, Op0: Src,
13037 Op1: DAG.getUNDEF(VT: NewVT), DL, DAG, Subtarget);
13038 SDValue Odd = lowerVZIP(Opc: RISCVISD::RI_VUNZIP2B_VL, Op0: Src,
13039 Op1: DAG.getUNDEF(VT: NewVT), DL, DAG, Subtarget);
13040 Even = DAG.getExtractSubvector(DL, VT, Vec: Even, Idx: 0);
13041 Odd = DAG.getExtractSubvector(DL, VT, Vec: Odd, Idx: 0);
13042 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
13043 }
13044
13045 // Freeze the sources so we can increase their use count.
13046 V1 = DAG.getFreeze(V: V1);
13047 V2 = DAG.getFreeze(V: V2);
13048 SDValue Even =
13049 lowerVZIP(Opc: RISCVISD::RI_VUNZIP2A_VL, Op0: V1, Op1: V2, DL, DAG, Subtarget);
13050 SDValue Odd =
13051 lowerVZIP(Opc: RISCVISD::RI_VUNZIP2B_VL, Op0: V1, Op1: V2, DL, DAG, Subtarget);
13052 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
13053 }
13054
13055 SmallVector<SDValue, 8> Ops(Op->op_values());
13056
13057 // Concatenate the vectors as one vector to deinterleave
13058 MVT ConcatVT =
13059 MVT::getVectorVT(VT: VecVT.getVectorElementType(),
13060 EC: VecVT.getVectorElementCount().multiplyCoefficientBy(
13061 RHS: PowerOf2Ceil(A: Factor)));
13062 if (Ops.size() < PowerOf2Ceil(A: Factor))
13063 Ops.append(NumInputs: PowerOf2Ceil(A: Factor) - Factor, Elt: DAG.getUNDEF(VT: VecVT));
13064 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT, Ops);
13065
13066 if (Factor == 2) {
13067 // We can deinterleave through vnsrl.wi if the element type is smaller than
13068 // ELEN
13069 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
13070 SDValue Even = getDeinterleaveShiftAndTrunc(DL, VT: VecVT, Src: Concat, Factor: 2, Index: 0, DAG);
13071 SDValue Odd = getDeinterleaveShiftAndTrunc(DL, VT: VecVT, Src: Concat, Factor: 2, Index: 1, DAG);
13072 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
13073 }
13074
13075 // For the indices, use the vmv.v.x of an i8 constant to fill the largest
13076 // possibly mask vector, then extract the required subvector. Doing this
13077 // (instead of a vid, vmsne sequence) reduces LMUL, and allows the mask
13078 // creation to be rematerialized during register allocation to reduce
13079 // register pressure if needed.
13080
13081 MVT MaskVT = ConcatVT.changeVectorElementType(EltVT: MVT::i1);
13082
13083 SDValue EvenSplat = DAG.getConstant(Val: 0b01010101, DL, VT: MVT::nxv8i8);
13084 EvenSplat = DAG.getBitcast(VT: MVT::nxv64i1, V: EvenSplat);
13085 SDValue EvenMask = DAG.getExtractSubvector(DL, VT: MaskVT, Vec: EvenSplat, Idx: 0);
13086
13087 SDValue OddSplat = DAG.getConstant(Val: 0b10101010, DL, VT: MVT::nxv8i8);
13088 OddSplat = DAG.getBitcast(VT: MVT::nxv64i1, V: OddSplat);
13089 SDValue OddMask = DAG.getExtractSubvector(DL, VT: MaskVT, Vec: OddSplat, Idx: 0);
13090
13091 // vcompress the even and odd elements into two separate vectors
13092 SDValue EvenWide = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: ConcatVT, N1: Concat,
13093 N2: EvenMask, N3: DAG.getUNDEF(VT: ConcatVT));
13094 SDValue OddWide = DAG.getNode(Opcode: ISD::VECTOR_COMPRESS, DL, VT: ConcatVT, N1: Concat,
13095 N2: OddMask, N3: DAG.getUNDEF(VT: ConcatVT));
13096
13097 // Extract the result half of the gather for even and odd
13098 SDValue Even = DAG.getExtractSubvector(DL, VT: VecVT, Vec: EvenWide, Idx: 0);
13099 SDValue Odd = DAG.getExtractSubvector(DL, VT: VecVT, Vec: OddWide, Idx: 0);
13100
13101 return DAG.getMergeValues(Ops: {Even, Odd}, dl: DL);
13102 }
13103
13104 // Store with unit-stride store and load it back with segmented load.
13105 MVT XLenVT = Subtarget.getXLenVT();
13106 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
13107 SDValue Passthru = DAG.getUNDEF(VT: ConcatVT);
13108
13109 // Allocate a stack slot.
13110 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
13111 SDValue StackPtr =
13112 DAG.CreateStackTemporary(Bytes: ConcatVT.getStoreSize(), Alignment);
13113 auto &MF = DAG.getMachineFunction();
13114 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13115 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13116
13117 SDValue StoreOps[] = {DAG.getEntryNode(),
13118 DAG.getTargetConstant(Val: Intrinsic::riscv_vse, DL, VT: XLenVT),
13119 Concat, StackPtr, VL};
13120
13121 SDValue Chain = DAG.getMemIntrinsicNode(
13122 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops: StoreOps,
13123 MemVT: ConcatVT.getVectorElementType(), PtrInfo, Alignment,
13124 Flags: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer());
13125
13126 static const Intrinsic::ID VlsegIntrinsicsIds[] = {
13127 Intrinsic::riscv_vlseg2_mask, Intrinsic::riscv_vlseg3_mask,
13128 Intrinsic::riscv_vlseg4_mask, Intrinsic::riscv_vlseg5_mask,
13129 Intrinsic::riscv_vlseg6_mask, Intrinsic::riscv_vlseg7_mask,
13130 Intrinsic::riscv_vlseg8_mask};
13131
13132 SDValue LoadOps[] = {
13133 Chain,
13134 DAG.getTargetConstant(Val: VlsegIntrinsicsIds[Factor - 2], DL, VT: XLenVT),
13135 Passthru,
13136 StackPtr,
13137 Mask,
13138 VL,
13139 DAG.getTargetConstant(
13140 Val: RISCVVType::TAIL_AGNOSTIC | RISCVVType::MASK_AGNOSTIC, DL, VT: XLenVT),
13141 DAG.getTargetConstant(Val: Log2_64(Value: VecVT.getScalarSizeInBits()), DL, VT: XLenVT)};
13142
13143 unsigned Sz =
13144 Factor * VecVT.getVectorMinNumElements() * VecVT.getScalarSizeInBits();
13145 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: Factor);
13146
13147 SDValue Load = DAG.getMemIntrinsicNode(
13148 Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: DAG.getVTList(VTs: {VecTupTy, MVT::Other}),
13149 Ops: LoadOps, MemVT: ConcatVT.getVectorElementType(), PtrInfo, Alignment,
13150 Flags: MachineMemOperand::MOLoad, Size: LocationSize::beforeOrAfterPointer());
13151
13152 SmallVector<SDValue, 8> Res(Factor);
13153
13154 for (unsigned i = 0U; i < Factor; ++i)
13155 Res[i] = DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT: VecVT, N1: Load,
13156 N2: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
13157
13158 return DAG.getMergeValues(Ops: Res, dl: DL);
13159}
13160
13161SDValue RISCVTargetLowering::lowerVECTOR_INTERLEAVE(SDValue Op,
13162 SelectionDAG &DAG) const {
13163 SDLoc DL(Op);
13164 MVT VecVT = Op.getSimpleValueType();
13165
13166 const unsigned Factor = Op.getNumOperands();
13167 assert(Factor <= 8);
13168
13169 // i1 vectors need to be widened to i8
13170 if (VecVT.getVectorElementType() == MVT::i1)
13171 return widenVectorOpsToi8(N: Op, DL, DAG);
13172
13173 // Convert to scalable vectors first.
13174 if (VecVT.isFixedLengthVector()) {
13175 MVT ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13176 SmallVector<SDValue, 8> Ops(Factor);
13177 for (unsigned i = 0U; i < Factor; ++i)
13178 Ops[i] = convertToScalableVector(VT: ContainerVT, V: Op.getOperand(i), DAG,
13179 Subtarget);
13180
13181 SmallVector<EVT, 8> VTs(Factor, ContainerVT);
13182 SDValue NewInterleave = DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs, Ops);
13183
13184 SmallVector<SDValue, 8> Res(Factor);
13185 for (unsigned i = 0U; i < Factor; ++i)
13186 Res[i] = convertFromScalableVector(VT: VecVT, V: NewInterleave.getValue(R: i), DAG,
13187 Subtarget);
13188 return DAG.getMergeValues(Ops: Res, dl: DL);
13189 }
13190
13191 MVT XLenVT = Subtarget.getXLenVT();
13192 auto [Mask, VL] = getDefaultScalableVLOps(VecVT, DL, DAG, Subtarget);
13193
13194 // If the VT is larger than LMUL=8, we need to split and reassemble.
13195 if ((VecVT.getSizeInBits().getKnownMinValue() * Factor) >
13196 (8 * RISCV::RVVBitsPerBlock)) {
13197 SmallVector<SDValue, 8> Ops(Factor * 2);
13198 for (unsigned i = 0; i != Factor; ++i) {
13199 auto [OpLo, OpHi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: i);
13200 Ops[i] = OpLo;
13201 Ops[i + Factor] = OpHi;
13202 }
13203
13204 SmallVector<EVT, 8> VTs(Factor, Ops[0].getValueType());
13205
13206 SDValue Res[] = {DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
13207 Ops: ArrayRef(Ops).take_front(N: Factor)),
13208 DAG.getNode(Opcode: ISD::VECTOR_INTERLEAVE, DL, ResultTys: VTs,
13209 Ops: ArrayRef(Ops).drop_front(N: Factor))};
13210
13211 SmallVector<SDValue, 8> Concats(Factor);
13212 for (unsigned i = 0; i != Factor; ++i) {
13213 unsigned IdxLo = 2 * i;
13214 unsigned IdxHi = 2 * i + 1;
13215 Concats[i] = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT,
13216 N1: Res[IdxLo / Factor].getValue(R: IdxLo % Factor),
13217 N2: Res[IdxHi / Factor].getValue(R: IdxHi % Factor));
13218 }
13219
13220 return DAG.getMergeValues(Ops: Concats, dl: DL);
13221 }
13222
13223 SDValue Interleaved;
13224
13225 // Spill to the stack using a segment store for simplicity.
13226 if (Factor != 2) {
13227 EVT MemVT =
13228 EVT::getVectorVT(Context&: *DAG.getContext(), VT: VecVT.getVectorElementType(),
13229 EC: VecVT.getVectorElementCount() * Factor);
13230
13231 // Allocate a stack slot.
13232 Align Alignment = DAG.getReducedAlign(VT: VecVT, /*UseABI=*/false);
13233 SDValue StackPtr =
13234 DAG.CreateStackTemporary(Bytes: MemVT.getStoreSize(), Alignment);
13235 EVT PtrVT = StackPtr.getValueType();
13236 auto &MF = DAG.getMachineFunction();
13237 auto FrameIndex = cast<FrameIndexSDNode>(Val: StackPtr.getNode())->getIndex();
13238 auto PtrInfo = MachinePointerInfo::getFixedStack(MF, FI: FrameIndex);
13239
13240 static const Intrinsic::ID IntrIds[] = {
13241 Intrinsic::riscv_vsseg2_mask, Intrinsic::riscv_vsseg3_mask,
13242 Intrinsic::riscv_vsseg4_mask, Intrinsic::riscv_vsseg5_mask,
13243 Intrinsic::riscv_vsseg6_mask, Intrinsic::riscv_vsseg7_mask,
13244 Intrinsic::riscv_vsseg8_mask,
13245 };
13246
13247 unsigned Sz =
13248 Factor * VecVT.getVectorMinNumElements() * VecVT.getScalarSizeInBits();
13249 EVT VecTupTy = MVT::getRISCVVectorTupleVT(Sz, NFields: Factor);
13250
13251 SDValue StoredVal = DAG.getUNDEF(VT: VecTupTy);
13252 for (unsigned i = 0; i < Factor; i++)
13253 StoredVal =
13254 DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: VecTupTy, N1: StoredVal,
13255 N2: Op.getOperand(i), N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
13256
13257 SDValue Ops[] = {DAG.getEntryNode(),
13258 DAG.getTargetConstant(Val: IntrIds[Factor - 2], DL, VT: XLenVT),
13259 StoredVal,
13260 StackPtr,
13261 Mask,
13262 VL,
13263 DAG.getTargetConstant(Val: Log2_64(Value: VecVT.getScalarSizeInBits()),
13264 DL, VT: XLenVT)};
13265
13266 SDValue Chain = DAG.getMemIntrinsicNode(
13267 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other), Ops,
13268 MemVT: VecVT.getVectorElementType(), PtrInfo, Alignment,
13269 Flags: MachineMemOperand::MOStore, Size: LocationSize::beforeOrAfterPointer());
13270
13271 SmallVector<SDValue, 8> Loads(Factor);
13272
13273 SDValue Increment = DAG.getTypeSize(DL, VT: PtrVT, TS: VecVT.getStoreSize());
13274 for (unsigned i = 0; i != Factor; ++i) {
13275 if (i != 0)
13276 StackPtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: Increment);
13277
13278 Loads[i] = DAG.getLoad(VT: VecVT, dl: DL, Chain, Ptr: StackPtr, PtrInfo);
13279 }
13280
13281 return DAG.getMergeValues(Ops: Loads, dl: DL);
13282 }
13283
13284 // Use ri.vzip2{a,b} if available
13285 // TODO: Figure out the best lowering for the spread variants
13286 if (Subtarget.hasVendorXRivosVizip() && !Op.getOperand(i: 0).isUndef() &&
13287 !Op.getOperand(i: 1).isUndef()) {
13288 // Freeze the sources so we can increase their use count.
13289 SDValue V1 = DAG.getFreeze(V: Op->getOperand(Num: 0));
13290 SDValue V2 = DAG.getFreeze(V: Op->getOperand(Num: 1));
13291 SDValue Lo = lowerVZIP(Opc: RISCVISD::RI_VZIP2A_VL, Op0: V1, Op1: V2, DL, DAG, Subtarget);
13292 SDValue Hi = lowerVZIP(Opc: RISCVISD::RI_VZIP2B_VL, Op0: V1, Op1: V2, DL, DAG, Subtarget);
13293 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
13294 }
13295
13296 // If the element type is smaller than ELEN, then we can interleave with
13297 // vwaddu.vv and vwmaccu.vx
13298 if (VecVT.getScalarSizeInBits() < Subtarget.getELen()) {
13299 Interleaved = getWideningInterleave(EvenV: Op.getOperand(i: 0), OddV: Op.getOperand(i: 1), DL,
13300 DAG, Subtarget);
13301 } else {
13302 // Otherwise, fallback to using vrgathere16.vv
13303 MVT ConcatVT =
13304 MVT::getVectorVT(VT: VecVT.getVectorElementType(),
13305 EC: VecVT.getVectorElementCount().multiplyCoefficientBy(RHS: 2));
13306 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ConcatVT,
13307 N1: Op.getOperand(i: 0), N2: Op.getOperand(i: 1));
13308
13309 MVT IdxVT = ConcatVT.changeVectorElementType(EltVT: MVT::i16);
13310
13311 // 0 1 2 3 4 5 6 7 ...
13312 SDValue StepVec = DAG.getStepVector(DL, ResVT: IdxVT);
13313
13314 // 1 1 1 1 1 1 1 1 ...
13315 SDValue Ones = DAG.getSplatVector(VT: IdxVT, DL, Op: DAG.getConstant(Val: 1, DL, VT: XLenVT));
13316
13317 // 1 0 1 0 1 0 1 0 ...
13318 SDValue OddMask = DAG.getNode(Opcode: ISD::AND, DL, VT: IdxVT, N1: StepVec, N2: Ones);
13319 OddMask = DAG.getSetCC(
13320 DL, VT: IdxVT.changeVectorElementType(EltVT: MVT::i1), LHS: OddMask,
13321 RHS: DAG.getSplatVector(VT: IdxVT, DL, Op: DAG.getConstant(Val: 0, DL, VT: XLenVT)),
13322 Cond: ISD::CondCode::SETNE);
13323
13324 SDValue VLMax = DAG.getSplatVector(VT: IdxVT, DL, Op: computeVLMax(VecVT, DL, DAG));
13325
13326 // Build up the index vector for interleaving the concatenated vector
13327 // 0 0 1 1 2 2 3 3 ...
13328 SDValue Idx = DAG.getNode(Opcode: ISD::SRL, DL, VT: IdxVT, N1: StepVec, N2: Ones);
13329 // 0 n 1 n+1 2 n+2 3 n+3 ...
13330 Idx =
13331 DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT: IdxVT, N1: Idx, N2: VLMax, N3: Idx, N4: OddMask, N5: VL);
13332
13333 // Then perform the interleave
13334 // v[0] v[n] v[1] v[n+1] v[2] v[n+2] v[3] v[n+3] ...
13335 SDValue TrueMask = getAllOnesMask(VecVT: IdxVT, VL, DL, DAG);
13336 Interleaved = DAG.getNode(Opcode: RISCVISD::VRGATHEREI16_VV_VL, DL, VT: ConcatVT,
13337 N1: Concat, N2: Idx, N3: DAG.getUNDEF(VT: ConcatVT), N4: TrueMask, N5: VL);
13338 }
13339
13340 // Extract the two halves from the interleaved result
13341 SDValue Lo = DAG.getExtractSubvector(DL, VT: VecVT, Vec: Interleaved, Idx: 0);
13342 SDValue Hi = DAG.getExtractSubvector(DL, VT: VecVT, Vec: Interleaved,
13343 Idx: VecVT.getVectorMinNumElements());
13344
13345 return DAG.getMergeValues(Ops: {Lo, Hi}, dl: DL);
13346}
13347
13348// Lower step_vector to the vid instruction. Any non-identity step value must
13349// be accounted for my manual expansion.
13350SDValue RISCVTargetLowering::lowerSTEP_VECTOR(SDValue Op,
13351 SelectionDAG &DAG) const {
13352 SDLoc DL(Op);
13353 MVT VT = Op.getSimpleValueType();
13354 assert(VT.isScalableVector() && "Expected scalable vector");
13355 MVT XLenVT = Subtarget.getXLenVT();
13356 auto [Mask, VL] = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
13357 SDValue StepVec = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT, N1: Mask, N2: VL);
13358 uint64_t StepValImm = Op.getConstantOperandVal(i: 0);
13359 if (StepValImm != 1) {
13360 if (isPowerOf2_64(Value: StepValImm)) {
13361 SDValue StepVal =
13362 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
13363 N2: DAG.getConstant(Val: Log2_64(Value: StepValImm), DL, VT: XLenVT), N3: VL);
13364 StepVec = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: StepVec, N2: StepVal);
13365 } else {
13366 SDValue StepVal = lowerScalarSplat(
13367 Passthru: SDValue(), Scalar: DAG.getConstant(Val: StepValImm, DL, VT: VT.getVectorElementType()),
13368 VL, VT, DL, DAG, Subtarget);
13369 StepVec = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: StepVec, N2: StepVal);
13370 }
13371 }
13372 return StepVec;
13373}
13374
13375// Implement vector_reverse using vrgather.vv with indices determined by
13376// subtracting the id of each element from (VLMAX-1). This will convert
13377// the indices like so:
13378// (0, 1,..., VLMAX-2, VLMAX-1) -> (VLMAX-1, VLMAX-2,..., 1, 0).
13379// TODO: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
13380SDValue RISCVTargetLowering::lowerVECTOR_REVERSE(SDValue Op,
13381 SelectionDAG &DAG) const {
13382 SDLoc DL(Op);
13383 MVT VecVT = Op.getSimpleValueType();
13384 if (VecVT.getVectorElementType() == MVT::i1) {
13385 MVT WidenVT = MVT::getVectorVT(VT: MVT::i8, EC: VecVT.getVectorElementCount());
13386 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: WidenVT, Operand: Op.getOperand(i: 0));
13387 SDValue Op2 = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: WidenVT, Operand: Op1);
13388 return DAG.getSetCC(DL, VT: VecVT, LHS: Op2,
13389 RHS: DAG.getConstant(Val: 0, DL, VT: Op2.getValueType()), Cond: ISD::SETNE);
13390 }
13391
13392 MVT ContainerVT = VecVT;
13393 SDValue Vec = Op.getOperand(i: 0);
13394 if (VecVT.isFixedLengthVector()) {
13395 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
13396 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
13397 }
13398
13399 MVT XLenVT = Subtarget.getXLenVT();
13400 auto [Mask, VL] = getDefaultVLOps(VecVT, ContainerVT, DL, DAG, Subtarget);
13401
13402 // On some uarchs vrgather.vv will read from every input register for each
13403 // output register, regardless of the indices. However to reverse a vector
13404 // each output register only needs to read from one register. So decompose it
13405 // into LMUL * M1 vrgather.vvs, so we get O(LMUL) performance instead of
13406 // O(LMUL^2).
13407 //
13408 // vsetvli a1, zero, e64, m4, ta, ma
13409 // vrgatherei16.vv v12, v8, v16
13410 // ->
13411 // vsetvli a1, zero, e64, m1, ta, ma
13412 // vrgather.vv v15, v8, v16
13413 // vrgather.vv v14, v9, v16
13414 // vrgather.vv v13, v10, v16
13415 // vrgather.vv v12, v11, v16
13416 if (ContainerVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: ContainerVT)) &&
13417 ContainerVT.getVectorElementCount().isKnownMultipleOf(RHS: 2)) {
13418 auto [Lo, Hi] = DAG.SplitVector(N: Vec, DL);
13419 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Lo.getValueType(), Operand: Lo);
13420 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: Hi.getValueType(), Operand: Hi);
13421 SDValue Concat = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: ContainerVT, N1: Hi, N2: Lo);
13422
13423 // Fixed length vectors might not fit exactly into their container, and so
13424 // leave a gap in the front of the vector after being reversed. Slide this
13425 // away.
13426 //
13427 // x x x x 3 2 1 0 <- v4i16 @ vlen=128
13428 // 0 1 2 3 x x x x <- reverse
13429 // x x x x 0 1 2 3 <- vslidedown.vx
13430 if (VecVT.isFixedLengthVector()) {
13431 SDValue Offset = DAG.getNode(
13432 Opcode: ISD::SUB, DL, VT: XLenVT,
13433 N1: DAG.getElementCount(DL, VT: XLenVT, EC: ContainerVT.getVectorElementCount()),
13434 N2: DAG.getElementCount(DL, VT: XLenVT, EC: VecVT.getVectorElementCount()));
13435 Concat =
13436 getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
13437 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Concat, Offset, Mask, VL);
13438 Concat = convertFromScalableVector(VT: VecVT, V: Concat, DAG, Subtarget);
13439 }
13440 return Concat;
13441 }
13442
13443 unsigned EltSize = ContainerVT.getScalarSizeInBits();
13444 unsigned MinSize = ContainerVT.getSizeInBits().getKnownMinValue();
13445 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
13446 unsigned MaxVLMAX =
13447 VecVT.isFixedLengthVector()
13448 ? VecVT.getVectorNumElements()
13449 : RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
13450
13451 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
13452 MVT IntVT = ContainerVT.changeVectorElementTypeToInteger();
13453
13454 // If this is SEW=8 and VLMAX is potentially more than 256, we need
13455 // to use vrgatherei16.vv.
13456 if (MaxVLMAX > 256 && EltSize == 8) {
13457 // If this is LMUL=8, we have to split before can use vrgatherei16.vv.
13458 // Reverse each half, then reassemble them in reverse order.
13459 // NOTE: It's also possible that after splitting that VLMAX no longer
13460 // requires vrgatherei16.vv.
13461 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
13462 auto [Lo, Hi] = DAG.SplitVectorOperand(N: Op.getNode(), OpNo: 0);
13463 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: VecVT);
13464 Lo = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: LoVT, Operand: Lo);
13465 Hi = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: HiVT, Operand: Hi);
13466 // Reassemble the low and high pieces reversed.
13467 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: VecVT, N1: Hi, N2: Lo);
13468 }
13469
13470 // Just promote the int type to i16 which will double the LMUL.
13471 IntVT = MVT::getVectorVT(VT: MVT::i16, EC: ContainerVT.getVectorElementCount());
13472 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
13473 }
13474
13475 // At LMUL > 1, do the index computation in 16 bits to reduce register
13476 // pressure.
13477 if (IntVT.getScalarType().bitsGT(VT: MVT::i16) &&
13478 IntVT.bitsGT(VT: RISCVTargetLowering::getM1VT(VT: IntVT))) {
13479 assert(isUInt<16>(MaxVLMAX - 1)); // Largest VLMAX is 65536 @ zvl65536b
13480 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
13481 IntVT = IntVT.changeVectorElementType(EltVT: MVT::i16);
13482 }
13483
13484 // Calculate VLMAX-1 for the desired SEW.
13485 SDValue VLMinus1 = DAG.getNode(
13486 Opcode: ISD::SUB, DL, VT: XLenVT,
13487 N1: DAG.getElementCount(DL, VT: XLenVT, EC: VecVT.getVectorElementCount()),
13488 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
13489
13490 // Splat VLMAX-1 taking care to handle SEW==64 on RV32.
13491 bool IsRV32E64 =
13492 !Subtarget.is64Bit() && IntVT.getVectorElementType() == MVT::i64;
13493 SDValue SplatVL;
13494 if (!IsRV32E64)
13495 SplatVL = DAG.getSplatVector(VT: IntVT, DL, Op: VLMinus1);
13496 else
13497 SplatVL = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IntVT, N1: DAG.getUNDEF(VT: IntVT),
13498 N2: VLMinus1, N3: DAG.getRegister(Reg: RISCV::X0, VT: XLenVT));
13499
13500 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: IntVT, N1: Mask, N2: VL);
13501 SDValue Indices = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: IntVT, N1: SplatVL, N2: VID,
13502 N3: DAG.getUNDEF(VT: IntVT), N4: Mask, N5: VL);
13503
13504 SDValue Gather = DAG.getNode(Opcode: GatherOpc, DL, VT: ContainerVT, N1: Vec, N2: Indices,
13505 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
13506 if (VecVT.isFixedLengthVector())
13507 Gather = convertFromScalableVector(VT: VecVT, V: Gather, DAG, Subtarget);
13508 return Gather;
13509}
13510
13511SDValue RISCVTargetLowering::lowerVECTOR_SPLICE(SDValue Op,
13512 SelectionDAG &DAG) const {
13513 SDLoc DL(Op);
13514 SDValue V1 = Op.getOperand(i: 0);
13515 SDValue V2 = Op.getOperand(i: 1);
13516 SDValue Offset = Op.getOperand(i: 2);
13517 MVT XLenVT = Subtarget.getXLenVT();
13518 MVT VecVT = Op.getSimpleValueType();
13519
13520 SDValue VLMax = computeVLMax(VecVT, DL, DAG);
13521
13522 SDValue DownOffset, UpOffset;
13523 if (Op.getOpcode() == ISD::VECTOR_SPLICE_LEFT) {
13524 // The operand is a TargetConstant, we need to rebuild it as a regular
13525 // constant.
13526 DownOffset = Offset;
13527 UpOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: Offset);
13528 } else {
13529 // The operand is a TargetConstant, we need to rebuild it as a regular
13530 // constant rather than negating the original operand.
13531 UpOffset = Offset;
13532 DownOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: Offset);
13533 }
13534
13535 SDValue TrueMask = getAllOnesMask(VecVT, VL: VLMax, DL, DAG);
13536
13537 SDValue SlideDown = getVSlidedown(
13538 DAG, Subtarget, DL, VT: VecVT, Passthru: DAG.getUNDEF(VT: VecVT), Op: V1, Offset: DownOffset, Mask: TrueMask,
13539 VL: Subtarget.hasVLDependentLatency() ? UpOffset
13540 : DAG.getRegister(Reg: RISCV::X0, VT: XLenVT));
13541 return getVSlideup(DAG, Subtarget, DL, VT: VecVT, Passthru: SlideDown, Op: V2, Offset: UpOffset,
13542 Mask: TrueMask, VL: DAG.getRegister(Reg: RISCV::X0, VT: XLenVT),
13543 Policy: RISCVVType::TAIL_AGNOSTIC);
13544}
13545
13546SDValue
13547RISCVTargetLowering::lowerFixedLengthVectorLoadToRVV(SDValue Op,
13548 SelectionDAG &DAG) const {
13549 SDLoc DL(Op);
13550 auto *Load = cast<LoadSDNode>(Val&: Op);
13551
13552 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
13553 Load->getMemoryVT(),
13554 *Load->getMemOperand()) &&
13555 "Expecting a correctly-aligned load");
13556
13557 MVT VT = Op.getSimpleValueType();
13558 MVT XLenVT = Subtarget.getXLenVT();
13559 MVT ContainerVT = getContainerForFixedLengthVector(VT);
13560
13561 // If we know the exact VLEN and our fixed length vector completely fills
13562 // the container, use a whole register load instead.
13563 const auto [MinVLMAX, MaxVLMAX] =
13564 RISCVTargetLowering::computeVLMAXBounds(VecVT: ContainerVT, Subtarget);
13565 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
13566 RISCVTargetLowering::getM1VT(VT: ContainerVT).bitsLE(VT: ContainerVT)) {
13567 MachineMemOperand *MMO = Load->getMemOperand();
13568 SDValue NewLoad =
13569 DAG.getLoad(VT: ContainerVT, dl: DL, Chain: Load->getChain(), Ptr: Load->getBasePtr(),
13570 PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getBaseAlign(), MMOFlags: MMO->getFlags(),
13571 AAInfo: MMO->getAAInfo(), Ranges: MMO->getRanges());
13572 SDValue Result = convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
13573 return DAG.getMergeValues(Ops: {Result, NewLoad.getValue(R: 1)}, dl: DL);
13574 }
13575
13576 SDValue VL = DAG.getConstant(Val: VT.getVectorNumElements(), DL, VT: XLenVT);
13577
13578 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
13579 SDValue IntID = DAG.getTargetConstant(
13580 Val: IsMaskOp ? Intrinsic::riscv_vlm : Intrinsic::riscv_vle, DL, VT: XLenVT);
13581 SmallVector<SDValue, 4> Ops{Load->getChain(), IntID};
13582 if (!IsMaskOp)
13583 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
13584 Ops.push_back(Elt: Load->getBasePtr());
13585 Ops.push_back(Elt: VL);
13586 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
13587 SDValue NewLoad =
13588 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
13589 MemVT: Load->getMemoryVT(), MMO: Load->getMemOperand());
13590
13591 SDValue Result = convertFromScalableVector(VT, V: NewLoad, DAG, Subtarget);
13592 return DAG.getMergeValues(Ops: {Result, NewLoad.getValue(R: 1)}, dl: DL);
13593}
13594
13595SDValue
13596RISCVTargetLowering::lowerFixedLengthVectorStoreToRVV(SDValue Op,
13597 SelectionDAG &DAG) const {
13598 SDLoc DL(Op);
13599 auto *Store = cast<StoreSDNode>(Val&: Op);
13600
13601 assert(allowsMemoryAccessForAlignment(*DAG.getContext(), DAG.getDataLayout(),
13602 Store->getMemoryVT(),
13603 *Store->getMemOperand()) &&
13604 "Expecting a correctly-aligned store");
13605
13606 SDValue StoreVal = Store->getValue();
13607 MVT VT = StoreVal.getSimpleValueType();
13608 MVT XLenVT = Subtarget.getXLenVT();
13609
13610 // If the size less than a byte, we need to pad with zeros to make a byte.
13611 if (VT.getVectorElementType() == MVT::i1 && VT.getVectorNumElements() < 8) {
13612 VT = MVT::v8i1;
13613 StoreVal =
13614 DAG.getInsertSubvector(DL, Vec: DAG.getConstant(Val: 0, DL, VT), SubVec: StoreVal, Idx: 0);
13615 }
13616
13617 MVT ContainerVT = getContainerForFixedLengthVector(VT);
13618
13619 SDValue NewValue =
13620 convertToScalableVector(VT: ContainerVT, V: StoreVal, DAG, Subtarget);
13621
13622 // If we know the exact VLEN and our fixed length vector completely fills
13623 // the container, use a whole register store instead.
13624 const auto [MinVLMAX, MaxVLMAX] =
13625 RISCVTargetLowering::computeVLMAXBounds(VecVT: ContainerVT, Subtarget);
13626 if (MinVLMAX == MaxVLMAX && MinVLMAX == VT.getVectorNumElements() &&
13627 RISCVTargetLowering::getM1VT(VT: ContainerVT).bitsLE(VT: ContainerVT)) {
13628 MachineMemOperand *MMO = Store->getMemOperand();
13629 return DAG.getStore(Chain: Store->getChain(), dl: DL, Val: NewValue, Ptr: Store->getBasePtr(),
13630 PtrInfo: MMO->getPointerInfo(), Alignment: MMO->getBaseAlign(),
13631 MMOFlags: MMO->getFlags(), AAInfo: MMO->getAAInfo());
13632 }
13633
13634 SDValue VL = DAG.getConstant(Val: VT.getVectorNumElements(), DL, VT: XLenVT);
13635
13636 bool IsMaskOp = VT.getVectorElementType() == MVT::i1;
13637 SDValue IntID = DAG.getTargetConstant(
13638 Val: IsMaskOp ? Intrinsic::riscv_vsm : Intrinsic::riscv_vse, DL, VT: XLenVT);
13639 return DAG.getMemIntrinsicNode(
13640 Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: DAG.getVTList(VT: MVT::Other),
13641 Ops: {Store->getChain(), IntID, NewValue, Store->getBasePtr(), VL},
13642 MemVT: Store->getMemoryVT(), MMO: Store->getMemOperand());
13643}
13644
13645SDValue RISCVTargetLowering::lowerMaskedLoad(SDValue Op,
13646 SelectionDAG &DAG) const {
13647 SDLoc DL(Op);
13648 MVT VT = Op.getSimpleValueType();
13649
13650 const auto *MemSD = cast<MemSDNode>(Val&: Op);
13651 EVT MemVT = MemSD->getMemoryVT();
13652 MachineMemOperand *MMO = MemSD->getMemOperand();
13653 SDValue Chain = MemSD->getChain();
13654 SDValue BasePtr = MemSD->getBasePtr();
13655
13656 SDValue Mask, PassThru, VL;
13657 bool IsExpandingLoad = false;
13658 if (const auto *VPLoad = dyn_cast<VPLoadSDNode>(Val&: Op)) {
13659 Mask = VPLoad->getMask();
13660 PassThru = DAG.getUNDEF(VT);
13661 VL = VPLoad->getVectorLength();
13662 } else {
13663 const auto *MLoad = cast<MaskedLoadSDNode>(Val&: Op);
13664 Mask = MLoad->getMask();
13665 PassThru = MLoad->getPassThru();
13666 IsExpandingLoad = MLoad->isExpandingLoad();
13667 }
13668
13669 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
13670
13671 MVT XLenVT = Subtarget.getXLenVT();
13672
13673 MVT ContainerVT = VT;
13674 if (VT.isFixedLengthVector()) {
13675 ContainerVT = getContainerForFixedLengthVector(VT);
13676 PassThru = convertToScalableVector(VT: ContainerVT, V: PassThru, DAG, Subtarget);
13677 if (!IsUnmasked) {
13678 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
13679 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
13680 }
13681 }
13682
13683 if (!VL)
13684 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
13685
13686 SDValue ExpandingVL;
13687 if (!IsUnmasked && IsExpandingLoad) {
13688 ExpandingVL = VL;
13689 VL =
13690 DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Mask,
13691 N2: getAllOnesMask(VecVT: Mask.getSimpleValueType(), VL, DL, DAG), N3: VL);
13692 }
13693
13694 unsigned IntID = IsUnmasked || IsExpandingLoad ? Intrinsic::riscv_vle
13695 : Intrinsic::riscv_vle_mask;
13696 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
13697 if (IntID == Intrinsic::riscv_vle)
13698 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
13699 else
13700 Ops.push_back(Elt: PassThru);
13701 Ops.push_back(Elt: BasePtr);
13702 if (IntID == Intrinsic::riscv_vle_mask)
13703 Ops.push_back(Elt: Mask);
13704 Ops.push_back(Elt: VL);
13705 if (IntID == Intrinsic::riscv_vle_mask)
13706 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT));
13707
13708 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
13709
13710 SDValue Result =
13711 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
13712 Chain = Result.getValue(R: 1);
13713 if (ExpandingVL) {
13714 MVT IndexVT = ContainerVT;
13715 if (ContainerVT.isFloatingPoint())
13716 IndexVT = ContainerVT.changeVectorElementTypeToInteger();
13717
13718 MVT IndexEltVT = IndexVT.getVectorElementType();
13719 bool UseVRGATHEREI16 = false;
13720 // If index vector is an i8 vector and the element count exceeds 256, we
13721 // should change the element type of index vector to i16 to avoid
13722 // overflow.
13723 if (IndexEltVT == MVT::i8 && VT.getVectorNumElements() > 256) {
13724 // FIXME: We need to do vector splitting manually for LMUL=8 cases.
13725 assert(getLMUL(IndexVT) != RISCVVType::LMUL_8);
13726 IndexVT = IndexVT.changeVectorElementType(EltVT: MVT::i16);
13727 UseVRGATHEREI16 = true;
13728 }
13729
13730 SDValue Iota =
13731 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: IndexVT,
13732 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_viota, DL, VT: XLenVT),
13733 N2: DAG.getUNDEF(VT: IndexVT), N3: Mask, N4: ExpandingVL);
13734 Result =
13735 DAG.getNode(Opcode: UseVRGATHEREI16 ? RISCVISD::VRGATHEREI16_VV_VL
13736 : RISCVISD::VRGATHER_VV_VL,
13737 DL, VT: ContainerVT, N1: Result, N2: Iota, N3: PassThru, N4: Mask, N5: ExpandingVL);
13738 }
13739
13740 if (VT.isFixedLengthVector())
13741 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
13742
13743 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
13744}
13745
13746SDValue RISCVTargetLowering::lowerLoadFF(SDValue Op, SelectionDAG &DAG) const {
13747 SDLoc DL(Op);
13748 MVT VT = Op->getSimpleValueType(ResNo: 0);
13749
13750 const auto *VPLoadFF = cast<VPLoadFFSDNode>(Val&: Op);
13751 EVT MemVT = VPLoadFF->getMemoryVT();
13752 MachineMemOperand *MMO = VPLoadFF->getMemOperand();
13753 SDValue Chain = VPLoadFF->getChain();
13754 SDValue BasePtr = VPLoadFF->getBasePtr();
13755
13756 SDValue Mask = VPLoadFF->getMask();
13757 SDValue VL = VPLoadFF->getVectorLength();
13758
13759 MVT XLenVT = Subtarget.getXLenVT();
13760
13761 MVT ContainerVT = VT;
13762 if (VT.isFixedLengthVector()) {
13763 ContainerVT = getContainerForFixedLengthVector(VT);
13764 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
13765 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
13766 }
13767
13768 unsigned IntID = Intrinsic::riscv_vleff_mask;
13769 SDValue Ops[] = {
13770 Chain,
13771 DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT),
13772 DAG.getUNDEF(VT: ContainerVT),
13773 BasePtr,
13774 Mask,
13775 VL,
13776 DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT)};
13777
13778 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, Op->getValueType(ResNo: 1), MVT::Other});
13779
13780 SDValue Result =
13781 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
13782 SDValue OutVL = Result.getValue(R: 1);
13783 Chain = Result.getValue(R: 2);
13784
13785 if (VT.isFixedLengthVector())
13786 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
13787
13788 return DAG.getMergeValues(Ops: {Result, OutVL, Chain}, dl: DL);
13789}
13790
13791SDValue RISCVTargetLowering::lowerMaskedStore(SDValue Op,
13792 SelectionDAG &DAG) const {
13793 SDLoc DL(Op);
13794
13795 const auto *MemSD = cast<MemSDNode>(Val&: Op);
13796 EVT MemVT = MemSD->getMemoryVT();
13797 MachineMemOperand *MMO = MemSD->getMemOperand();
13798 SDValue Chain = MemSD->getChain();
13799 SDValue BasePtr = MemSD->getBasePtr();
13800 SDValue Val, Mask, VL;
13801
13802 bool IsCompressingStore = false;
13803 if (const auto *VPStore = dyn_cast<VPStoreSDNode>(Val&: Op)) {
13804 Val = VPStore->getValue();
13805 Mask = VPStore->getMask();
13806 VL = VPStore->getVectorLength();
13807 } else {
13808 const auto *MStore = cast<MaskedStoreSDNode>(Val&: Op);
13809 Val = MStore->getValue();
13810 Mask = MStore->getMask();
13811 IsCompressingStore = MStore->isCompressingStore();
13812 }
13813
13814 bool IsUnmasked =
13815 ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()) || IsCompressingStore;
13816
13817 MVT VT = Val.getSimpleValueType();
13818 MVT XLenVT = Subtarget.getXLenVT();
13819
13820 MVT ContainerVT = VT;
13821 if (VT.isFixedLengthVector()) {
13822 ContainerVT = getContainerForFixedLengthVector(VT);
13823
13824 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
13825 if (!IsUnmasked || IsCompressingStore) {
13826 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
13827 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
13828 }
13829 }
13830
13831 if (!VL)
13832 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
13833
13834 if (IsCompressingStore) {
13835 Val = DAG.getNode(
13836 Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ContainerVT,
13837 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_vcompress, DL, VT: XLenVT),
13838 N2: DAG.getUNDEF(VT: ContainerVT), N3: Val, N4: Mask, N5: VL);
13839 VL =
13840 DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Mask,
13841 N2: getAllOnesMask(VecVT: Mask.getSimpleValueType(), VL, DL, DAG), N3: VL);
13842 }
13843
13844 unsigned IntID =
13845 IsUnmasked ? Intrinsic::riscv_vse : Intrinsic::riscv_vse_mask;
13846 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
13847 Ops.push_back(Elt: Val);
13848 Ops.push_back(Elt: BasePtr);
13849 if (!IsUnmasked)
13850 Ops.push_back(Elt: Mask);
13851 Ops.push_back(Elt: VL);
13852
13853 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
13854 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
13855}
13856
13857SDValue RISCVTargetLowering::lowerVectorCompress(SDValue Op,
13858 SelectionDAG &DAG) const {
13859 SDLoc DL(Op);
13860 SDValue Val = Op.getOperand(i: 0);
13861 SDValue Mask = Op.getOperand(i: 1);
13862 SDValue Passthru = Op.getOperand(i: 2);
13863
13864 MVT VT = Val.getSimpleValueType();
13865 MVT XLenVT = Subtarget.getXLenVT();
13866 MVT ContainerVT = VT;
13867 if (VT.isFixedLengthVector()) {
13868 ContainerVT = getContainerForFixedLengthVector(VT);
13869 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
13870 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
13871 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
13872 Passthru = convertToScalableVector(VT: ContainerVT, V: Passthru, DAG, Subtarget);
13873 }
13874
13875 SDValue VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
13876 SDValue Res =
13877 DAG.getNode(Opcode: ISD::INTRINSIC_WO_CHAIN, DL, VT: ContainerVT,
13878 N1: DAG.getTargetConstant(Val: Intrinsic::riscv_vcompress, DL, VT: XLenVT),
13879 N2: Passthru, N3: Val, N4: Mask, N5: VL);
13880
13881 if (VT.isFixedLengthVector())
13882 Res = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
13883
13884 return Res;
13885}
13886
13887SDValue RISCVTargetLowering::lowerVectorStrictFSetcc(SDValue Op,
13888 SelectionDAG &DAG) const {
13889 unsigned Opc = Op.getOpcode();
13890 SDLoc DL(Op);
13891 SDValue Chain = Op.getOperand(i: 0);
13892 SDValue Op1 = Op.getOperand(i: 1);
13893 SDValue Op2 = Op.getOperand(i: 2);
13894 SDValue CC = Op.getOperand(i: 3);
13895 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
13896 MVT VT = Op.getSimpleValueType();
13897 MVT InVT = Op1.getSimpleValueType();
13898
13899 // RVV VMFEQ/VMFNE ignores qNan, so we expand strict_fsetccs with OEQ/UNE
13900 // condition code.
13901 if (Opc == ISD::STRICT_FSETCCS) {
13902 // Expand strict_fsetccs(x, oeq) to
13903 // (and strict_fsetccs(x, y, oge), strict_fsetccs(x, y, ole))
13904 SDVTList VTList = Op->getVTList();
13905 if (CCVal == ISD::SETEQ || CCVal == ISD::SETOEQ) {
13906 SDValue OLECCVal = DAG.getCondCode(Cond: ISD::SETOLE);
13907 SDValue Tmp1 = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op1,
13908 N3: Op2, N4: OLECCVal);
13909 SDValue Tmp2 = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op2,
13910 N3: Op1, N4: OLECCVal);
13911 SDValue OutChain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other,
13912 N1: Tmp1.getValue(R: 1), N2: Tmp2.getValue(R: 1));
13913 // Tmp1 and Tmp2 might be the same node.
13914 if (Tmp1 != Tmp2)
13915 Tmp1 = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Tmp1, N2: Tmp2);
13916 return DAG.getMergeValues(Ops: {Tmp1, OutChain}, dl: DL);
13917 }
13918
13919 // Expand (strict_fsetccs x, y, une) to (not (strict_fsetccs x, y, oeq))
13920 if (CCVal == ISD::SETNE || CCVal == ISD::SETUNE) {
13921 SDValue OEQCCVal = DAG.getCondCode(Cond: ISD::SETOEQ);
13922 SDValue OEQ = DAG.getNode(Opcode: ISD::STRICT_FSETCCS, DL, VTList, N1: Chain, N2: Op1,
13923 N3: Op2, N4: OEQCCVal);
13924 SDValue Res = DAG.getNOT(DL, Val: OEQ, VT);
13925 return DAG.getMergeValues(Ops: {Res, OEQ.getValue(R: 1)}, dl: DL);
13926 }
13927 }
13928
13929 MVT ContainerInVT = InVT;
13930 if (InVT.isFixedLengthVector()) {
13931 ContainerInVT = getContainerForFixedLengthVector(VT: InVT);
13932 Op1 = convertToScalableVector(VT: ContainerInVT, V: Op1, DAG, Subtarget);
13933 Op2 = convertToScalableVector(VT: ContainerInVT, V: Op2, DAG, Subtarget);
13934 }
13935 MVT MaskVT = getMaskTypeFor(VecVT: ContainerInVT);
13936
13937 auto [Mask, VL] = getDefaultVLOps(VecVT: InVT, ContainerVT: ContainerInVT, DL, DAG, Subtarget);
13938
13939 SDValue Res;
13940 if (Opc == ISD::STRICT_FSETCC &&
13941 (CCVal == ISD::SETLT || CCVal == ISD::SETOLT || CCVal == ISD::SETLE ||
13942 CCVal == ISD::SETOLE)) {
13943 // VMFLT/VMFLE/VMFGT/VMFGE raise exception for qNan. Generate a mask to only
13944 // active when both input elements are ordered.
13945 SDValue True = getAllOnesMask(VecVT: ContainerInVT, VL, DL, DAG);
13946 SDValue OrderMask1 = DAG.getNode(
13947 Opcode: RISCVISD::STRICT_FSETCC_VL, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
13948 Ops: {Chain, Op1, Op1, DAG.getCondCode(Cond: ISD::SETOEQ), DAG.getUNDEF(VT: MaskVT),
13949 True, VL});
13950 SDValue OrderMask2 = DAG.getNode(
13951 Opcode: RISCVISD::STRICT_FSETCC_VL, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
13952 Ops: {Chain, Op2, Op2, DAG.getCondCode(Cond: ISD::SETOEQ), DAG.getUNDEF(VT: MaskVT),
13953 True, VL});
13954 Mask =
13955 DAG.getNode(Opcode: RISCVISD::VMAND_VL, DL, VT: MaskVT, N1: OrderMask1, N2: OrderMask2, N3: VL);
13956 // Use Mask as the passthru operand to let the result be 0 if either of the
13957 // inputs is unordered.
13958 Res = DAG.getNode(Opcode: RISCVISD::STRICT_FSETCCS_VL, DL,
13959 VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
13960 Ops: {Chain, Op1, Op2, CC, Mask, Mask, VL});
13961 } else {
13962 unsigned RVVOpc = Opc == ISD::STRICT_FSETCC ? RISCVISD::STRICT_FSETCC_VL
13963 : RISCVISD::STRICT_FSETCCS_VL;
13964 Res = DAG.getNode(Opcode: RVVOpc, DL, VTList: DAG.getVTList(VT1: MaskVT, VT2: MVT::Other),
13965 Ops: {Chain, Op1, Op2, CC, DAG.getUNDEF(VT: MaskVT), Mask, VL});
13966 }
13967
13968 if (VT.isFixedLengthVector()) {
13969 SDValue SubVec = convertFromScalableVector(VT, V: Res, DAG, Subtarget);
13970 return DAG.getMergeValues(Ops: {SubVec, Res.getValue(R: 1)}, dl: DL);
13971 }
13972 return Res;
13973}
13974
13975// Lower vector ABS to smax(X, sub(0, X)).
13976SDValue RISCVTargetLowering::lowerABS(SDValue Op, SelectionDAG &DAG) const {
13977 SDLoc DL(Op);
13978 MVT VT = Op.getSimpleValueType();
13979 SDValue X = Op.getOperand(i: 0);
13980
13981 assert((Op.getOpcode() == ISD::VP_ABS || VT.isFixedLengthVector()) &&
13982 "Unexpected type for ISD::ABS");
13983
13984 MVT ContainerVT = VT;
13985 if (VT.isFixedLengthVector()) {
13986 ContainerVT = getContainerForFixedLengthVector(VT);
13987 X = convertToScalableVector(VT: ContainerVT, V: X, DAG, Subtarget);
13988 }
13989
13990 SDValue Mask, VL;
13991 if (Op->getOpcode() == ISD::VP_ABS) {
13992 Mask = Op->getOperand(Num: 1);
13993 if (VT.isFixedLengthVector())
13994 Mask = convertToScalableVector(VT: getMaskTypeFor(VecVT: ContainerVT), V: Mask, DAG,
13995 Subtarget);
13996 VL = Op->getOperand(Num: 2);
13997 } else
13998 std::tie(args&: Mask, args&: VL) = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
13999
14000 SDValue Result;
14001 if (Subtarget.hasStdExtZvabd()) {
14002 Result = DAG.getNode(Opcode: RISCVISD::ABS_VL, DL, VT: ContainerVT, N1: X,
14003 N2: DAG.getUNDEF(VT: ContainerVT), N3: Mask, N4: VL);
14004 } else {
14005 SDValue SplatZero = DAG.getNode(
14006 Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT),
14007 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()), N3: VL);
14008 SDValue NegX = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: ContainerVT, N1: SplatZero, N2: X,
14009 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
14010 Result = DAG.getNode(Opcode: RISCVISD::SMAX_VL, DL, VT: ContainerVT, N1: X, N2: NegX,
14011 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
14012 }
14013 if (VT.isFixedLengthVector())
14014 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14015 return Result;
14016}
14017
14018SDValue RISCVTargetLowering::lowerToScalableOp(SDValue Op,
14019 SelectionDAG &DAG) const {
14020 const auto &TSInfo =
14021 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
14022
14023 unsigned NewOpc = getRISCVVLOp(Op);
14024 bool HasPassthruOp = TSInfo.hasPassthruOp(Opcode: NewOpc);
14025 bool HasMask = TSInfo.hasMaskOp(Opcode: NewOpc);
14026
14027 MVT VT = Op.getSimpleValueType();
14028 MVT ContainerVT = getContainerForFixedLengthVector(VT);
14029
14030 // Create list of operands by converting existing ones to scalable types.
14031 SmallVector<SDValue, 6> Ops;
14032 for (const SDValue &V : Op->op_values()) {
14033 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
14034
14035 // Pass through non-vector operands.
14036 if (!V.getValueType().isVector()) {
14037 Ops.push_back(Elt: V);
14038 continue;
14039 }
14040
14041 // "cast" fixed length vector to a scalable vector.
14042 assert(useRVVForFixedLengthVectorVT(V.getSimpleValueType()) &&
14043 "Only fixed length vectors are supported!");
14044 MVT VContainerVT = ContainerVT.changeVectorElementType(
14045 EltVT: V.getSimpleValueType().getVectorElementType());
14046 Ops.push_back(Elt: convertToScalableVector(VT: VContainerVT, V, DAG, Subtarget));
14047 }
14048
14049 SDLoc DL(Op);
14050 auto [Mask, VL] = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget);
14051 if (HasPassthruOp)
14052 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14053 if (HasMask)
14054 Ops.push_back(Elt: Mask);
14055 Ops.push_back(Elt: VL);
14056
14057 // StrictFP operations have two result values. Their lowered result should
14058 // have same result count.
14059 if (Op->isStrictFPOpcode()) {
14060 SDValue ScalableRes =
14061 DAG.getNode(Opcode: NewOpc, DL, VTList: DAG.getVTList(VT1: ContainerVT, VT2: MVT::Other), Ops,
14062 Flags: Op->getFlags());
14063 SDValue SubVec = convertFromScalableVector(VT, V: ScalableRes, DAG, Subtarget);
14064 return DAG.getMergeValues(Ops: {SubVec, ScalableRes.getValue(R: 1)}, dl: DL);
14065 }
14066
14067 SDValue ScalableRes =
14068 DAG.getNode(Opcode: NewOpc, DL, VT: ContainerVT, Ops, Flags: Op->getFlags());
14069 return convertFromScalableVector(VT, V: ScalableRes, DAG, Subtarget);
14070}
14071
14072// Lower a VP_* ISD node to the corresponding RISCVISD::*_VL node:
14073// * Operands of each node are assumed to be in the same order.
14074// * The EVL operand is promoted from i32 to i64 on RV64.
14075// * Fixed-length vectors are converted to their scalable-vector container
14076// types.
14077SDValue RISCVTargetLowering::lowerVPOp(SDValue Op, SelectionDAG &DAG) const {
14078 const auto &TSInfo =
14079 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
14080
14081 unsigned RISCVISDOpc = getRISCVVLOp(Op);
14082 bool HasPassthruOp = TSInfo.hasPassthruOp(Opcode: RISCVISDOpc);
14083
14084 SDLoc DL(Op);
14085 MVT VT = Op.getSimpleValueType();
14086 SmallVector<SDValue, 4> Ops;
14087
14088 MVT ContainerVT = VT;
14089 if (VT.isFixedLengthVector())
14090 ContainerVT = getContainerForFixedLengthVector(VT);
14091
14092 for (const auto &OpIdx : enumerate(First: Op->ops())) {
14093 SDValue V = OpIdx.value();
14094 assert(!isa<VTSDNode>(V) && "Unexpected VTSDNode node!");
14095 // Add dummy passthru value before the mask. Or if there isn't a mask,
14096 // before EVL.
14097 if (HasPassthruOp) {
14098 auto MaskIdx = ISD::getVPMaskIdx(Opcode: Op.getOpcode());
14099 if (MaskIdx) {
14100 if (*MaskIdx == OpIdx.index())
14101 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14102 } else if (ISD::getVPExplicitVectorLengthIdx(Opcode: Op.getOpcode()) ==
14103 OpIdx.index()) {
14104 if (Op.getOpcode() == ISD::VP_MERGE) {
14105 // For VP_MERGE, copy the false operand instead of an undef value.
14106 Ops.push_back(Elt: Ops.back());
14107 } else {
14108 assert(Op.getOpcode() == ISD::VP_SELECT);
14109 // For VP_SELECT, add an undef value.
14110 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14111 }
14112 }
14113 }
14114 // VFCVT_RM_X_F_VL requires a rounding mode to be injected before the VL.
14115 if (RISCVISDOpc == RISCVISD::VFCVT_RM_X_F_VL &&
14116 ISD::getVPExplicitVectorLengthIdx(Opcode: Op.getOpcode()) == OpIdx.index())
14117 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVFPRndMode::DYN, DL,
14118 VT: Subtarget.getXLenVT()));
14119 // Pass through operands which aren't fixed-length vectors.
14120 if (!V.getValueType().isFixedLengthVector()) {
14121 Ops.push_back(Elt: V);
14122 continue;
14123 }
14124 // "cast" fixed length vector to a scalable vector.
14125 MVT OpVT = V.getSimpleValueType();
14126 MVT ContainerVT = getContainerForFixedLengthVector(VT: OpVT);
14127 assert(useRVVForFixedLengthVectorVT(OpVT) &&
14128 "Only fixed length vectors are supported!");
14129 Ops.push_back(Elt: convertToScalableVector(VT: ContainerVT, V, DAG, Subtarget));
14130 }
14131
14132 if (!VT.isFixedLengthVector())
14133 return DAG.getNode(Opcode: RISCVISDOpc, DL, VT, Ops, Flags: Op->getFlags());
14134
14135 SDValue VPOp = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: ContainerVT, Ops, Flags: Op->getFlags());
14136
14137 return convertFromScalableVector(VT, V: VPOp, DAG, Subtarget);
14138}
14139
14140SDValue RISCVTargetLowering::lowerVPExtMaskOp(SDValue Op,
14141 SelectionDAG &DAG) const {
14142 SDLoc DL(Op);
14143 MVT VT = Op.getSimpleValueType();
14144
14145 SDValue Src = Op.getOperand(i: 0);
14146 // NOTE: Mask is dropped.
14147 SDValue VL = Op.getOperand(i: 2);
14148
14149 MVT ContainerVT = VT;
14150 if (VT.isFixedLengthVector()) {
14151 ContainerVT = getContainerForFixedLengthVector(VT);
14152 MVT SrcVT = MVT::getVectorVT(VT: MVT::i1, EC: ContainerVT.getVectorElementCount());
14153 Src = convertToScalableVector(VT: SrcVT, V: Src, DAG, Subtarget);
14154 }
14155
14156 MVT XLenVT = Subtarget.getXLenVT();
14157 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
14158 SDValue ZeroSplat = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14159 N1: DAG.getUNDEF(VT: ContainerVT), N2: Zero, N3: VL);
14160
14161 SDValue SplatValue = DAG.getSignedConstant(
14162 Val: Op.getOpcode() == ISD::VP_ZERO_EXTEND ? 1 : -1, DL, VT: XLenVT);
14163 SDValue Splat = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14164 N1: DAG.getUNDEF(VT: ContainerVT), N2: SplatValue, N3: VL);
14165
14166 SDValue Result = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: Src, N2: Splat,
14167 N3: ZeroSplat, N4: DAG.getUNDEF(VT: ContainerVT), N5: VL);
14168 if (!VT.isFixedLengthVector())
14169 return Result;
14170 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14171}
14172
14173SDValue RISCVTargetLowering::lowerVPSetCCMaskOp(SDValue Op,
14174 SelectionDAG &DAG) const {
14175 SDLoc DL(Op);
14176 MVT VT = Op.getSimpleValueType();
14177
14178 SDValue Op1 = Op.getOperand(i: 0);
14179 SDValue Op2 = Op.getOperand(i: 1);
14180 ISD::CondCode Condition = cast<CondCodeSDNode>(Val: Op.getOperand(i: 2))->get();
14181 // NOTE: Mask is dropped.
14182 SDValue VL = Op.getOperand(i: 4);
14183
14184 MVT ContainerVT = VT;
14185 if (VT.isFixedLengthVector()) {
14186 ContainerVT = getContainerForFixedLengthVector(VT);
14187 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
14188 Op2 = convertToScalableVector(VT: ContainerVT, V: Op2, DAG, Subtarget);
14189 }
14190
14191 SDValue Result;
14192 SDValue AllOneMask = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
14193
14194 switch (Condition) {
14195 default:
14196 break;
14197 // X != Y --> (X^Y)
14198 case ISD::SETNE:
14199 Result = DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Op1, N2: Op2, N3: VL);
14200 break;
14201 // X == Y --> ~(X^Y)
14202 case ISD::SETEQ: {
14203 SDValue Temp =
14204 DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Op1, N2: Op2, N3: VL);
14205 Result =
14206 DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Temp, N2: AllOneMask, N3: VL);
14207 break;
14208 }
14209 // X >s Y --> X == 0 & Y == 1 --> ~X & Y
14210 // X <u Y --> X == 0 & Y == 1 --> ~X & Y
14211 case ISD::SETGT:
14212 case ISD::SETULT: {
14213 SDValue Temp =
14214 DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Op1, N2: AllOneMask, N3: VL);
14215 Result = DAG.getNode(Opcode: RISCVISD::VMAND_VL, DL, VT: ContainerVT, N1: Temp, N2: Op2, N3: VL);
14216 break;
14217 }
14218 // X <s Y --> X == 1 & Y == 0 --> ~Y & X
14219 // X >u Y --> X == 1 & Y == 0 --> ~Y & X
14220 case ISD::SETLT:
14221 case ISD::SETUGT: {
14222 SDValue Temp =
14223 DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Op2, N2: AllOneMask, N3: VL);
14224 Result = DAG.getNode(Opcode: RISCVISD::VMAND_VL, DL, VT: ContainerVT, N1: Op1, N2: Temp, N3: VL);
14225 break;
14226 }
14227 // X >=s Y --> X == 0 | Y == 1 --> ~X | Y
14228 // X <=u Y --> X == 0 | Y == 1 --> ~X | Y
14229 case ISD::SETGE:
14230 case ISD::SETULE: {
14231 SDValue Temp =
14232 DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Op1, N2: AllOneMask, N3: VL);
14233 Result = DAG.getNode(Opcode: RISCVISD::VMOR_VL, DL, VT: ContainerVT, N1: Temp, N2: Op2, N3: VL);
14234 break;
14235 }
14236 // X <=s Y --> X == 1 | Y == 0 --> ~Y | X
14237 // X >=u Y --> X == 1 | Y == 0 --> ~Y | X
14238 case ISD::SETLE:
14239 case ISD::SETUGE: {
14240 SDValue Temp =
14241 DAG.getNode(Opcode: RISCVISD::VMXOR_VL, DL, VT: ContainerVT, N1: Op2, N2: AllOneMask, N3: VL);
14242 Result = DAG.getNode(Opcode: RISCVISD::VMOR_VL, DL, VT: ContainerVT, N1: Temp, N2: Op1, N3: VL);
14243 break;
14244 }
14245 }
14246
14247 if (!VT.isFixedLengthVector())
14248 return Result;
14249 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14250}
14251
14252// Lower Floating-Point/Integer Type-Convert VP SDNodes
14253SDValue RISCVTargetLowering::lowerVPFPIntConvOp(SDValue Op,
14254 SelectionDAG &DAG) const {
14255 SDLoc DL(Op);
14256
14257 SDValue Src = Op.getOperand(i: 0);
14258 SDValue Mask = Op.getOperand(i: 1);
14259 SDValue VL = Op.getOperand(i: 2);
14260 unsigned RISCVISDOpc = getRISCVVLOp(Op);
14261
14262 MVT DstVT = Op.getSimpleValueType();
14263 MVT SrcVT = Src.getSimpleValueType();
14264 if (DstVT.isFixedLengthVector()) {
14265 DstVT = getContainerForFixedLengthVector(VT: DstVT);
14266 SrcVT = getContainerForFixedLengthVector(VT: SrcVT);
14267 Src = convertToScalableVector(VT: SrcVT, V: Src, DAG, Subtarget);
14268 MVT MaskVT = getMaskTypeFor(VecVT: DstVT);
14269 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14270 }
14271
14272 unsigned DstEltSize = DstVT.getScalarSizeInBits();
14273 unsigned SrcEltSize = SrcVT.getScalarSizeInBits();
14274
14275 SDValue Result;
14276 if (DstEltSize >= SrcEltSize) { // Single-width and widening conversion.
14277 if (SrcVT.isInteger()) {
14278 assert(DstVT.isFloatingPoint() && "Wrong input/output vector types");
14279
14280 unsigned RISCVISDExtOpc = RISCVISDOpc == RISCVISD::SINT_TO_FP_VL
14281 ? RISCVISD::VSEXT_VL
14282 : RISCVISD::VZEXT_VL;
14283
14284 // Do we need to do any pre-widening before converting?
14285 if (SrcEltSize == 1) {
14286 MVT IntVT = DstVT.changeVectorElementTypeToInteger();
14287 MVT XLenVT = Subtarget.getXLenVT();
14288 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
14289 SDValue ZeroSplat = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IntVT,
14290 N1: DAG.getUNDEF(VT: IntVT), N2: Zero, N3: VL);
14291 SDValue One = DAG.getSignedConstant(
14292 Val: RISCVISDExtOpc == RISCVISD::VZEXT_VL ? 1 : -1, DL, VT: XLenVT);
14293 SDValue OneSplat = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IntVT,
14294 N1: DAG.getUNDEF(VT: IntVT), N2: One, N3: VL);
14295 Src = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: IntVT, N1: Src, N2: OneSplat,
14296 N3: ZeroSplat, N4: DAG.getUNDEF(VT: IntVT), N5: VL);
14297 } else if (DstEltSize > (2 * SrcEltSize)) {
14298 // Widen before converting.
14299 MVT IntVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: DstEltSize / 2),
14300 EC: DstVT.getVectorElementCount());
14301 Src = DAG.getNode(Opcode: RISCVISDExtOpc, DL, VT: IntVT, N1: Src, N2: Mask, N3: VL);
14302 }
14303
14304 Result = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: DstVT, N1: Src, N2: Mask, N3: VL);
14305 } else {
14306 assert(SrcVT.isFloatingPoint() && DstVT.isInteger() &&
14307 "Wrong input/output vector types");
14308
14309 // Convert f16 to f32 then convert f32 to i64.
14310 if (DstEltSize > (2 * SrcEltSize)) {
14311 assert(SrcVT.getVectorElementType() == MVT::f16 && "Unexpected type!");
14312 MVT InterimFVT =
14313 MVT::getVectorVT(VT: MVT::f32, EC: DstVT.getVectorElementCount());
14314 Src =
14315 DAG.getNode(Opcode: RISCVISD::FP_EXTEND_VL, DL, VT: InterimFVT, N1: Src, N2: Mask, N3: VL);
14316 }
14317
14318 Result = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: DstVT, N1: Src, N2: Mask, N3: VL);
14319 }
14320 } else { // Narrowing + Conversion
14321 if (SrcVT.isInteger()) {
14322 assert(DstVT.isFloatingPoint() && "Wrong input/output vector types");
14323 // First do a narrowing convert to an FP type half the size, then round
14324 // the FP type to a small FP type if needed.
14325
14326 MVT InterimFVT = DstVT;
14327 if (SrcEltSize > (2 * DstEltSize)) {
14328 assert(SrcEltSize == (4 * DstEltSize) && "Unexpected types!");
14329 assert(DstVT.getVectorElementType() == MVT::f16 && "Unexpected type!");
14330 InterimFVT = MVT::getVectorVT(VT: MVT::f32, EC: DstVT.getVectorElementCount());
14331 }
14332
14333 Result = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: InterimFVT, N1: Src, N2: Mask, N3: VL);
14334
14335 if (InterimFVT != DstVT) {
14336 Src = Result;
14337 Result = DAG.getNode(Opcode: RISCVISD::FP_ROUND_VL, DL, VT: DstVT, N1: Src, N2: Mask, N3: VL);
14338 }
14339 } else {
14340 assert(SrcVT.isFloatingPoint() && DstVT.isInteger() &&
14341 "Wrong input/output vector types");
14342 // First do a narrowing conversion to an integer half the size, then
14343 // truncate if needed.
14344
14345 if (DstEltSize == 1) {
14346 // First convert to the same size integer, then convert to mask using
14347 // setcc.
14348 assert(SrcEltSize >= 16 && "Unexpected FP type!");
14349 MVT InterimIVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SrcEltSize),
14350 EC: DstVT.getVectorElementCount());
14351 Result = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: InterimIVT, N1: Src, N2: Mask, N3: VL);
14352
14353 // Compare the integer result to 0. The integer should be 0 or 1/-1,
14354 // otherwise the conversion was undefined.
14355 MVT XLenVT = Subtarget.getXLenVT();
14356 SDValue SplatZero = DAG.getConstant(Val: 0, DL, VT: XLenVT);
14357 SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: InterimIVT,
14358 N1: DAG.getUNDEF(VT: InterimIVT), N2: SplatZero, N3: VL);
14359 Result = DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: DstVT,
14360 Ops: {Result, SplatZero, DAG.getCondCode(Cond: ISD::SETNE),
14361 DAG.getUNDEF(VT: DstVT), Mask, VL});
14362 } else {
14363 MVT InterimIVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SrcEltSize / 2),
14364 EC: DstVT.getVectorElementCount());
14365
14366 Result = DAG.getNode(Opcode: RISCVISDOpc, DL, VT: InterimIVT, N1: Src, N2: Mask, N3: VL);
14367
14368 while (InterimIVT != DstVT) {
14369 SrcEltSize /= 2;
14370 Src = Result;
14371 InterimIVT = MVT::getVectorVT(VT: MVT::getIntegerVT(BitWidth: SrcEltSize / 2),
14372 EC: DstVT.getVectorElementCount());
14373 Result = DAG.getNode(Opcode: RISCVISD::TRUNCATE_VECTOR_VL, DL, VT: InterimIVT,
14374 N1: Src, N2: Mask, N3: VL);
14375 }
14376 }
14377 }
14378 }
14379
14380 MVT VT = Op.getSimpleValueType();
14381 if (!VT.isFixedLengthVector())
14382 return Result;
14383 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14384}
14385
14386SDValue RISCVTargetLowering::lowerVPMergeMask(SDValue Op,
14387 SelectionDAG &DAG) const {
14388 SDLoc DL(Op);
14389 MVT VT = Op.getSimpleValueType();
14390 MVT XLenVT = Subtarget.getXLenVT();
14391
14392 SDValue Mask = Op.getOperand(i: 0);
14393 SDValue TrueVal = Op.getOperand(i: 1);
14394 SDValue FalseVal = Op.getOperand(i: 2);
14395 SDValue VL = Op.getOperand(i: 3);
14396
14397 // Use default legalization if a vector of EVL type would be legal.
14398 EVT EVLVecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: VL.getValueType(),
14399 EC: VT.getVectorElementCount());
14400 if (isTypeLegal(VT: EVLVecVT))
14401 return SDValue();
14402
14403 MVT ContainerVT = VT;
14404 if (VT.isFixedLengthVector()) {
14405 ContainerVT = getContainerForFixedLengthVector(VT);
14406 Mask = convertToScalableVector(VT: ContainerVT, V: Mask, DAG, Subtarget);
14407 TrueVal = convertToScalableVector(VT: ContainerVT, V: TrueVal, DAG, Subtarget);
14408 FalseVal = convertToScalableVector(VT: ContainerVT, V: FalseVal, DAG, Subtarget);
14409 }
14410
14411 // Promote to a vector of i8.
14412 MVT PromotedVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
14413
14414 // Promote TrueVal and FalseVal using VLMax.
14415 // FIXME: Is there a better way to do this?
14416 SDValue VLMax = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
14417 SDValue SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: PromotedVT,
14418 N1: DAG.getUNDEF(VT: PromotedVT),
14419 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: VLMax);
14420 SDValue SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: PromotedVT,
14421 N1: DAG.getUNDEF(VT: PromotedVT),
14422 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: VLMax);
14423 TrueVal = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: TrueVal, N2: SplatOne,
14424 N3: SplatZero, N4: DAG.getUNDEF(VT: PromotedVT), N5: VL);
14425 // Any element past VL uses FalseVal, so use VLMax
14426 FalseVal = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: FalseVal,
14427 N2: SplatOne, N3: SplatZero, N4: DAG.getUNDEF(VT: PromotedVT), N5: VLMax);
14428
14429 // VP_MERGE the two promoted values.
14430 SDValue VPMerge = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: PromotedVT, N1: Mask,
14431 N2: TrueVal, N3: FalseVal, N4: FalseVal, N5: VL);
14432
14433 // Convert back to mask.
14434 SDValue TrueMask = DAG.getNode(Opcode: RISCVISD::VMSET_VL, DL, VT: ContainerVT, Operand: VL);
14435 SDValue Result = DAG.getNode(
14436 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
14437 Ops: {VPMerge, DAG.getConstant(Val: 0, DL, VT: PromotedVT), DAG.getCondCode(Cond: ISD::SETNE),
14438 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), TrueMask, VLMax});
14439
14440 if (VT.isFixedLengthVector())
14441 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14442 return Result;
14443}
14444
14445SDValue
14446RISCVTargetLowering::lowerVPSpliceExperimental(SDValue Op,
14447 SelectionDAG &DAG) const {
14448 using namespace SDPatternMatch;
14449
14450 SDLoc DL(Op);
14451
14452 SDValue Op1 = Op.getOperand(i: 0);
14453 SDValue Op2 = Op.getOperand(i: 1);
14454 SDValue Offset = Op.getOperand(i: 2);
14455 SDValue Mask = Op.getOperand(i: 3);
14456 SDValue EVL1 = Op.getOperand(i: 4);
14457 SDValue EVL2 = Op.getOperand(i: 5);
14458
14459 const MVT XLenVT = Subtarget.getXLenVT();
14460 MVT VT = Op.getSimpleValueType();
14461 MVT ContainerVT = VT;
14462 if (VT.isFixedLengthVector()) {
14463 ContainerVT = getContainerForFixedLengthVector(VT);
14464 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
14465 Op2 = convertToScalableVector(VT: ContainerVT, V: Op2, DAG, Subtarget);
14466 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14467 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14468 }
14469
14470 bool IsMaskVector = VT.getVectorElementType() == MVT::i1;
14471 if (IsMaskVector) {
14472 ContainerVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
14473
14474 // Expand input operands
14475 SDValue SplatOneOp1 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14476 N1: DAG.getUNDEF(VT: ContainerVT),
14477 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL1);
14478 SDValue SplatZeroOp1 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14479 N1: DAG.getUNDEF(VT: ContainerVT),
14480 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL1);
14481 Op1 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: Op1, N2: SplatOneOp1,
14482 N3: SplatZeroOp1, N4: DAG.getUNDEF(VT: ContainerVT), N5: EVL1);
14483
14484 SDValue SplatOneOp2 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14485 N1: DAG.getUNDEF(VT: ContainerVT),
14486 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL2);
14487 SDValue SplatZeroOp2 = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
14488 N1: DAG.getUNDEF(VT: ContainerVT),
14489 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL2);
14490 Op2 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: ContainerVT, N1: Op2, N2: SplatOneOp2,
14491 N3: SplatZeroOp2, N4: DAG.getUNDEF(VT: ContainerVT), N5: EVL2);
14492 }
14493
14494 auto getVectorFirstEle = [](SDValue Vec) {
14495 SDValue FirstEle;
14496 if (sd_match(N: Vec, P: m_InsertElt(Vec: m_Value(), Val: m_Value(N&: FirstEle), Idx: m_Zero())))
14497 return FirstEle;
14498
14499 if (Vec.getOpcode() == ISD::SPLAT_VECTOR ||
14500 Vec.getOpcode() == ISD::BUILD_VECTOR)
14501 return Vec.getOperand(i: 0);
14502
14503 return SDValue();
14504 };
14505
14506 if (!IsMaskVector && isNullConstant(V: Offset) && isOneConstant(V: EVL1))
14507 if (auto FirstEle = getVectorFirstEle(Op->getOperand(Num: 0))) {
14508 MVT EltVT = ContainerVT.getVectorElementType();
14509 SDValue Result;
14510 if ((EltVT == MVT::f16 && !Subtarget.hasVInstructionsF16()) ||
14511 EltVT == MVT::bf16) {
14512 EltVT = EltVT.changeTypeToInteger();
14513 ContainerVT = ContainerVT.changeVectorElementType(EltVT);
14514 Op2 = DAG.getBitcast(VT: ContainerVT, V: Op2);
14515 FirstEle =
14516 DAG.getAnyExtOrTrunc(Op: DAG.getBitcast(VT: EltVT, V: FirstEle), DL, VT: XLenVT);
14517 }
14518 Result = DAG.getNode(Opcode: EltVT.isFloatingPoint() ? RISCVISD::VFSLIDE1UP_VL
14519 : RISCVISD::VSLIDE1UP_VL,
14520 DL, VT: ContainerVT, N1: DAG.getUNDEF(VT: ContainerVT), N2: Op2,
14521 N3: FirstEle, N4: Mask, N5: EVL2);
14522 Result = DAG.getBitcast(
14523 VT: ContainerVT.changeVectorElementType(EltVT: VT.getVectorElementType()),
14524 V: Result);
14525 return VT.isFixedLengthVector()
14526 ? convertFromScalableVector(VT, V: Result, DAG, Subtarget)
14527 : Result;
14528 }
14529
14530 int64_t ImmValue = cast<ConstantSDNode>(Val&: Offset)->getSExtValue();
14531 SDValue DownOffset, UpOffset;
14532 if (ImmValue >= 0) {
14533 // The operand is a TargetConstant, we need to rebuild it as a regular
14534 // constant.
14535 DownOffset = DAG.getConstant(Val: ImmValue, DL, VT: XLenVT);
14536 UpOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL1, N2: DownOffset);
14537 } else {
14538 // The operand is a TargetConstant, we need to rebuild it as a regular
14539 // constant rather than negating the original operand.
14540 UpOffset = DAG.getConstant(Val: -ImmValue, DL, VT: XLenVT);
14541 DownOffset = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL1, N2: UpOffset);
14542 }
14543
14544 if (ImmValue != 0)
14545 Op1 = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
14546 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Op1, Offset: DownOffset, Mask,
14547 VL: Subtarget.hasVLDependentLatency() ? UpOffset : EVL2);
14548 SDValue Result = getVSlideup(DAG, Subtarget, DL, VT: ContainerVT, Passthru: Op1, Op: Op2,
14549 Offset: UpOffset, Mask, VL: EVL2, Policy: RISCVVType::TAIL_AGNOSTIC);
14550
14551 if (IsMaskVector) {
14552 // Truncate Result back to a mask vector (Result has same EVL as Op2)
14553 Result = DAG.getNode(
14554 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT.changeVectorElementType(EltVT: MVT::i1),
14555 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: ContainerVT),
14556 DAG.getCondCode(Cond: ISD::SETNE), DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)),
14557 Mask, EVL2});
14558 }
14559
14560 if (!VT.isFixedLengthVector())
14561 return Result;
14562 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14563}
14564
14565SDValue
14566RISCVTargetLowering::lowerVPReverseExperimental(SDValue Op,
14567 SelectionDAG &DAG) const {
14568 SDLoc DL(Op);
14569 MVT VT = Op.getSimpleValueType();
14570 MVT XLenVT = Subtarget.getXLenVT();
14571
14572 SDValue Op1 = Op.getOperand(i: 0);
14573 SDValue Mask = Op.getOperand(i: 1);
14574 SDValue EVL = Op.getOperand(i: 2);
14575
14576 MVT ContainerVT = VT;
14577 if (VT.isFixedLengthVector()) {
14578 ContainerVT = getContainerForFixedLengthVector(VT);
14579 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
14580 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14581 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14582 }
14583
14584 MVT GatherVT = ContainerVT;
14585 MVT IndicesVT = ContainerVT.changeVectorElementTypeToInteger();
14586 // Check if we are working with mask vectors
14587 bool IsMaskVector = ContainerVT.getVectorElementType() == MVT::i1;
14588 if (IsMaskVector) {
14589 GatherVT = IndicesVT = ContainerVT.changeVectorElementType(EltVT: MVT::i8);
14590
14591 // Expand input operand
14592 SDValue SplatOne = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
14593 N1: DAG.getUNDEF(VT: IndicesVT),
14594 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT), N3: EVL);
14595 SDValue SplatZero = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
14596 N1: DAG.getUNDEF(VT: IndicesVT),
14597 N2: DAG.getConstant(Val: 0, DL, VT: XLenVT), N3: EVL);
14598 Op1 = DAG.getNode(Opcode: RISCVISD::VMERGE_VL, DL, VT: IndicesVT, N1: Op1, N2: SplatOne,
14599 N3: SplatZero, N4: DAG.getUNDEF(VT: IndicesVT), N5: EVL);
14600 }
14601
14602 unsigned EltSize = GatherVT.getScalarSizeInBits();
14603 unsigned MinSize = GatherVT.getSizeInBits().getKnownMinValue();
14604 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
14605 unsigned MaxVLMAX =
14606 RISCVTargetLowering::computeVLMAX(VectorBits: VectorBitsMax, EltSize, MinSize);
14607
14608 unsigned GatherOpc = RISCVISD::VRGATHER_VV_VL;
14609 // If this is SEW=8 and VLMAX is unknown or more than 256, we need
14610 // to use vrgatherei16.vv.
14611 // TODO: It's also possible to use vrgatherei16.vv for other types to
14612 // decrease register width for the index calculation.
14613 // NOTE: This code assumes VLMAX <= 65536 for LMUL=8 SEW=16.
14614 if (MaxVLMAX > 256 && EltSize == 8) {
14615 // If this is LMUL=8, we have to split before using vrgatherei16.vv.
14616 // Split the vector in half and reverse each half using a full register
14617 // reverse.
14618 // Swap the halves and concatenate them.
14619 // Slide the concatenated result by (VLMax - VL).
14620 if (MinSize == (8 * RISCV::RVVBitsPerBlock)) {
14621 auto [LoVT, HiVT] = DAG.GetSplitDestVTs(VT: GatherVT);
14622 auto [Lo, Hi] = DAG.SplitVector(N: Op1, DL);
14623
14624 SDValue LoRev = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: LoVT, Operand: Lo);
14625 SDValue HiRev = DAG.getNode(Opcode: ISD::VECTOR_REVERSE, DL, VT: HiVT, Operand: Hi);
14626
14627 // Reassemble the low and high pieces reversed.
14628 // NOTE: this Result is unmasked (because we do not need masks for
14629 // shuffles). If in the future this has to change, we can use a SELECT_VL
14630 // between Result and UNDEF using the mask originally passed to VP_REVERSE
14631 SDValue Result =
14632 DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: GatherVT, N1: HiRev, N2: LoRev);
14633
14634 // Slide off any elements from past EVL that were reversed into the low
14635 // elements.
14636 SDValue VLMax =
14637 DAG.getElementCount(DL, VT: XLenVT, EC: GatherVT.getVectorElementCount());
14638 SDValue Diff = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: VLMax, N2: EVL);
14639
14640 Result = getVSlidedown(DAG, Subtarget, DL, VT: GatherVT,
14641 Passthru: DAG.getUNDEF(VT: GatherVT), Op: Result, Offset: Diff, Mask, VL: EVL);
14642
14643 if (IsMaskVector) {
14644 // Truncate Result back to a mask vector
14645 Result =
14646 DAG.getNode(Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
14647 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: GatherVT),
14648 DAG.getCondCode(Cond: ISD::SETNE),
14649 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), Mask, EVL});
14650 }
14651
14652 if (!VT.isFixedLengthVector())
14653 return Result;
14654 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14655 }
14656
14657 // Just promote the int type to i16 which will double the LMUL.
14658 IndicesVT = MVT::getVectorVT(VT: MVT::i16, EC: IndicesVT.getVectorElementCount());
14659 GatherOpc = RISCVISD::VRGATHEREI16_VV_VL;
14660 }
14661
14662 SDValue VID = DAG.getNode(Opcode: RISCVISD::VID_VL, DL, VT: IndicesVT, N1: Mask, N2: EVL);
14663 SDValue VecLen =
14664 DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: EVL, N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
14665 SDValue VecLenSplat = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: IndicesVT,
14666 N1: DAG.getUNDEF(VT: IndicesVT), N2: VecLen, N3: EVL);
14667 SDValue VRSUB = DAG.getNode(Opcode: RISCVISD::SUB_VL, DL, VT: IndicesVT, N1: VecLenSplat, N2: VID,
14668 N3: DAG.getUNDEF(VT: IndicesVT), N4: Mask, N5: EVL);
14669 SDValue Result = DAG.getNode(Opcode: GatherOpc, DL, VT: GatherVT, N1: Op1, N2: VRSUB,
14670 N3: DAG.getUNDEF(VT: GatherVT), N4: Mask, N5: EVL);
14671
14672 if (IsMaskVector) {
14673 // Truncate Result back to a mask vector
14674 Result = DAG.getNode(
14675 Opcode: RISCVISD::SETCC_VL, DL, VT: ContainerVT,
14676 Ops: {Result, DAG.getConstant(Val: 0, DL, VT: GatherVT), DAG.getCondCode(Cond: ISD::SETNE),
14677 DAG.getUNDEF(VT: getMaskTypeFor(VecVT: ContainerVT)), Mask, EVL});
14678 }
14679
14680 if (!VT.isFixedLengthVector())
14681 return Result;
14682 return convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14683}
14684
14685SDValue RISCVTargetLowering::lowerLogicVPOp(SDValue Op,
14686 SelectionDAG &DAG) const {
14687 MVT VT = Op.getSimpleValueType();
14688 if (VT.getVectorElementType() != MVT::i1)
14689 return lowerVPOp(Op, DAG);
14690
14691 // It is safe to drop mask parameter as masked-off elements are undef.
14692 SDValue Op1 = Op->getOperand(Num: 0);
14693 SDValue Op2 = Op->getOperand(Num: 1);
14694 SDValue VL = Op->getOperand(Num: 3);
14695
14696 MVT ContainerVT = VT;
14697 const bool IsFixed = VT.isFixedLengthVector();
14698 if (IsFixed) {
14699 ContainerVT = getContainerForFixedLengthVector(VT);
14700 Op1 = convertToScalableVector(VT: ContainerVT, V: Op1, DAG, Subtarget);
14701 Op2 = convertToScalableVector(VT: ContainerVT, V: Op2, DAG, Subtarget);
14702 }
14703
14704 SDLoc DL(Op);
14705 SDValue Val = DAG.getNode(Opcode: getRISCVVLOp(Op), DL, VT: ContainerVT, N1: Op1, N2: Op2, N3: VL);
14706 if (!IsFixed)
14707 return Val;
14708 return convertFromScalableVector(VT, V: Val, DAG, Subtarget);
14709}
14710
14711SDValue RISCVTargetLowering::lowerVPStridedLoad(SDValue Op,
14712 SelectionDAG &DAG) const {
14713 SDLoc DL(Op);
14714 MVT XLenVT = Subtarget.getXLenVT();
14715 MVT VT = Op.getSimpleValueType();
14716 MVT ContainerVT = VT;
14717 if (VT.isFixedLengthVector())
14718 ContainerVT = getContainerForFixedLengthVector(VT);
14719
14720 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
14721
14722 auto *VPNode = cast<VPStridedLoadSDNode>(Val&: Op);
14723 // Check if the mask is known to be all ones
14724 SDValue Mask = VPNode->getMask();
14725 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
14726
14727 SDValue IntID = DAG.getTargetConstant(Val: IsUnmasked ? Intrinsic::riscv_vlse
14728 : Intrinsic::riscv_vlse_mask,
14729 DL, VT: XLenVT);
14730 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID,
14731 DAG.getUNDEF(VT: ContainerVT), VPNode->getBasePtr(),
14732 VPNode->getStride()};
14733 if (!IsUnmasked) {
14734 if (VT.isFixedLengthVector()) {
14735 MVT MaskVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
14736 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14737 }
14738 Ops.push_back(Elt: Mask);
14739 }
14740 Ops.push_back(Elt: VPNode->getVectorLength());
14741 if (!IsUnmasked) {
14742 SDValue Policy =
14743 DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT);
14744 Ops.push_back(Elt: Policy);
14745 }
14746
14747 SDValue Result =
14748 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops,
14749 MemVT: VPNode->getMemoryVT(), MMO: VPNode->getMemOperand());
14750 SDValue Chain = Result.getValue(R: 1);
14751
14752 if (VT.isFixedLengthVector())
14753 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14754
14755 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
14756}
14757
14758SDValue RISCVTargetLowering::lowerVPStridedStore(SDValue Op,
14759 SelectionDAG &DAG) const {
14760 SDLoc DL(Op);
14761 MVT XLenVT = Subtarget.getXLenVT();
14762
14763 auto *VPNode = cast<VPStridedStoreSDNode>(Val&: Op);
14764 SDValue StoreVal = VPNode->getValue();
14765 MVT VT = StoreVal.getSimpleValueType();
14766 MVT ContainerVT = VT;
14767 if (VT.isFixedLengthVector()) {
14768 ContainerVT = getContainerForFixedLengthVector(VT);
14769 StoreVal = convertToScalableVector(VT: ContainerVT, V: StoreVal, DAG, Subtarget);
14770 }
14771
14772 // Check if the mask is known to be all ones
14773 SDValue Mask = VPNode->getMask();
14774 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
14775
14776 SDValue IntID = DAG.getTargetConstant(Val: IsUnmasked ? Intrinsic::riscv_vsse
14777 : Intrinsic::riscv_vsse_mask,
14778 DL, VT: XLenVT);
14779 SmallVector<SDValue, 8> Ops{VPNode->getChain(), IntID, StoreVal,
14780 VPNode->getBasePtr(), VPNode->getStride()};
14781 if (!IsUnmasked) {
14782 if (VT.isFixedLengthVector()) {
14783 MVT MaskVT = ContainerVT.changeVectorElementType(EltVT: MVT::i1);
14784 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14785 }
14786 Ops.push_back(Elt: Mask);
14787 }
14788 Ops.push_back(Elt: VPNode->getVectorLength());
14789
14790 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: VPNode->getVTList(),
14791 Ops, MemVT: VPNode->getMemoryVT(),
14792 MMO: VPNode->getMemOperand());
14793}
14794
14795// Custom lower MGATHER/VP_GATHER to a legalized form for RVV. It will then be
14796// matched to a RVV indexed load. The RVV indexed load instructions only
14797// support the "unsigned unscaled" addressing mode; indices are implicitly
14798// zero-extended or truncated to XLEN and are treated as byte offsets. Any
14799// signed or scaled indexing is extended to the XLEN value type and scaled
14800// accordingly.
14801SDValue RISCVTargetLowering::lowerMaskedGather(SDValue Op,
14802 SelectionDAG &DAG) const {
14803 SDLoc DL(Op);
14804 MVT VT = Op.getSimpleValueType();
14805
14806 const auto *MemSD = cast<MemSDNode>(Val: Op.getNode());
14807 EVT MemVT = MemSD->getMemoryVT();
14808 MachineMemOperand *MMO = MemSD->getMemOperand();
14809 SDValue Chain = MemSD->getChain();
14810 SDValue BasePtr = MemSD->getBasePtr();
14811
14812 [[maybe_unused]] ISD::LoadExtType LoadExtType;
14813 SDValue Index, Mask, PassThru, VL;
14814
14815 if (auto *VPGN = dyn_cast<VPGatherSDNode>(Val: Op.getNode())) {
14816 Index = VPGN->getIndex();
14817 Mask = VPGN->getMask();
14818 PassThru = DAG.getUNDEF(VT);
14819 VL = VPGN->getVectorLength();
14820 // VP doesn't support extending loads.
14821 LoadExtType = ISD::NON_EXTLOAD;
14822 } else {
14823 // Else it must be a MGATHER.
14824 auto *MGN = cast<MaskedGatherSDNode>(Val: Op.getNode());
14825 Index = MGN->getIndex();
14826 Mask = MGN->getMask();
14827 PassThru = MGN->getPassThru();
14828 LoadExtType = MGN->getExtensionType();
14829 }
14830
14831 MVT IndexVT = Index.getSimpleValueType();
14832 MVT XLenVT = Subtarget.getXLenVT();
14833
14834 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
14835 "Unexpected VTs!");
14836 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
14837 // Targets have to explicitly opt-in for extending vector loads.
14838 assert(LoadExtType == ISD::NON_EXTLOAD &&
14839 "Unexpected extending MGATHER/VP_GATHER");
14840
14841 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
14842 // the selection of the masked intrinsics doesn't do this for us.
14843 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
14844
14845 MVT ContainerVT = VT;
14846 if (VT.isFixedLengthVector()) {
14847 ContainerVT = getContainerForFixedLengthVector(VT);
14848 IndexVT = MVT::getVectorVT(VT: IndexVT.getVectorElementType(),
14849 EC: ContainerVT.getVectorElementCount());
14850
14851 Index = convertToScalableVector(VT: IndexVT, V: Index, DAG, Subtarget);
14852
14853 if (!IsUnmasked) {
14854 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14855 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14856 PassThru = convertToScalableVector(VT: ContainerVT, V: PassThru, DAG, Subtarget);
14857 }
14858 }
14859
14860 if (!VL)
14861 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
14862
14863 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(VT: XLenVT)) {
14864 IndexVT = IndexVT.changeVectorElementType(EltVT: XLenVT);
14865 Index = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IndexVT, Operand: Index);
14866 }
14867
14868 unsigned IntID =
14869 IsUnmasked ? Intrinsic::riscv_vluxei : Intrinsic::riscv_vluxei_mask;
14870 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
14871 if (IsUnmasked)
14872 Ops.push_back(Elt: DAG.getUNDEF(VT: ContainerVT));
14873 else
14874 Ops.push_back(Elt: PassThru);
14875 Ops.push_back(Elt: BasePtr);
14876 Ops.push_back(Elt: Index);
14877 if (!IsUnmasked)
14878 Ops.push_back(Elt: Mask);
14879 Ops.push_back(Elt: VL);
14880 if (!IsUnmasked)
14881 Ops.push_back(Elt: DAG.getTargetConstant(Val: RISCVVType::TAIL_AGNOSTIC, DL, VT: XLenVT));
14882
14883 SDVTList VTs = DAG.getVTList(VTs: {ContainerVT, MVT::Other});
14884 SDValue Result =
14885 DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs, Ops, MemVT, MMO);
14886 Chain = Result.getValue(R: 1);
14887
14888 if (VT.isFixedLengthVector())
14889 Result = convertFromScalableVector(VT, V: Result, DAG, Subtarget);
14890
14891 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
14892}
14893
14894// Custom lower MSCATTER/VP_SCATTER to a legalized form for RVV. It will then be
14895// matched to a RVV indexed store. The RVV indexed store instructions only
14896// support the "unsigned unscaled" addressing mode; indices are implicitly
14897// zero-extended or truncated to XLEN and are treated as byte offsets. Any
14898// signed or scaled indexing is extended to the XLEN value type and scaled
14899// accordingly.
14900SDValue RISCVTargetLowering::lowerMaskedScatter(SDValue Op,
14901 SelectionDAG &DAG) const {
14902 SDLoc DL(Op);
14903 const auto *MemSD = cast<MemSDNode>(Val: Op.getNode());
14904 EVT MemVT = MemSD->getMemoryVT();
14905 MachineMemOperand *MMO = MemSD->getMemOperand();
14906 SDValue Chain = MemSD->getChain();
14907 SDValue BasePtr = MemSD->getBasePtr();
14908
14909 [[maybe_unused]] bool IsTruncatingStore = false;
14910 SDValue Index, Mask, Val, VL;
14911
14912 if (auto *VPSN = dyn_cast<VPScatterSDNode>(Val: Op.getNode())) {
14913 Index = VPSN->getIndex();
14914 Mask = VPSN->getMask();
14915 Val = VPSN->getValue();
14916 VL = VPSN->getVectorLength();
14917 // VP doesn't support truncating stores.
14918 IsTruncatingStore = false;
14919 } else {
14920 // Else it must be a MSCATTER.
14921 auto *MSN = cast<MaskedScatterSDNode>(Val: Op.getNode());
14922 Index = MSN->getIndex();
14923 Mask = MSN->getMask();
14924 Val = MSN->getValue();
14925 IsTruncatingStore = MSN->isTruncatingStore();
14926 }
14927
14928 MVT VT = Val.getSimpleValueType();
14929 MVT IndexVT = Index.getSimpleValueType();
14930 MVT XLenVT = Subtarget.getXLenVT();
14931
14932 assert(VT.getVectorElementCount() == IndexVT.getVectorElementCount() &&
14933 "Unexpected VTs!");
14934 assert(BasePtr.getSimpleValueType() == XLenVT && "Unexpected pointer type");
14935 // Targets have to explicitly opt-in for extending vector loads and
14936 // truncating vector stores.
14937 assert(!IsTruncatingStore && "Unexpected truncating MSCATTER/VP_SCATTER");
14938
14939 // If the mask is known to be all ones, optimize to an unmasked intrinsic;
14940 // the selection of the masked intrinsics doesn't do this for us.
14941 bool IsUnmasked = ISD::isConstantSplatVectorAllOnes(N: Mask.getNode());
14942
14943 MVT ContainerVT = VT;
14944 if (VT.isFixedLengthVector()) {
14945 ContainerVT = getContainerForFixedLengthVector(VT);
14946 IndexVT = MVT::getVectorVT(VT: IndexVT.getVectorElementType(),
14947 EC: ContainerVT.getVectorElementCount());
14948
14949 Index = convertToScalableVector(VT: IndexVT, V: Index, DAG, Subtarget);
14950 Val = convertToScalableVector(VT: ContainerVT, V: Val, DAG, Subtarget);
14951
14952 if (!IsUnmasked) {
14953 MVT MaskVT = getMaskTypeFor(VecVT: ContainerVT);
14954 Mask = convertToScalableVector(VT: MaskVT, V: Mask, DAG, Subtarget);
14955 }
14956 }
14957
14958 if (!VL)
14959 VL = getDefaultVLOps(VecVT: VT, ContainerVT, DL, DAG, Subtarget).second;
14960
14961 if (XLenVT == MVT::i32 && IndexVT.getVectorElementType().bitsGT(VT: XLenVT)) {
14962 IndexVT = IndexVT.changeVectorElementType(EltVT: XLenVT);
14963 Index = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: IndexVT, Operand: Index);
14964 }
14965
14966 unsigned IntID =
14967 IsUnmasked ? Intrinsic::riscv_vsoxei : Intrinsic::riscv_vsoxei_mask;
14968 SmallVector<SDValue, 8> Ops{Chain, DAG.getTargetConstant(Val: IntID, DL, VT: XLenVT)};
14969 Ops.push_back(Elt: Val);
14970 Ops.push_back(Elt: BasePtr);
14971 Ops.push_back(Elt: Index);
14972 if (!IsUnmasked)
14973 Ops.push_back(Elt: Mask);
14974 Ops.push_back(Elt: VL);
14975
14976 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL,
14977 VTList: DAG.getVTList(VT: MVT::Other), Ops, MemVT, MMO);
14978}
14979
14980SDValue RISCVTargetLowering::lowerGET_ROUNDING(SDValue Op,
14981 SelectionDAG &DAG) const {
14982 const MVT XLenVT = Subtarget.getXLenVT();
14983 SDLoc DL(Op);
14984 SDValue Chain = Op->getOperand(Num: 0);
14985 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::frm, DL, VT: XLenVT);
14986 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
14987 SDValue RM = DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
14988
14989 // Encoding used for rounding mode in RISC-V differs from that used in
14990 // FLT_ROUNDS. To convert it the RISC-V rounding mode is used as an index in a
14991 // table, which consists of a sequence of 4-bit fields, each representing
14992 // corresponding FLT_ROUNDS mode.
14993 static const int Table =
14994 (int(RoundingMode::NearestTiesToEven) << 4 * RISCVFPRndMode::RNE) |
14995 (int(RoundingMode::TowardZero) << 4 * RISCVFPRndMode::RTZ) |
14996 (int(RoundingMode::TowardNegative) << 4 * RISCVFPRndMode::RDN) |
14997 (int(RoundingMode::TowardPositive) << 4 * RISCVFPRndMode::RUP) |
14998 (int(RoundingMode::NearestTiesToAway) << 4 * RISCVFPRndMode::RMM);
14999
15000 SDValue Shift =
15001 DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: RM, N2: DAG.getConstant(Val: 2, DL, VT: XLenVT));
15002 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT,
15003 N1: DAG.getConstant(Val: Table, DL, VT: XLenVT), N2: Shift);
15004 SDValue Masked = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Shifted,
15005 N2: DAG.getConstant(Val: 7, DL, VT: XLenVT));
15006
15007 return DAG.getMergeValues(Ops: {Masked, Chain}, dl: DL);
15008}
15009
15010SDValue RISCVTargetLowering::lowerSET_ROUNDING(SDValue Op,
15011 SelectionDAG &DAG) const {
15012 const MVT XLenVT = Subtarget.getXLenVT();
15013 SDLoc DL(Op);
15014 SDValue Chain = Op->getOperand(Num: 0);
15015 SDValue RMValue = Op->getOperand(Num: 1);
15016 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::frm, DL, VT: XLenVT);
15017
15018 // Encoding used for rounding mode in RISC-V differs from that used in
15019 // FLT_ROUNDS. To convert it the C rounding mode is used as an index in
15020 // a table, which consists of a sequence of 4-bit fields, each representing
15021 // corresponding RISC-V mode.
15022 static const unsigned Table =
15023 (RISCVFPRndMode::RNE << 4 * int(RoundingMode::NearestTiesToEven)) |
15024 (RISCVFPRndMode::RTZ << 4 * int(RoundingMode::TowardZero)) |
15025 (RISCVFPRndMode::RDN << 4 * int(RoundingMode::TowardNegative)) |
15026 (RISCVFPRndMode::RUP << 4 * int(RoundingMode::TowardPositive)) |
15027 (RISCVFPRndMode::RMM << 4 * int(RoundingMode::NearestTiesToAway));
15028
15029 RMValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: RMValue);
15030
15031 SDValue Shift = DAG.getNode(Opcode: ISD::SHL, DL, VT: XLenVT, N1: RMValue,
15032 N2: DAG.getConstant(Val: 2, DL, VT: XLenVT));
15033 SDValue Shifted = DAG.getNode(Opcode: ISD::SRL, DL, VT: XLenVT,
15034 N1: DAG.getConstant(Val: Table, DL, VT: XLenVT), N2: Shift);
15035 RMValue = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: Shifted,
15036 N2: DAG.getConstant(Val: 0x7, DL, VT: XLenVT));
15037 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15038 N3: RMValue);
15039}
15040
15041SDValue RISCVTargetLowering::lowerGET_FPENV(SDValue Op,
15042 SelectionDAG &DAG) const {
15043 const MVT XLenVT = Subtarget.getXLenVT();
15044 SDLoc DL(Op);
15045 SDValue Chain = Op->getOperand(Num: 0);
15046 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15047 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
15048 return DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
15049}
15050
15051SDValue RISCVTargetLowering::lowerSET_FPENV(SDValue Op,
15052 SelectionDAG &DAG) const {
15053 const MVT XLenVT = Subtarget.getXLenVT();
15054 SDLoc DL(Op);
15055 SDValue Chain = Op->getOperand(Num: 0);
15056 SDValue EnvValue = Op->getOperand(Num: 1);
15057 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15058
15059 EnvValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: EnvValue);
15060 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15061 N3: EnvValue);
15062}
15063
15064SDValue RISCVTargetLowering::lowerRESET_FPENV(SDValue Op,
15065 SelectionDAG &DAG) const {
15066 const MVT XLenVT = Subtarget.getXLenVT();
15067 SDLoc DL(Op);
15068 SDValue Chain = Op->getOperand(Num: 0);
15069 SDValue EnvValue = DAG.getRegister(Reg: RISCV::X0, VT: XLenVT);
15070 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15071
15072 return DAG.getNode(Opcode: RISCVISD::WRITE_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15073 N3: EnvValue);
15074}
15075
15076const uint64_t ModeMask64 = ~RISCVExceptFlags::ALL;
15077const uint32_t ModeMask32 = ~RISCVExceptFlags::ALL;
15078
15079SDValue RISCVTargetLowering::lowerGET_FPMODE(SDValue Op,
15080 SelectionDAG &DAG) const {
15081 const MVT XLenVT = Subtarget.getXLenVT();
15082 SDLoc DL(Op);
15083 SDValue Chain = Op->getOperand(Num: 0);
15084 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15085 SDVTList VTs = DAG.getVTList(VT1: XLenVT, VT2: MVT::Other);
15086 SDValue Result = DAG.getNode(Opcode: RISCVISD::READ_CSR, DL, VTList: VTs, N1: Chain, N2: SysRegNo);
15087 Chain = Result.getValue(R: 1);
15088 return DAG.getMergeValues(Ops: {Result, Chain}, dl: DL);
15089}
15090
15091SDValue RISCVTargetLowering::lowerSET_FPMODE(SDValue Op,
15092 SelectionDAG &DAG) const {
15093 const MVT XLenVT = Subtarget.getXLenVT();
15094 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
15095 SDLoc DL(Op);
15096 SDValue Chain = Op->getOperand(Num: 0);
15097 SDValue EnvValue = Op->getOperand(Num: 1);
15098 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15099 SDValue ModeMask = DAG.getConstant(Val: ModeMaskValue, DL, VT: XLenVT);
15100
15101 EnvValue = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: XLenVT, Operand: EnvValue);
15102 EnvValue = DAG.getNode(Opcode: ISD::AND, DL, VT: XLenVT, N1: EnvValue, N2: ModeMask);
15103 Chain = DAG.getNode(Opcode: RISCVISD::CLEAR_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15104 N3: ModeMask);
15105 return DAG.getNode(Opcode: RISCVISD::SET_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15106 N3: EnvValue);
15107}
15108
15109SDValue RISCVTargetLowering::lowerRESET_FPMODE(SDValue Op,
15110 SelectionDAG &DAG) const {
15111 const MVT XLenVT = Subtarget.getXLenVT();
15112 const uint64_t ModeMaskValue = Subtarget.is64Bit() ? ModeMask64 : ModeMask32;
15113 SDLoc DL(Op);
15114 SDValue Chain = Op->getOperand(Num: 0);
15115 SDValue SysRegNo = DAG.getTargetConstant(Val: RISCVSysReg::fcsr, DL, VT: XLenVT);
15116 SDValue ModeMask = DAG.getConstant(Val: ModeMaskValue, DL, VT: XLenVT);
15117
15118 return DAG.getNode(Opcode: RISCVISD::CLEAR_CSR, DL, VT: MVT::Other, N1: Chain, N2: SysRegNo,
15119 N3: ModeMask);
15120}
15121
15122SDValue RISCVTargetLowering::lowerEH_DWARF_CFA(SDValue Op,
15123 SelectionDAG &DAG) const {
15124 MachineFunction &MF = DAG.getMachineFunction();
15125
15126 bool isRISCV64 = Subtarget.is64Bit();
15127 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
15128
15129 int FI = MF.getFrameInfo().CreateFixedObject(Size: isRISCV64 ? 8 : 4, SPOffset: 0, IsImmutable: false);
15130 return DAG.getFrameIndex(FI, VT: PtrVT);
15131}
15132
15133// Returns the opcode of the target-specific SDNode that implements the 32-bit
15134// form of the given Opcode.
15135static unsigned getRISCVWOpcode(unsigned Opcode) {
15136 switch (Opcode) {
15137 default:
15138 llvm_unreachable("Unexpected opcode");
15139 case ISD::SHL:
15140 return RISCVISD::SLLW;
15141 case ISD::SRA:
15142 return RISCVISD::SRAW;
15143 case ISD::SRL:
15144 return RISCVISD::SRLW;
15145 case ISD::SDIV:
15146 return RISCVISD::DIVW;
15147 case ISD::UDIV:
15148 return RISCVISD::DIVUW;
15149 case ISD::UREM:
15150 return RISCVISD::REMUW;
15151 case ISD::ROTL:
15152 return RISCVISD::ROLW;
15153 case ISD::ROTR:
15154 return RISCVISD::RORW;
15155 }
15156}
15157
15158// Converts the given i8/i16/i32 operation to a target-specific SelectionDAG
15159// node. Because i8/i16/i32 isn't a legal type for RV64, these operations would
15160// otherwise be promoted to i64, making it difficult to select the
15161// SLLW/DIVUW/.../*W later one because the fact the operation was originally of
15162// type i8/i16/i32 is lost.
15163static SDValue customLegalizeToWOp(SDNode *N, SelectionDAG &DAG,
15164 unsigned ExtOpc = ISD::ANY_EXTEND) {
15165 SDLoc DL(N);
15166 unsigned WOpcode = getRISCVWOpcode(Opcode: N->getOpcode());
15167 SDValue NewOp0 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15168 SDValue NewOp1 = DAG.getNode(Opcode: ExtOpc, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15169 SDValue NewRes = DAG.getNode(Opcode: WOpcode, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
15170 // ReplaceNodeResults requires we maintain the same type for the return value.
15171 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: N->getValueType(ResNo: 0), Operand: NewRes);
15172}
15173
15174// Converts the given 32-bit operation to a i64 operation with signed extension
15175// semantic to reduce the signed extension instructions.
15176static SDValue customLegalizeToWOpWithSExt(SDNode *N, SelectionDAG &DAG) {
15177 SDLoc DL(N);
15178 SDValue NewOp0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15179 SDValue NewOp1 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15180 SDValue NewWOp = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
15181 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
15182 N2: DAG.getValueType(MVT::i32));
15183 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes);
15184}
15185
15186void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
15187 SmallVectorImpl<SDValue> &Results,
15188 SelectionDAG &DAG) const {
15189 SDLoc DL(N);
15190 switch (N->getOpcode()) {
15191 default:
15192 llvm_unreachable("Don't know how to custom type legalize this operation!");
15193 case ISD::STRICT_FP_TO_SINT:
15194 case ISD::STRICT_FP_TO_UINT:
15195 case ISD::FP_TO_SINT:
15196 case ISD::FP_TO_UINT: {
15197 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15198 "Unexpected custom legalisation");
15199 bool IsStrict = N->isStrictFPOpcode();
15200 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT ||
15201 N->getOpcode() == ISD::STRICT_FP_TO_SINT;
15202 SDValue Op0 = IsStrict ? N->getOperand(Num: 1) : N->getOperand(Num: 0);
15203 if (getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) !=
15204 TargetLowering::TypeSoftenFloat) {
15205 if (!isTypeLegal(VT: Op0.getValueType()))
15206 return;
15207 if (IsStrict) {
15208 SDValue Chain = N->getOperand(Num: 0);
15209 // In absence of Zfh, promote f16 to f32, then convert.
15210 if (Op0.getValueType() == MVT::f16 &&
15211 !Subtarget.hasStdExtZfhOrZhinx()) {
15212 Op0 = DAG.getNode(Opcode: ISD::STRICT_FP_EXTEND, DL, ResultTys: {MVT::f32, MVT::Other},
15213 Ops: {Chain, Op0});
15214 Chain = Op0.getValue(R: 1);
15215 }
15216 unsigned Opc = IsSigned ? RISCVISD::STRICT_FCVT_W_RV64
15217 : RISCVISD::STRICT_FCVT_WU_RV64;
15218 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other);
15219 SDValue Res = DAG.getNode(
15220 Opcode: Opc, DL, VTList: VTs, N1: Chain, N2: Op0,
15221 N3: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: MVT::i64));
15222 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15223 Results.push_back(Elt: Res.getValue(R: 1));
15224 return;
15225 }
15226 // For bf16, or f16 in absence of Zfh, promote [b]f16 to f32 and then
15227 // convert.
15228 if ((Op0.getValueType() == MVT::f16 &&
15229 !Subtarget.hasStdExtZfhOrZhinx()) ||
15230 Op0.getValueType() == MVT::bf16)
15231 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
15232
15233 unsigned Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
15234 SDValue Res =
15235 DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: Op0,
15236 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RTZ, DL, VT: MVT::i64));
15237 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15238 return;
15239 }
15240 // If the FP type needs to be softened, emit a library call using the 'si'
15241 // version. If we left it to default legalization we'd end up with 'di'. If
15242 // the FP type doesn't need to be softened just let generic type
15243 // legalization promote the result type.
15244 RTLIB::Libcall LC;
15245 if (IsSigned)
15246 LC = RTLIB::getFPTOSINT(OpVT: Op0.getValueType(), RetVT: N->getValueType(ResNo: 0));
15247 else
15248 LC = RTLIB::getFPTOUINT(OpVT: Op0.getValueType(), RetVT: N->getValueType(ResNo: 0));
15249 MakeLibCallOptions CallOptions;
15250 EVT OpVT = Op0.getValueType();
15251 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: N->getValueType(ResNo: 0));
15252 SDValue Chain = IsStrict ? N->getOperand(Num: 0) : SDValue();
15253 SDValue Result;
15254 std::tie(args&: Result, args&: Chain) =
15255 makeLibCall(DAG, LC, RetVT: N->getValueType(ResNo: 0), Ops: Op0, CallOptions, dl: DL, Chain);
15256 Results.push_back(Elt: Result);
15257 if (IsStrict)
15258 Results.push_back(Elt: Chain);
15259 break;
15260 }
15261 case ISD::LROUND: {
15262 SDValue Op0 = N->getOperand(Num: 0);
15263 EVT Op0VT = Op0.getValueType();
15264 if (getTypeAction(Context&: *DAG.getContext(), VT: Op0.getValueType()) !=
15265 TargetLowering::TypeSoftenFloat) {
15266 if (!isTypeLegal(VT: Op0VT))
15267 return;
15268
15269 // In absence of Zfh, promote f16 to f32, then convert.
15270 if (Op0.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfhOrZhinx())
15271 Op0 = DAG.getNode(Opcode: ISD::FP_EXTEND, DL, VT: MVT::f32, Operand: Op0);
15272
15273 SDValue Res =
15274 DAG.getNode(Opcode: RISCVISD::FCVT_W_RV64, DL, VT: MVT::i64, N1: Op0,
15275 N2: DAG.getTargetConstant(Val: RISCVFPRndMode::RMM, DL, VT: MVT::i64));
15276 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15277 return;
15278 }
15279 // If the FP type needs to be softened, emit a library call to lround. We'll
15280 // need to truncate the result. We assume any value that doesn't fit in i32
15281 // is allowed to return an unspecified value.
15282 RTLIB::Libcall LC =
15283 Op0.getValueType() == MVT::f64 ? RTLIB::LROUND_F64 : RTLIB::LROUND_F32;
15284 MakeLibCallOptions CallOptions;
15285 EVT OpVT = Op0.getValueType();
15286 CallOptions.setTypeListBeforeSoften(OpsVT: OpVT, RetVT: MVT::i64);
15287 SDValue Result = makeLibCall(DAG, LC, RetVT: MVT::i64, Ops: Op0, CallOptions, dl: DL).first;
15288 Result = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Result);
15289 Results.push_back(Elt: Result);
15290 break;
15291 }
15292 case ISD::READCYCLECOUNTER:
15293 case ISD::READSTEADYCOUNTER: {
15294 assert(!Subtarget.is64Bit() && "READCYCLECOUNTER/READSTEADYCOUNTER only "
15295 "has custom type legalization on riscv32");
15296
15297 SDValue LoCounter, HiCounter;
15298 MVT XLenVT = Subtarget.getXLenVT();
15299 if (N->getOpcode() == ISD::READCYCLECOUNTER) {
15300 LoCounter = DAG.getTargetConstant(Val: RISCVSysReg::cycle, DL, VT: XLenVT);
15301 HiCounter = DAG.getTargetConstant(Val: RISCVSysReg::cycleh, DL, VT: XLenVT);
15302 } else {
15303 LoCounter = DAG.getTargetConstant(Val: RISCVSysReg::time, DL, VT: XLenVT);
15304 HiCounter = DAG.getTargetConstant(Val: RISCVSysReg::timeh, DL, VT: XLenVT);
15305 }
15306 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32, VT3: MVT::Other);
15307 SDValue RCW = DAG.getNode(Opcode: RISCVISD::READ_COUNTER_WIDE, DL, VTList: VTs,
15308 N1: N->getOperand(Num: 0), N2: LoCounter, N3: HiCounter);
15309
15310 Results.push_back(
15311 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: RCW, N2: RCW.getValue(R: 1)));
15312 Results.push_back(Elt: RCW.getValue(R: 2));
15313 break;
15314 }
15315 case ISD::LOAD: {
15316 if (!ISD::isNON_EXTLoad(N))
15317 return;
15318
15319 // Use a SEXTLOAD instead of the default EXTLOAD. Similar to the
15320 // sext_inreg we emit for ADD/SUB/MUL/SLLI.
15321 LoadSDNode *Ld = cast<LoadSDNode>(Val: N);
15322
15323 if (N->getValueType(ResNo: 0) == MVT::i64) {
15324 assert(Subtarget.hasStdExtZilsd() && !Subtarget.is64Bit() &&
15325 "Unexpected custom legalisation");
15326
15327 if (Ld->getAlign() < Subtarget.getZilsdAlign())
15328 return;
15329
15330 SDLoc DL(N);
15331 SDValue Result = DAG.getMemIntrinsicNode(
15332 Opcode: RISCVISD::LD_RV32, dl: DL,
15333 VTList: DAG.getVTList(VTs: {MVT::i32, MVT::i32, MVT::Other}),
15334 Ops: {Ld->getChain(), Ld->getBasePtr()}, MemVT: MVT::i64, MMO: Ld->getMemOperand());
15335 SDValue Lo = Result.getValue(R: 0);
15336 SDValue Hi = Result.getValue(R: 1);
15337 SDValue Pair = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Lo, N2: Hi);
15338 Results.append(IL: {Pair, Result.getValue(R: 2)});
15339 return;
15340 }
15341
15342 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15343 "Unexpected custom legalisation");
15344
15345 SDLoc dl(N);
15346 SDValue Res = DAG.getExtLoad(ExtType: ISD::SEXTLOAD, dl, VT: MVT::i64, Chain: Ld->getChain(),
15347 Ptr: Ld->getBasePtr(), MemVT: Ld->getMemoryVT(),
15348 MMO: Ld->getMemOperand());
15349 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL: dl, VT: MVT::i32, Operand: Res));
15350 Results.push_back(Elt: Res.getValue(R: 1));
15351 return;
15352 }
15353 case ISD::MUL: {
15354 unsigned Size = N->getSimpleValueType(ResNo: 0).getSizeInBits();
15355 unsigned XLen = Subtarget.getXLen();
15356 if (Size > XLen) {
15357 // This multiply needs to be expanded, try to use MULH+MUL or WMUL if
15358 // possible. We duplicate the default legalization to
15359 // MULHU/MULHS/UMUL_LOHI/SMUL_LOHI to minimize the number of calls to
15360 // MaskedValueIsZero and ComputeNumSignBits
15361 // FIXME: Should we have a target independent MULHSU/WMULSU node? Are
15362 // there are other targets that could use it?
15363 assert(Size == (XLen * 2) && "Unexpected custom legalisation");
15364
15365 auto MakeMULPair = [&](SDValue L, SDValue R, unsigned HighOpc,
15366 unsigned LoHiOpc) {
15367 MVT XLenVT = Subtarget.getXLenVT();
15368 L = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: XLenVT, Operand: L);
15369 R = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: XLenVT, Operand: R);
15370 SDValue Lo, Hi;
15371 if (Subtarget.hasStdExtP() && !Subtarget.is64Bit()) {
15372 SDVTList VTs = DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32);
15373 Lo = DAG.getNode(Opcode: LoHiOpc, DL, VTList: VTs, N1: L, N2: R);
15374 Hi = Lo.getValue(R: 1);
15375 } else {
15376 Lo = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: L, N2: R);
15377 Hi = DAG.getNode(Opcode: HighOpc, DL, VT: XLenVT, N1: L, N2: R);
15378 }
15379 return DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: N->getValueType(ResNo: 0), N1: Lo, N2: Hi);
15380 };
15381
15382 SDValue LHS = N->getOperand(Num: 0);
15383 SDValue RHS = N->getOperand(Num: 1);
15384
15385 APInt HighMask = APInt::getHighBitsSet(numBits: Size, hiBitsSet: XLen);
15386 bool LHSIsU = DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask);
15387 bool RHSIsU = DAG.MaskedValueIsZero(Op: RHS, Mask: HighMask);
15388 if (LHSIsU && RHSIsU) {
15389 Results.push_back(Elt: MakeMULPair(LHS, RHS, ISD::MULHU, ISD::UMUL_LOHI));
15390 return;
15391 }
15392
15393 bool LHSIsS = DAG.ComputeNumSignBits(Op: LHS) > XLen;
15394 bool RHSIsS = DAG.ComputeNumSignBits(Op: RHS) > XLen;
15395 if (LHSIsS && RHSIsS)
15396 Results.push_back(Elt: MakeMULPair(LHS, RHS, ISD::MULHS, ISD::SMUL_LOHI));
15397 else if (RHSIsU && LHSIsS)
15398 Results.push_back(
15399 Elt: MakeMULPair(LHS, RHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
15400 else if (LHSIsU && RHSIsS)
15401 Results.push_back(
15402 Elt: MakeMULPair(RHS, LHS, RISCVISD::MULHSU, RISCVISD::WMULSU));
15403
15404 return;
15405 }
15406 [[fallthrough]];
15407 }
15408 case ISD::ADD:
15409 case ISD::SUB:
15410 if (N->getValueType(ResNo: 0) == MVT::i64) {
15411 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
15412 "Unexpected custom legalisation");
15413
15414 // Expand to ADDD/SUBD.
15415 auto [LHSLo, LHSHi] =
15416 DAG.SplitScalar(N: N->getOperand(Num: 0), DL, LoVT: MVT::i32, HiVT: MVT::i32);
15417 auto [RHSLo, RHSHi] =
15418 DAG.SplitScalar(N: N->getOperand(Num: 1), DL, LoVT: MVT::i32, HiVT: MVT::i32);
15419 unsigned Opc =
15420 N->getOpcode() == ISD::ADD ? RISCVISD::ADDD : RISCVISD::SUBD;
15421 SDValue Res = DAG.getNode(Opcode: Opc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
15422 N1: LHSLo, N2: LHSHi, N3: RHSLo, N4: RHSHi);
15423 Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Res, N2: Res.getValue(R: 1));
15424 Results.push_back(Elt: Res);
15425 return;
15426 }
15427
15428 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15429 "Unexpected custom legalisation");
15430 Results.push_back(Elt: customLegalizeToWOpWithSExt(N, DAG));
15431 break;
15432 case ISD::SHL:
15433 case ISD::SRA:
15434 case ISD::SRL: {
15435 EVT VT = N->getValueType(ResNo: 0);
15436 if (VT.isFixedLengthVector() && Subtarget.hasStdExtP()) {
15437 assert(Subtarget.is64Bit() && (VT == MVT::v2i16 || VT == MVT::v4i8) &&
15438 "Unexpected vector type for P-extension shift");
15439
15440 // If shift amount is a splat, don't scalarize - let normal widening
15441 // and SIMD patterns handle it (pslli.h, psrli.h, etc.)
15442 SDValue ShiftAmt = N->getOperand(Num: 1);
15443 if (DAG.isSplatValue(V: ShiftAmt, /*AllowUndefs=*/true))
15444 break;
15445
15446 EVT WidenVT = getTypeToTransformTo(Context&: *DAG.getContext(), VT);
15447 unsigned WidenNumElts = WidenVT.getVectorNumElements();
15448 // Unroll with OrigNumElts operations, padding result to WidenNumElts
15449 SDValue Res = DAG.UnrollVectorOp(N, ResNE: WidenNumElts);
15450 Results.push_back(Elt: Res);
15451 break;
15452 }
15453
15454 if (VT == MVT::i64) {
15455 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
15456 "Unexpected custom legalisation");
15457
15458 SDValue LHS = N->getOperand(Num: 0);
15459 SDValue ShAmt = N->getOperand(Num: 1);
15460
15461 unsigned WideOpc = 0;
15462 APInt HighMask = APInt::getHighBitsSet(numBits: 64, hiBitsSet: 32);
15463 if (DAG.MaskedValueIsZero(Op: LHS, Mask: HighMask))
15464 WideOpc = RISCVISD::WSLL;
15465 else if (DAG.ComputeMaxSignificantBits(Op: LHS) <= 32)
15466 WideOpc = RISCVISD::WSLA;
15467
15468 if (WideOpc) {
15469 SDValue Res =
15470 DAG.getNode(Opcode: WideOpc, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
15471 N1: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: LHS),
15472 N2: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: ShAmt));
15473 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: N->getValueType(ResNo: 0),
15474 N1: Res, N2: Res.getValue(R: 1)));
15475 return;
15476 }
15477
15478 // Only handle constant shifts < 32. Non-constant shifts are handled by
15479 // lowerShiftLeftParts/lowerShiftRightParts, and shifts >= 32 use default
15480 // legalization.
15481 auto *ShAmtC = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
15482 if (!ShAmtC || ShAmtC->getZExtValue() >= 32)
15483 break;
15484
15485 auto [Lo, Hi] = DAG.SplitScalar(N: LHS, DL, LoVT: MVT::i32, HiVT: MVT::i32);
15486
15487 SDValue LoRes, HiRes;
15488 if (N->getOpcode() == ISD::SHL) {
15489 // Lo = slli Lo, shamt
15490 // Hi = nsrli {Hi, Lo}, (32 - shamt)
15491 uint64_t ShAmtVal = ShAmtC->getZExtValue();
15492 LoRes = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i32, N1: Lo, N2: ShAmt);
15493 HiRes = DAG.getNode(Opcode: RISCVISD::NSRL, DL, VT: MVT::i32, N1: Lo, N2: Hi,
15494 N3: DAG.getConstant(Val: 32 - ShAmtVal, DL, VT: MVT::i32));
15495 } else {
15496 bool IsSRA = N->getOpcode() == ISD::SRA;
15497 LoRes = DAG.getNode(Opcode: IsSRA ? RISCVISD::NSRA : RISCVISD::NSRL, DL,
15498 VT: MVT::i32, N1: Lo, N2: Hi, N3: ShAmt);
15499 HiRes =
15500 DAG.getNode(Opcode: IsSRA ? ISD::SRA : ISD::SRL, DL, VT: MVT::i32, N1: Hi, N2: ShAmt);
15501 }
15502 SDValue Res = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: LoRes, N2: HiRes);
15503 Results.push_back(Elt: Res);
15504 return;
15505 }
15506
15507 assert(VT == MVT::i32 && Subtarget.is64Bit() &&
15508 "Unexpected custom legalisation");
15509 if (N->getOperand(Num: 1).getOpcode() != ISD::Constant) {
15510 // If we can use a BSET instruction, allow default promotion to apply.
15511 if (N->getOpcode() == ISD::SHL && Subtarget.hasStdExtZbs() &&
15512 isOneConstant(V: N->getOperand(Num: 0)))
15513 break;
15514 Results.push_back(Elt: customLegalizeToWOp(N, DAG));
15515 break;
15516 }
15517
15518 // Custom legalize ISD::SHL by placing a SIGN_EXTEND_INREG after. This is
15519 // similar to customLegalizeToWOpWithSExt, but we must zero_extend the
15520 // shift amount.
15521 if (N->getOpcode() == ISD::SHL) {
15522 SDLoc DL(N);
15523 SDValue NewOp0 =
15524 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15525 SDValue NewOp1 =
15526 DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15527 SDValue NewWOp = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
15528 SDValue NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewWOp,
15529 N2: DAG.getValueType(MVT::i32));
15530 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes));
15531 }
15532
15533 break;
15534 }
15535 case ISD::ROTL:
15536 case ISD::ROTR:
15537 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15538 "Unexpected custom legalisation");
15539 assert((Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb() ||
15540 Subtarget.hasVendorXTHeadBb()) &&
15541 "Unexpected custom legalization");
15542 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: 1)) &&
15543 !(Subtarget.hasStdExtZbb() || Subtarget.hasStdExtZbkb()))
15544 return;
15545 Results.push_back(Elt: customLegalizeToWOp(N, DAG));
15546 break;
15547 case ISD::CTTZ:
15548 case ISD::CTTZ_ZERO_UNDEF:
15549 case ISD::CTLZ:
15550 case ISD::CTLZ_ZERO_UNDEF:
15551 case ISD::CTLS: {
15552 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15553 "Unexpected custom legalisation");
15554
15555 SDValue NewOp0 =
15556 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15557 unsigned Opc;
15558 switch (N->getOpcode()) {
15559 default: llvm_unreachable("Unexpected opcode");
15560 case ISD::CTTZ:
15561 case ISD::CTTZ_ZERO_UNDEF:
15562 Opc = RISCVISD::CTZW;
15563 break;
15564 case ISD::CTLZ:
15565 case ISD::CTLZ_ZERO_UNDEF:
15566 Opc = RISCVISD::CLZW;
15567 break;
15568 case ISD::CTLS:
15569 Opc = RISCVISD::CLSW;
15570 break;
15571 }
15572
15573 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, Operand: NewOp0);
15574 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15575 return;
15576 }
15577 case ISD::SDIV:
15578 case ISD::UDIV:
15579 case ISD::UREM: {
15580 MVT VT = N->getSimpleValueType(ResNo: 0);
15581 assert((VT == MVT::i8 || VT == MVT::i16 || VT == MVT::i32) &&
15582 Subtarget.is64Bit() && Subtarget.hasStdExtM() &&
15583 "Unexpected custom legalisation");
15584 // Don't promote division/remainder by constant since we should expand those
15585 // to multiply by magic constant.
15586 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
15587 if (N->getOperand(Num: 1).getOpcode() == ISD::Constant &&
15588 !isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
15589 return;
15590
15591 // If the input is i32, use ANY_EXTEND since the W instructions don't read
15592 // the upper 32 bits. For other types we need to sign or zero extend
15593 // based on the opcode.
15594 unsigned ExtOpc = ISD::ANY_EXTEND;
15595 if (VT != MVT::i32)
15596 ExtOpc = N->getOpcode() == ISD::SDIV ? ISD::SIGN_EXTEND
15597 : ISD::ZERO_EXTEND;
15598
15599 Results.push_back(Elt: customLegalizeToWOp(N, DAG, ExtOpc));
15600 break;
15601 }
15602 case ISD::SADDO:
15603 case ISD::SSUBO: {
15604 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15605 "Unexpected custom legalisation");
15606
15607 // This is similar to the default legalization, but we return the
15608 // sext_inreg instead of the add/sub.
15609 bool IsAdd = N->getOpcode() == ISD::SADDO;
15610 SDValue LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15611 SDValue RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15612 SDValue Op =
15613 DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL, VT: MVT::i64, N1: LHS, N2: RHS);
15614 SDValue Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Op,
15615 N2: DAG.getValueType(MVT::i32));
15616
15617 SDValue Overflow;
15618
15619 // If the RHS is a constant, we can simplify ConditionRHS below. Otherwise
15620 // use the default legalization.
15621 if (IsAdd && isa<ConstantSDNode>(Val: N->getOperand(Num: 1))) {
15622 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: MVT::i64);
15623
15624 // For an addition, the result should be less than one of the operands
15625 // (LHS) if and only if the other operand (RHS) is negative, otherwise
15626 // there will be overflow.
15627 EVT OType = N->getValueType(ResNo: 1);
15628 SDValue ResultLowerThanLHS =
15629 DAG.getSetCC(DL, VT: OType, LHS: Res, RHS: LHS, Cond: ISD::SETLT);
15630 SDValue ConditionRHS = DAG.getSetCC(DL, VT: OType, LHS: RHS, RHS: Zero, Cond: ISD::SETLT);
15631
15632 Overflow =
15633 DAG.getNode(Opcode: ISD::XOR, DL, VT: OType, N1: ConditionRHS, N2: ResultLowerThanLHS);
15634 } else {
15635 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res, RHS: Op, Cond: ISD::SETNE);
15636 }
15637
15638 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15639 Results.push_back(Elt: Overflow);
15640 return;
15641 }
15642 case ISD::UADDO:
15643 case ISD::USUBO: {
15644 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15645 "Unexpected custom legalisation");
15646 bool IsAdd = N->getOpcode() == ISD::UADDO;
15647 // Create an ADDW or SUBW.
15648 SDValue LHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15649 SDValue RHS = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15650 SDValue Res =
15651 DAG.getNode(Opcode: IsAdd ? ISD::ADD : ISD::SUB, DL, VT: MVT::i64, N1: LHS, N2: RHS);
15652 Res = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Res,
15653 N2: DAG.getValueType(MVT::i32));
15654
15655 SDValue Overflow;
15656 if (IsAdd && isOneConstant(V: RHS)) {
15657 // Special case uaddo X, 1 overflowed if the addition result is 0.
15658 // The general case (X + C) < C is not necessarily beneficial. Although we
15659 // reduce the live range of X, we may introduce the materialization of
15660 // constant C, especially when the setcc result is used by branch. We have
15661 // no compare with constant and branch instructions.
15662 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res,
15663 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i64), Cond: ISD::SETEQ);
15664 } else if (IsAdd && isAllOnesConstant(V: RHS)) {
15665 // Special case uaddo X, -1 overflowed if X != 0.
15666 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: N->getOperand(Num: 0),
15667 RHS: DAG.getConstant(Val: 0, DL, VT: MVT::i32), Cond: ISD::SETNE);
15668 } else {
15669 // Sign extend the LHS and perform an unsigned compare with the ADDW
15670 // result. Since the inputs are sign extended from i32, this is equivalent
15671 // to comparing the lower 32 bits.
15672 LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15673 Overflow = DAG.getSetCC(DL, VT: N->getValueType(ResNo: 1), LHS: Res, RHS: LHS,
15674 Cond: IsAdd ? ISD::SETULT : ISD::SETUGT);
15675 }
15676
15677 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15678 Results.push_back(Elt: Overflow);
15679 return;
15680 }
15681 case ISD::UADDSAT:
15682 case ISD::USUBSAT:
15683 case ISD::SADDSAT:
15684 case ISD::SSUBSAT: {
15685 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15686 "Unexpected custom legalisation");
15687
15688 if (Subtarget.hasStdExtP()) {
15689 // On RV64, map scalar i32 saturating add/sub through lane 0 of a packed
15690 // v2i32 operation so we can select ps*.w instructions.
15691 SDValue LHS = DAG.getNode(
15692 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
15693 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0)));
15694 SDValue RHS = DAG.getNode(
15695 Opcode: ISD::SCALAR_TO_VECTOR, DL, VT: MVT::v2i32,
15696 Operand: DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1)));
15697 SDValue VecRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: MVT::v2i32, N1: LHS, N2: RHS);
15698 SDValue Zero = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
15699 Results.push_back(
15700 Elt: DAG.getNode(Opcode: ISD::EXTRACT_VECTOR_ELT, DL, VT: MVT::i32, N1: VecRes, N2: Zero));
15701 return;
15702 }
15703
15704 assert(!Subtarget.hasStdExtZbb() && "Unexpected custom legalisation");
15705 Results.push_back(Elt: expandAddSubSat(Node: N, DAG));
15706 return;
15707 }
15708 case ISD::ABS: {
15709 assert(N->getValueType(0) == MVT::i32 && Subtarget.is64Bit() &&
15710 "Unexpected custom legalisation");
15711
15712 if (Subtarget.hasStdExtP()) {
15713 SDValue Src =
15714 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15715 SDValue Abs = DAG.getNode(Opcode: RISCVISD::ABSW, DL, VT: MVT::i64, Operand: Src);
15716 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Abs));
15717 return;
15718 }
15719
15720 if (Subtarget.hasStdExtZbb()) {
15721 // Emit a special node that will be expanded to NEGW+MAX at isel.
15722 // This allows us to remember that the result is sign extended. Expanding
15723 // to NEGW+MAX here requires a Freeze which breaks ComputeNumSignBits.
15724 SDValue Src = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL, VT: MVT::i64,
15725 Operand: N->getOperand(Num: 0));
15726 SDValue Abs = DAG.getNode(Opcode: RISCVISD::NEGW_MAX, DL, VT: MVT::i64, Operand: Src);
15727 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Abs));
15728 return;
15729 }
15730
15731 // Expand abs to Y = (sraiw X, 31); subw(xor(X, Y), Y)
15732 SDValue Src = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 0));
15733
15734 // Freeze the source so we can increase it's use count.
15735 Src = DAG.getFreeze(V: Src);
15736
15737 // Copy sign bit to all bits using the sraiw pattern.
15738 SDValue SignFill = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: Src,
15739 N2: DAG.getValueType(MVT::i32));
15740 SignFill = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i64, N1: SignFill,
15741 N2: DAG.getConstant(Val: 31, DL, VT: MVT::i64));
15742
15743 SDValue NewRes = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i64, N1: Src, N2: SignFill);
15744 NewRes = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: NewRes, N2: SignFill);
15745
15746 // NOTE: The result is only required to be anyextended, but sext is
15747 // consistent with type legalization of sub.
15748 NewRes = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: NewRes,
15749 N2: DAG.getValueType(MVT::i32));
15750 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: NewRes));
15751 return;
15752 }
15753 case ISD::BITCAST: {
15754 EVT VT = N->getValueType(ResNo: 0);
15755 assert(VT.isInteger() && !VT.isVector() && "Unexpected VT!");
15756 SDValue Op0 = N->getOperand(Num: 0);
15757 EVT Op0VT = Op0.getValueType();
15758 MVT XLenVT = Subtarget.getXLenVT();
15759 if (VT == MVT::i16 &&
15760 ((Op0VT == MVT::f16 && Subtarget.hasStdExtZfhminOrZhinxmin()) ||
15761 (Op0VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()))) {
15762 SDValue FPConv = DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: XLenVT, Operand: Op0);
15763 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: FPConv));
15764 } else if (VT == MVT::i32 && Op0VT == MVT::f32 && Subtarget.is64Bit() &&
15765 Subtarget.hasStdExtFOrZfinx()) {
15766 SDValue FPConv =
15767 DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Op0);
15768 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: FPConv));
15769 } else if (VT == MVT::i64 && Op0VT == MVT::f64 && !Subtarget.is64Bit() &&
15770 Subtarget.hasStdExtDOrZdinx()) {
15771 SDValue NewReg = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
15772 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Op0);
15773 SDValue Lo = NewReg.getValue(R: 0);
15774 SDValue Hi = NewReg.getValue(R: 1);
15775 // For big-endian, swap the order when building the i64 pair.
15776 if (!Subtarget.isLittleEndian())
15777 std::swap(a&: Lo, b&: Hi);
15778 SDValue RetReg = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: Lo, N2: Hi);
15779 Results.push_back(Elt: RetReg);
15780 } else if (!VT.isVector() && Op0VT.isFixedLengthVector() &&
15781 isTypeLegal(VT: Op0VT)) {
15782 // Custom-legalize bitcasts from fixed-length vector types to illegal
15783 // scalar types in order to improve codegen. Bitcast the vector to a
15784 // one-element vector type whose element type is the same as the result
15785 // type, and extract the first element.
15786 EVT BVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 1);
15787 if (isTypeLegal(VT: BVT)) {
15788 SDValue BVec = DAG.getBitcast(VT: BVT, V: Op0);
15789 Results.push_back(Elt: DAG.getExtractVectorElt(DL, VT, Vec: BVec, Idx: 0));
15790 }
15791 }
15792 break;
15793 }
15794 case ISD::BITREVERSE: {
15795 assert(N->getValueType(0) == MVT::i8 && Subtarget.hasStdExtZbkb() &&
15796 "Unexpected custom legalisation");
15797 MVT XLenVT = Subtarget.getXLenVT();
15798 SDValue NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 0));
15799 SDValue NewRes = DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT: XLenVT, Operand: NewOp);
15800 // ReplaceNodeResults requires we maintain the same type for the return
15801 // value.
15802 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i8, Operand: NewRes));
15803 break;
15804 }
15805 case RISCVISD::BREV8:
15806 case RISCVISD::ORC_B: {
15807 MVT VT = N->getSimpleValueType(ResNo: 0);
15808 MVT XLenVT = Subtarget.getXLenVT();
15809 assert((VT == MVT::i16 || (VT == MVT::i32 && Subtarget.is64Bit())) &&
15810 "Unexpected custom legalisation");
15811 assert(((N->getOpcode() == RISCVISD::BREV8 && Subtarget.hasStdExtZbkb()) ||
15812 (N->getOpcode() == RISCVISD::ORC_B && Subtarget.hasStdExtZbb())) &&
15813 "Unexpected extension");
15814 SDValue NewOp = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: XLenVT, Operand: N->getOperand(Num: 0));
15815 SDValue NewRes = DAG.getNode(Opcode: N->getOpcode(), DL, VT: XLenVT, Operand: NewOp);
15816 // ReplaceNodeResults requires we maintain the same type for the return
15817 // value.
15818 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: NewRes));
15819 break;
15820 }
15821 case RISCVISD::ASUB:
15822 case RISCVISD::ASUBU:
15823 case RISCVISD::MULHSU:
15824 case RISCVISD::MULHR:
15825 case RISCVISD::MULHRU:
15826 case RISCVISD::MULHRSU: {
15827 MVT VT = N->getSimpleValueType(ResNo: 0);
15828 SDValue Op0 = N->getOperand(Num: 0);
15829 SDValue Op1 = N->getOperand(Num: 1);
15830 unsigned Opcode = N->getOpcode();
15831 // PMULH* variants don't support i8
15832 [[maybe_unused]] bool IsMulH =
15833 Opcode == RISCVISD::MULHSU || Opcode == RISCVISD::MULHR ||
15834 Opcode == RISCVISD::MULHRU || Opcode == RISCVISD::MULHRSU;
15835 assert(VT == MVT::v2i16 || (!IsMulH && VT == MVT::v4i8));
15836 MVT NewVT = MVT::v4i16;
15837 if (VT == MVT::v4i8)
15838 NewVT = MVT::v8i8;
15839 SDValue Undef = DAG.getUNDEF(VT);
15840 Op0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, Ops: {Op0, Undef});
15841 Op1 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: NewVT, Ops: {Op1, Undef});
15842 Results.push_back(Elt: DAG.getNode(Opcode, DL, VT: NewVT, Ops: {Op0, Op1}));
15843 return;
15844 }
15845 case ISD::EXTRACT_VECTOR_ELT: {
15846 // Custom-legalize an EXTRACT_VECTOR_ELT where XLEN<SEW, as the SEW element
15847 // type is illegal (currently only vXi64 RV32).
15848 // With vmv.x.s, when SEW > XLEN, only the least-significant XLEN bits are
15849 // transferred to the destination register. We issue two of these from the
15850 // upper- and lower- halves of the SEW-bit vector element, slid down to the
15851 // first element.
15852 SDValue Vec = N->getOperand(Num: 0);
15853 SDValue Idx = N->getOperand(Num: 1);
15854
15855 // The vector type hasn't been legalized yet so we can't issue target
15856 // specific nodes if it needs legalization.
15857 // FIXME: We would manually legalize if it's important.
15858 if (!isTypeLegal(VT: Vec.getValueType()))
15859 return;
15860
15861 MVT VecVT = Vec.getSimpleValueType();
15862
15863 assert(!Subtarget.is64Bit() && N->getValueType(0) == MVT::i64 &&
15864 VecVT.getVectorElementType() == MVT::i64 &&
15865 "Unexpected EXTRACT_VECTOR_ELT legalization");
15866
15867 // If this is a fixed vector, we need to convert it to a scalable vector.
15868 MVT ContainerVT = VecVT;
15869 if (VecVT.isFixedLengthVector()) {
15870 ContainerVT = getContainerForFixedLengthVector(VT: VecVT);
15871 Vec = convertToScalableVector(VT: ContainerVT, V: Vec, DAG, Subtarget);
15872 }
15873
15874 MVT XLenVT = Subtarget.getXLenVT();
15875
15876 // Use a VL of 1 to avoid processing more elements than we need.
15877 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT, DL, DAG, Subtarget);
15878
15879 // Unless the index is known to be 0, we must slide the vector down to get
15880 // the desired element into index 0.
15881 if (!isNullConstant(V: Idx)) {
15882 Vec = getVSlidedown(DAG, Subtarget, DL, VT: ContainerVT,
15883 Passthru: DAG.getUNDEF(VT: ContainerVT), Op: Vec, Offset: Idx, Mask, VL);
15884 }
15885
15886 // Extract the lower XLEN bits of the correct vector element.
15887 SDValue EltLo = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
15888
15889 // To extract the upper XLEN bits of the vector element, shift the first
15890 // element right by 32 bits and re-extract the lower XLEN bits.
15891 SDValue ThirtyTwoV = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: ContainerVT,
15892 N1: DAG.getUNDEF(VT: ContainerVT),
15893 N2: DAG.getConstant(Val: 32, DL, VT: XLenVT), N3: VL);
15894 SDValue LShr32 =
15895 DAG.getNode(Opcode: RISCVISD::SRL_VL, DL, VT: ContainerVT, N1: Vec, N2: ThirtyTwoV,
15896 N3: DAG.getUNDEF(VT: ContainerVT), N4: Mask, N5: VL);
15897
15898 SDValue EltHi = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: LShr32);
15899
15900 Results.push_back(Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: EltLo, N2: EltHi));
15901 break;
15902 }
15903 case ISD::INTRINSIC_WO_CHAIN: {
15904 unsigned IntNo = N->getConstantOperandVal(Num: 0);
15905 switch (IntNo) {
15906 default:
15907 llvm_unreachable(
15908 "Don't know how to custom type legalize this intrinsic!");
15909 case Intrinsic::experimental_get_vector_length: {
15910 SDValue Res = lowerGetVectorLength(N, DAG, Subtarget);
15911 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15912 return;
15913 }
15914 case Intrinsic::riscv_orc_b:
15915 case Intrinsic::riscv_brev8:
15916 case Intrinsic::riscv_sha256sig0:
15917 case Intrinsic::riscv_sha256sig1:
15918 case Intrinsic::riscv_sha256sum0:
15919 case Intrinsic::riscv_sha256sum1:
15920 case Intrinsic::riscv_sm3p0:
15921 case Intrinsic::riscv_sm3p1: {
15922 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
15923 return;
15924 unsigned Opc;
15925 switch (IntNo) {
15926 case Intrinsic::riscv_orc_b: Opc = RISCVISD::ORC_B; break;
15927 case Intrinsic::riscv_brev8: Opc = RISCVISD::BREV8; break;
15928 case Intrinsic::riscv_sha256sig0: Opc = RISCVISD::SHA256SIG0; break;
15929 case Intrinsic::riscv_sha256sig1: Opc = RISCVISD::SHA256SIG1; break;
15930 case Intrinsic::riscv_sha256sum0: Opc = RISCVISD::SHA256SUM0; break;
15931 case Intrinsic::riscv_sha256sum1: Opc = RISCVISD::SHA256SUM1; break;
15932 case Intrinsic::riscv_sm3p0: Opc = RISCVISD::SM3P0; break;
15933 case Intrinsic::riscv_sm3p1: Opc = RISCVISD::SM3P1; break;
15934 }
15935
15936 SDValue NewOp =
15937 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15938 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, Operand: NewOp);
15939 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15940 return;
15941 }
15942 case Intrinsic::riscv_sm4ks:
15943 case Intrinsic::riscv_sm4ed: {
15944 unsigned Opc =
15945 IntNo == Intrinsic::riscv_sm4ks ? RISCVISD::SM4KS : RISCVISD::SM4ED;
15946 SDValue NewOp0 =
15947 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15948 SDValue NewOp1 =
15949 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
15950 SDValue Res =
15951 DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1, N3: N->getOperand(Num: 3));
15952 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15953 return;
15954 }
15955 case Intrinsic::riscv_mopr: {
15956 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
15957 return;
15958 SDValue NewOp =
15959 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15960 SDValue Res = DAG.getNode(
15961 Opcode: RISCVISD::MOP_R, DL, VT: MVT::i64, N1: NewOp,
15962 N2: DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 2), DL, VT: MVT::i64));
15963 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15964 return;
15965 }
15966 case Intrinsic::riscv_moprr: {
15967 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
15968 return;
15969 SDValue NewOp0 =
15970 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15971 SDValue NewOp1 =
15972 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
15973 SDValue Res = DAG.getNode(
15974 Opcode: RISCVISD::MOP_RR, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1,
15975 N3: DAG.getTargetConstant(Val: N->getConstantOperandVal(Num: 3), DL, VT: MVT::i64));
15976 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
15977 return;
15978 }
15979 case Intrinsic::riscv_clmulh:
15980 case Intrinsic::riscv_clmulr: {
15981 if (!Subtarget.is64Bit() || N->getValueType(ResNo: 0) != MVT::i32)
15982 return;
15983
15984 // Extend inputs to XLen, and shift by 32. This will add 64 trailing zeros
15985 // to the full 128-bit clmul result of multiplying two xlen values.
15986 // Perform clmulr or clmulh on the shifted values. Finally, extract the
15987 // upper 32 bits.
15988 //
15989 // The alternative is to mask the inputs to 32 bits and use clmul, but
15990 // that requires two shifts to mask each input without zext.w.
15991 // FIXME: If the inputs are known zero extended or could be freely
15992 // zero extended, the mask form would be better.
15993 SDValue NewOp0 =
15994 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 1));
15995 SDValue NewOp1 =
15996 DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N->getOperand(Num: 2));
15997 NewOp0 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp0,
15998 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
15999 NewOp1 = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: NewOp1,
16000 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
16001 unsigned Opc =
16002 IntNo == Intrinsic::riscv_clmulh ? ISD::CLMULH : ISD::CLMULR;
16003 SDValue Res = DAG.getNode(Opcode: Opc, DL, VT: MVT::i64, N1: NewOp0, N2: NewOp1);
16004 Res = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Res,
16005 N2: DAG.getConstant(Val: 32, DL, VT: MVT::i64));
16006 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Res));
16007 return;
16008 }
16009 case Intrinsic::riscv_vmv_x_s: {
16010 EVT VT = N->getValueType(ResNo: 0);
16011 MVT XLenVT = Subtarget.getXLenVT();
16012 if (VT.bitsLT(VT: XLenVT)) {
16013 // Simple case just extract using vmv.x.s and truncate.
16014 SDValue Extract = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL,
16015 VT: Subtarget.getXLenVT(), Operand: N->getOperand(Num: 1));
16016 Results.push_back(Elt: DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Extract));
16017 return;
16018 }
16019
16020 assert(VT == MVT::i64 && !Subtarget.is64Bit() &&
16021 "Unexpected custom legalization");
16022
16023 // We need to do the move in two steps.
16024 SDValue Vec = N->getOperand(Num: 1);
16025 MVT VecVT = Vec.getSimpleValueType();
16026
16027 // First extract the lower XLEN bits of the element.
16028 SDValue EltLo = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: Vec);
16029
16030 // To extract the upper XLEN bits of the vector element, shift the first
16031 // element right by 32 bits and re-extract the lower XLEN bits.
16032 auto [Mask, VL] = getDefaultVLOps(NumElts: 1, ContainerVT: VecVT, DL, DAG, Subtarget);
16033
16034 SDValue ThirtyTwoV =
16035 DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: VecVT, N1: DAG.getUNDEF(VT: VecVT),
16036 N2: DAG.getConstant(Val: 32, DL, VT: XLenVT), N3: VL);
16037 SDValue LShr32 = DAG.getNode(Opcode: RISCVISD::SRL_VL, DL, VT: VecVT, N1: Vec, N2: ThirtyTwoV,
16038 N3: DAG.getUNDEF(VT: VecVT), N4: Mask, N5: VL);
16039 SDValue EltHi = DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: XLenVT, Operand: LShr32);
16040
16041 Results.push_back(
16042 Elt: DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: MVT::i64, N1: EltLo, N2: EltHi));
16043 break;
16044 }
16045 }
16046 break;
16047 }
16048 case ISD::VECREDUCE_ADD:
16049 case ISD::VECREDUCE_AND:
16050 case ISD::VECREDUCE_OR:
16051 case ISD::VECREDUCE_XOR:
16052 case ISD::VECREDUCE_SMAX:
16053 case ISD::VECREDUCE_UMAX:
16054 case ISD::VECREDUCE_SMIN:
16055 case ISD::VECREDUCE_UMIN:
16056 if (SDValue V = lowerVECREDUCE(Op: SDValue(N, 0), DAG))
16057 Results.push_back(Elt: V);
16058 break;
16059 case ISD::VP_REDUCE_ADD:
16060 case ISD::VP_REDUCE_AND:
16061 case ISD::VP_REDUCE_OR:
16062 case ISD::VP_REDUCE_XOR:
16063 case ISD::VP_REDUCE_SMAX:
16064 case ISD::VP_REDUCE_UMAX:
16065 case ISD::VP_REDUCE_SMIN:
16066 case ISD::VP_REDUCE_UMIN:
16067 if (SDValue V = lowerVPREDUCE(Op: SDValue(N, 0), DAG))
16068 Results.push_back(Elt: V);
16069 break;
16070 case ISD::GET_ROUNDING: {
16071 SDVTList VTs = DAG.getVTList(VT1: Subtarget.getXLenVT(), VT2: MVT::Other);
16072 SDValue Res = DAG.getNode(Opcode: ISD::GET_ROUNDING, DL, VTList: VTs, N: N->getOperand(Num: 0));
16073 Results.push_back(Elt: Res.getValue(R: 0));
16074 Results.push_back(Elt: Res.getValue(R: 1));
16075 break;
16076 }
16077 }
16078}
16079
16080/// Given a binary operator, return the *associative* generic ISD::VECREDUCE_OP
16081/// which corresponds to it.
16082static unsigned getVecReduceOpcode(unsigned Opc) {
16083 switch (Opc) {
16084 default:
16085 llvm_unreachable("Unhandled binary to transform reduction");
16086 case ISD::ADD:
16087 return ISD::VECREDUCE_ADD;
16088 case ISD::UMAX:
16089 return ISD::VECREDUCE_UMAX;
16090 case ISD::SMAX:
16091 return ISD::VECREDUCE_SMAX;
16092 case ISD::UMIN:
16093 return ISD::VECREDUCE_UMIN;
16094 case ISD::SMIN:
16095 return ISD::VECREDUCE_SMIN;
16096 case ISD::AND:
16097 return ISD::VECREDUCE_AND;
16098 case ISD::OR:
16099 return ISD::VECREDUCE_OR;
16100 case ISD::XOR:
16101 return ISD::VECREDUCE_XOR;
16102 case ISD::FADD:
16103 // Note: This is the associative form of the generic reduction opcode.
16104 return ISD::VECREDUCE_FADD;
16105 case ISD::FMAXNUM:
16106 return ISD::VECREDUCE_FMAX;
16107 case ISD::FMINNUM:
16108 return ISD::VECREDUCE_FMIN;
16109 }
16110}
16111
16112/// Perform two related transforms whose purpose is to incrementally recognize
16113/// an explode_vector followed by scalar reduction as a vector reduction node.
16114/// This exists to recover from a deficiency in SLP which can't handle
16115/// forests with multiple roots sharing common nodes. In some cases, one
16116/// of the trees will be vectorized, and the other will remain (unprofitably)
16117/// scalarized.
16118static SDValue
16119combineBinOpOfExtractToReduceTree(SDNode *N, SelectionDAG &DAG,
16120 const RISCVSubtarget &Subtarget) {
16121
16122 // This transforms need to run before all integer types have been legalized
16123 // to i64 (so that the vector element type matches the add type), and while
16124 // it's safe to introduce odd sized vector types.
16125 if (DAG.NewNodesMustHaveLegalTypes)
16126 return SDValue();
16127
16128 // Without V, this transform isn't useful. We could form the (illegal)
16129 // operations and let them be scalarized again, but there's really no point.
16130 if (!Subtarget.hasVInstructions())
16131 return SDValue();
16132
16133 const SDLoc DL(N);
16134 const EVT VT = N->getValueType(ResNo: 0);
16135 const unsigned Opc = N->getOpcode();
16136
16137 if (!VT.isInteger()) {
16138 switch (Opc) {
16139 default:
16140 return SDValue();
16141 case ISD::FADD:
16142 // For FADD, we only handle the case with reassociation allowed. We
16143 // could handle strict reduction order, but at the moment, there's no
16144 // known reason to, and the complexity isn't worth it.
16145 if (!N->getFlags().hasAllowReassociation())
16146 return SDValue();
16147 break;
16148 case ISD::FMAXNUM:
16149 case ISD::FMINNUM:
16150 break;
16151 }
16152 }
16153
16154 const unsigned ReduceOpc = getVecReduceOpcode(Opc);
16155 assert(Opc == ISD::getVecReduceBaseOpcode(ReduceOpc) &&
16156 "Inconsistent mappings");
16157 SDValue LHS = N->getOperand(Num: 0);
16158 SDValue RHS = N->getOperand(Num: 1);
16159
16160 if (!LHS.hasOneUse() || !RHS.hasOneUse())
16161 return SDValue();
16162
16163 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT)
16164 std::swap(a&: LHS, b&: RHS);
16165
16166 if (RHS.getOpcode() != ISD::EXTRACT_VECTOR_ELT ||
16167 !isa<ConstantSDNode>(Val: RHS.getOperand(i: 1)))
16168 return SDValue();
16169
16170 uint64_t RHSIdx = cast<ConstantSDNode>(Val: RHS.getOperand(i: 1))->getLimitedValue();
16171 SDValue SrcVec = RHS.getOperand(i: 0);
16172 EVT SrcVecVT = SrcVec.getValueType();
16173 assert(SrcVecVT.getVectorElementType() == VT);
16174 if (SrcVecVT.isScalableVector())
16175 return SDValue();
16176
16177 if (SrcVecVT.getScalarSizeInBits() > Subtarget.getELen())
16178 return SDValue();
16179
16180 // match binop (extract_vector_elt V, 0), (extract_vector_elt V, 1) to
16181 // reduce_op (extract_subvector [2 x VT] from V). This will form the
16182 // root of our reduction tree. TODO: We could extend this to any two
16183 // adjacent aligned constant indices if desired.
16184 if (LHS.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
16185 LHS.getOperand(i: 0) == SrcVec && isa<ConstantSDNode>(Val: LHS.getOperand(i: 1))) {
16186 uint64_t LHSIdx =
16187 cast<ConstantSDNode>(Val: LHS.getOperand(i: 1))->getLimitedValue();
16188 if (0 == std::min(a: LHSIdx, b: RHSIdx) && 1 == std::max(a: LHSIdx, b: RHSIdx)) {
16189 EVT ReduceVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: 2);
16190 SDValue Vec = DAG.getExtractSubvector(DL, VT: ReduceVT, Vec: SrcVec, Idx: 0);
16191 return DAG.getNode(Opcode: ReduceOpc, DL, VT, Operand: Vec, Flags: N->getFlags());
16192 }
16193 }
16194
16195 // Match (binop (reduce (extract_subvector V, 0),
16196 // (extract_vector_elt V, sizeof(SubVec))))
16197 // into a reduction of one more element from the original vector V.
16198 if (LHS.getOpcode() != ReduceOpc)
16199 return SDValue();
16200
16201 SDValue ReduceVec = LHS.getOperand(i: 0);
16202 if (ReduceVec.getOpcode() == ISD::EXTRACT_SUBVECTOR &&
16203 ReduceVec.hasOneUse() && ReduceVec.getOperand(i: 0) == RHS.getOperand(i: 0) &&
16204 isNullConstant(V: ReduceVec.getOperand(i: 1)) &&
16205 ReduceVec.getValueType().getVectorNumElements() == RHSIdx) {
16206 // For illegal types (e.g. 3xi32), most will be combined again into a
16207 // wider (hopefully legal) type. If this is a terminal state, we are
16208 // relying on type legalization here to produce something reasonable
16209 // and this lowering quality could probably be improved. (TODO)
16210 EVT ReduceVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT, NumElements: RHSIdx + 1);
16211 SDValue Vec = DAG.getExtractSubvector(DL, VT: ReduceVT, Vec: SrcVec, Idx: 0);
16212 return DAG.getNode(Opcode: ReduceOpc, DL, VT, Operand: Vec,
16213 Flags: ReduceVec->getFlags() & N->getFlags());
16214 }
16215
16216 return SDValue();
16217}
16218
16219
16220// Try to fold (<bop> x, (reduction.<bop> vec, start))
16221static SDValue combineBinOpToReduce(SDNode *N, SelectionDAG &DAG,
16222 const RISCVSubtarget &Subtarget) {
16223 auto BinOpToRVVReduce = [](unsigned Opc) {
16224 switch (Opc) {
16225 default:
16226 llvm_unreachable("Unhandled binary to transform reduction");
16227 case ISD::ADD:
16228 return RISCVISD::VECREDUCE_ADD_VL;
16229 case ISD::UMAX:
16230 return RISCVISD::VECREDUCE_UMAX_VL;
16231 case ISD::SMAX:
16232 return RISCVISD::VECREDUCE_SMAX_VL;
16233 case ISD::UMIN:
16234 return RISCVISD::VECREDUCE_UMIN_VL;
16235 case ISD::SMIN:
16236 return RISCVISD::VECREDUCE_SMIN_VL;
16237 case ISD::AND:
16238 return RISCVISD::VECREDUCE_AND_VL;
16239 case ISD::OR:
16240 return RISCVISD::VECREDUCE_OR_VL;
16241 case ISD::XOR:
16242 return RISCVISD::VECREDUCE_XOR_VL;
16243 case ISD::FADD:
16244 return RISCVISD::VECREDUCE_FADD_VL;
16245 case ISD::FMAXNUM:
16246 return RISCVISD::VECREDUCE_FMAX_VL;
16247 case ISD::FMINNUM:
16248 return RISCVISD::VECREDUCE_FMIN_VL;
16249 }
16250 };
16251
16252 auto IsReduction = [&BinOpToRVVReduce](SDValue V, unsigned Opc) {
16253 return V.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
16254 isNullConstant(V: V.getOperand(i: 1)) &&
16255 V.getOperand(i: 0).getOpcode() == BinOpToRVVReduce(Opc);
16256 };
16257
16258 unsigned Opc = N->getOpcode();
16259 unsigned ReduceIdx;
16260 if (IsReduction(N->getOperand(Num: 0), Opc))
16261 ReduceIdx = 0;
16262 else if (IsReduction(N->getOperand(Num: 1), Opc))
16263 ReduceIdx = 1;
16264 else
16265 return SDValue();
16266
16267 // Skip if FADD disallows reassociation but the combiner needs.
16268 if (Opc == ISD::FADD && !N->getFlags().hasAllowReassociation())
16269 return SDValue();
16270
16271 SDValue Extract = N->getOperand(Num: ReduceIdx);
16272 SDValue Reduce = Extract.getOperand(i: 0);
16273 if (!Extract.hasOneUse() || !Reduce.hasOneUse())
16274 return SDValue();
16275
16276 SDValue ScalarV = Reduce.getOperand(i: 2);
16277 EVT ScalarVT = ScalarV.getValueType();
16278 if (ScalarV.getOpcode() == ISD::INSERT_SUBVECTOR &&
16279 ScalarV.getOperand(i: 0)->isUndef() &&
16280 isNullConstant(V: ScalarV.getOperand(i: 2)))
16281 ScalarV = ScalarV.getOperand(i: 1);
16282
16283 // Make sure that ScalarV is a splat with VL=1.
16284 if (ScalarV.getOpcode() != RISCVISD::VFMV_S_F_VL &&
16285 ScalarV.getOpcode() != RISCVISD::VMV_S_X_VL &&
16286 ScalarV.getOpcode() != RISCVISD::VMV_V_X_VL)
16287 return SDValue();
16288
16289 if (!isNonZeroAVL(AVL: ScalarV.getOperand(i: 2)))
16290 return SDValue();
16291
16292 // Check the scalar of ScalarV is neutral element
16293 // TODO: Deal with value other than neutral element.
16294 if (!isNeutralConstant(Opc: N->getOpcode(), Flags: N->getFlags(), V: ScalarV.getOperand(i: 1),
16295 OperandNo: 0))
16296 return SDValue();
16297
16298 // If the AVL is zero, operand 0 will be returned. So it's not safe to fold.
16299 // FIXME: We might be able to improve this if operand 0 is undef.
16300 if (!isNonZeroAVL(AVL: Reduce.getOperand(i: 5)))
16301 return SDValue();
16302
16303 SDValue NewStart = N->getOperand(Num: 1 - ReduceIdx);
16304
16305 SDLoc DL(N);
16306 SDValue NewScalarV =
16307 lowerScalarInsert(Scalar: NewStart, VL: ScalarV.getOperand(i: 2),
16308 VT: ScalarV.getSimpleValueType(), DL, DAG, Subtarget);
16309
16310 // If we looked through an INSERT_SUBVECTOR we need to restore it.
16311 if (ScalarVT != ScalarV.getValueType())
16312 NewScalarV =
16313 DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: ScalarVT), SubVec: NewScalarV, Idx: 0);
16314
16315 SDValue Ops[] = {Reduce.getOperand(i: 0), Reduce.getOperand(i: 1),
16316 NewScalarV, Reduce.getOperand(i: 3),
16317 Reduce.getOperand(i: 4), Reduce.getOperand(i: 5)};
16318 SDValue NewReduce =
16319 DAG.getNode(Opcode: Reduce.getOpcode(), DL, VT: Reduce.getValueType(), Ops);
16320 return DAG.getNode(Opcode: Extract.getOpcode(), DL, VT: Extract.getValueType(), N1: NewReduce,
16321 N2: Extract.getOperand(i: 1));
16322}
16323
16324// Optimize (add (shl x, c0), (shl y, c1)) ->
16325// (SLLI (SH*ADD x, y), c0), if c1-c0 equals to [1|2|3].
16326// or
16327// (SLLI (QC.SHLADD x, y, c1 - c0), c0), if 4 <= (c1-c0) <=31.
16328static SDValue transformAddShlImm(SDNode *N, SelectionDAG &DAG,
16329 const RISCVSubtarget &Subtarget) {
16330 // Perform this optimization only in the zba/xandesperf/xqciac/xtheadba
16331 // extension.
16332 if (!Subtarget.hasShlAdd(ShAmt: 3))
16333 return SDValue();
16334
16335 // Skip for vector types and larger types.
16336 EVT VT = N->getValueType(ResNo: 0);
16337 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
16338 return SDValue();
16339
16340 // The two operand nodes must be SHL and have no other use.
16341 SDValue N0 = N->getOperand(Num: 0);
16342 SDValue N1 = N->getOperand(Num: 1);
16343 if (N0->getOpcode() != ISD::SHL || N1->getOpcode() != ISD::SHL ||
16344 !N0->hasOneUse() || !N1->hasOneUse())
16345 return SDValue();
16346
16347 // Check c0 and c1.
16348 auto *N0C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
16349 auto *N1C = dyn_cast<ConstantSDNode>(Val: N1->getOperand(Num: 1));
16350 if (!N0C || !N1C)
16351 return SDValue();
16352 int64_t C0 = N0C->getSExtValue();
16353 int64_t C1 = N1C->getSExtValue();
16354 if (C0 <= 0 || C1 <= 0)
16355 return SDValue();
16356
16357 int64_t Diff = std::abs(i: C0 - C1);
16358 if (!Subtarget.hasShlAdd(ShAmt: Diff))
16359 return SDValue();
16360
16361 // Build nodes.
16362 SDLoc DL(N);
16363 int64_t Bits = std::min(a: C0, b: C1);
16364 SDValue NS = (C0 < C1) ? N0->getOperand(Num: 0) : N1->getOperand(Num: 0);
16365 SDValue NL = (C0 > C1) ? N0->getOperand(Num: 0) : N1->getOperand(Num: 0);
16366 SDValue SHADD = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: NL,
16367 N2: DAG.getTargetConstant(Val: Diff, DL, VT), N3: NS);
16368 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: SHADD, N2: DAG.getConstant(Val: Bits, DL, VT));
16369}
16370
16371// Check if this SDValue is an add immediate that is fed by a shift of 1, 2,
16372// or 3.
16373static SDValue combineShlAddIAddImpl(SDNode *N, SDValue AddI, SDValue Other,
16374 SelectionDAG &DAG) {
16375 using namespace llvm::SDPatternMatch;
16376
16377 // Looking for a reg-reg add and not an addi.
16378 if (isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
16379 return SDValue();
16380
16381 // Based on testing it seems that performance degrades if the ADDI has
16382 // more than 2 uses.
16383 if (AddI->use_size() > 2)
16384 return SDValue();
16385
16386 APInt AddVal;
16387 SDValue SHLVal;
16388 if (!sd_match(N: AddI, P: m_Add(L: m_Value(N&: SHLVal), R: m_ConstInt(V&: AddVal))))
16389 return SDValue();
16390
16391 APInt VShift;
16392 if (!sd_match(N: SHLVal, P: m_OneUse(P: m_Shl(L: m_Value(), R: m_ConstInt(V&: VShift)))))
16393 return SDValue();
16394
16395 if (VShift.slt(RHS: 1) || VShift.sgt(RHS: 3))
16396 return SDValue();
16397
16398 SDLoc DL(N);
16399 EVT VT = N->getValueType(ResNo: 0);
16400 // The shift must be positive but the add can be signed.
16401 uint64_t ShlConst = VShift.getZExtValue();
16402 int64_t AddConst = AddVal.getSExtValue();
16403
16404 SDValue SHADD = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: SHLVal->getOperand(Num: 0),
16405 N2: DAG.getTargetConstant(Val: ShlConst, DL, VT), N3: Other);
16406 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: SHADD,
16407 N2: DAG.getSignedConstant(Val: AddConst, DL, VT));
16408}
16409
16410// Optimize (add (add (shl x, c0), c1), y) ->
16411// (ADDI (SH*ADD y, x), c1), if c0 equals to [1|2|3].
16412static SDValue combineShlAddIAdd(SDNode *N, SelectionDAG &DAG,
16413 const RISCVSubtarget &Subtarget) {
16414 // Perform this optimization only in the zba extension.
16415 if (!ReassocShlAddiAdd || !Subtarget.hasShlAdd(ShAmt: 3))
16416 return SDValue();
16417
16418 // Skip for vector types and larger types.
16419 EVT VT = N->getValueType(ResNo: 0);
16420 if (VT != Subtarget.getXLenVT())
16421 return SDValue();
16422
16423 SDValue AddI = N->getOperand(Num: 0);
16424 SDValue Other = N->getOperand(Num: 1);
16425 if (SDValue V = combineShlAddIAddImpl(N, AddI, Other, DAG))
16426 return V;
16427 if (SDValue V = combineShlAddIAddImpl(N, AddI: Other, Other: AddI, DAG))
16428 return V;
16429 return SDValue();
16430}
16431
16432// Combine a constant select operand into its use:
16433//
16434// (and (select cond, -1, c), x)
16435// -> (select cond, x, (and x, c)) [AllOnes=1]
16436// (or (select cond, 0, c), x)
16437// -> (select cond, x, (or x, c)) [AllOnes=0]
16438// (xor (select cond, 0, c), x)
16439// -> (select cond, x, (xor x, c)) [AllOnes=0]
16440// (add (select cond, 0, c), x)
16441// -> (select cond, x, (add x, c)) [AllOnes=0]
16442// (sub x, (select cond, 0, c))
16443// -> (select cond, x, (sub x, c)) [AllOnes=0]
16444static SDValue combineSelectAndUse(SDNode *N, SDValue Slct, SDValue OtherOp,
16445 SelectionDAG &DAG, bool AllOnes,
16446 const RISCVSubtarget &Subtarget) {
16447 EVT VT = N->getValueType(ResNo: 0);
16448
16449 // Skip vectors.
16450 if (VT.isVector())
16451 return SDValue();
16452
16453 if (!Subtarget.hasConditionalMoveFusion()) {
16454 // (select cond, x, (and x, c)) has custom lowering with Zicond.
16455 if (!Subtarget.hasCZEROLike() || N->getOpcode() != ISD::AND)
16456 return SDValue();
16457
16458 // Maybe harmful when condition code has multiple use.
16459 if (Slct.getOpcode() == ISD::SELECT && !Slct.getOperand(i: 0).hasOneUse())
16460 return SDValue();
16461
16462 // Maybe harmful when VT is wider than XLen.
16463 if (VT.getSizeInBits() > Subtarget.getXLen())
16464 return SDValue();
16465 }
16466
16467 if ((Slct.getOpcode() != ISD::SELECT &&
16468 Slct.getOpcode() != RISCVISD::SELECT_CC) ||
16469 !Slct.hasOneUse())
16470 return SDValue();
16471
16472 auto isZeroOrAllOnes = [](SDValue N, bool AllOnes) {
16473 return AllOnes ? isAllOnesConstant(V: N) : isNullConstant(V: N);
16474 };
16475
16476 bool SwapSelectOps;
16477 unsigned OpOffset = Slct.getOpcode() == RISCVISD::SELECT_CC ? 2 : 0;
16478 SDValue TrueVal = Slct.getOperand(i: 1 + OpOffset);
16479 SDValue FalseVal = Slct.getOperand(i: 2 + OpOffset);
16480 SDValue NonConstantVal;
16481 if (isZeroOrAllOnes(TrueVal, AllOnes)) {
16482 SwapSelectOps = false;
16483 NonConstantVal = FalseVal;
16484 } else if (isZeroOrAllOnes(FalseVal, AllOnes)) {
16485 SwapSelectOps = true;
16486 NonConstantVal = TrueVal;
16487 } else
16488 return SDValue();
16489
16490 // Slct is now know to be the desired identity constant when CC is true.
16491 TrueVal = OtherOp;
16492 FalseVal = DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT, N1: OtherOp, N2: NonConstantVal);
16493 // Unless SwapSelectOps says the condition should be false.
16494 if (SwapSelectOps)
16495 std::swap(a&: TrueVal, b&: FalseVal);
16496
16497 if (Slct.getOpcode() == RISCVISD::SELECT_CC)
16498 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL: SDLoc(N), VT,
16499 Ops: {Slct.getOperand(i: 0), Slct.getOperand(i: 1),
16500 Slct.getOperand(i: 2), TrueVal, FalseVal});
16501
16502 return DAG.getNode(Opcode: ISD::SELECT, DL: SDLoc(N), VT,
16503 Ops: {Slct.getOperand(i: 0), TrueVal, FalseVal});
16504}
16505
16506// Attempt combineSelectAndUse on each operand of a commutative operator N.
16507static SDValue combineSelectAndUseCommutative(SDNode *N, SelectionDAG &DAG,
16508 bool AllOnes,
16509 const RISCVSubtarget &Subtarget) {
16510 SDValue N0 = N->getOperand(Num: 0);
16511 SDValue N1 = N->getOperand(Num: 1);
16512 if (SDValue Result = combineSelectAndUse(N, Slct: N0, OtherOp: N1, DAG, AllOnes, Subtarget))
16513 return Result;
16514 if (SDValue Result = combineSelectAndUse(N, Slct: N1, OtherOp: N0, DAG, AllOnes, Subtarget))
16515 return Result;
16516 return SDValue();
16517}
16518
16519// Transform (add (mul x, c0), c1) ->
16520// (add (mul (add x, c1/c0), c0), c1%c0).
16521// if c1/c0 and c1%c0 are simm12, while c1 is not. A special corner case
16522// that should be excluded is when c0*(c1/c0) is simm12, which will lead
16523// to an infinite loop in DAGCombine if transformed.
16524// Or transform (add (mul x, c0), c1) ->
16525// (add (mul (add x, c1/c0+1), c0), c1%c0-c0),
16526// if c1/c0+1 and c1%c0-c0 are simm12, while c1 is not. A special corner
16527// case that should be excluded is when c0*(c1/c0+1) is simm12, which will
16528// lead to an infinite loop in DAGCombine if transformed.
16529// Or transform (add (mul x, c0), c1) ->
16530// (add (mul (add x, c1/c0-1), c0), c1%c0+c0),
16531// if c1/c0-1 and c1%c0+c0 are simm12, while c1 is not. A special corner
16532// case that should be excluded is when c0*(c1/c0-1) is simm12, which will
16533// lead to an infinite loop in DAGCombine if transformed.
16534// Or transform (add (mul x, c0), c1) ->
16535// (mul (add x, c1/c0), c0).
16536// if c1%c0 is zero, and c1/c0 is simm12 while c1 is not.
16537static SDValue transformAddImmMulImm(SDNode *N, SelectionDAG &DAG,
16538 const RISCVSubtarget &Subtarget) {
16539 // Skip for vector types and larger types.
16540 EVT VT = N->getValueType(ResNo: 0);
16541 if (VT.isVector() || VT.getSizeInBits() > Subtarget.getXLen())
16542 return SDValue();
16543 // The first operand node must be a MUL and has no other use.
16544 SDValue N0 = N->getOperand(Num: 0);
16545 if (!N0->hasOneUse() || N0->getOpcode() != ISD::MUL)
16546 return SDValue();
16547 // Check if c0 and c1 match above conditions.
16548 auto *N0C = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
16549 auto *N1C = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
16550 if (!N0C || !N1C)
16551 return SDValue();
16552 // If N0C has multiple uses it's possible one of the cases in
16553 // DAGCombiner::isMulAddWithConstProfitable will be true, which would result
16554 // in an infinite loop.
16555 if (!N0C->hasOneUse())
16556 return SDValue();
16557 int64_t C0 = N0C->getSExtValue();
16558 int64_t C1 = N1C->getSExtValue();
16559 int64_t CA, CB;
16560 if (C0 == -1 || C0 == 0 || C0 == 1 || isInt<12>(x: C1))
16561 return SDValue();
16562 // Search for proper CA (non-zero) and CB that both are simm12.
16563 if ((C1 / C0) != 0 && isInt<12>(x: C1 / C0) && isInt<12>(x: C1 % C0) &&
16564 !isInt<12>(x: C0 * (C1 / C0))) {
16565 CA = C1 / C0;
16566 CB = C1 % C0;
16567 } else if ((C1 / C0 + 1) != 0 && isInt<12>(x: C1 / C0 + 1) &&
16568 isInt<12>(x: C1 % C0 - C0) && !isInt<12>(x: C0 * (C1 / C0 + 1))) {
16569 CA = C1 / C0 + 1;
16570 CB = C1 % C0 - C0;
16571 } else if ((C1 / C0 - 1) != 0 && isInt<12>(x: C1 / C0 - 1) &&
16572 isInt<12>(x: C1 % C0 + C0) && !isInt<12>(x: C0 * (C1 / C0 - 1))) {
16573 CA = C1 / C0 - 1;
16574 CB = C1 % C0 + C0;
16575 } else
16576 return SDValue();
16577 // Build new nodes (add (mul (add x, c1/c0), c0), c1%c0).
16578 SDLoc DL(N);
16579 SDValue New0 = DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: N0->getOperand(Num: 0),
16580 N2: DAG.getSignedConstant(Val: CA, DL, VT));
16581 SDValue New1 =
16582 DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: New0, N2: DAG.getSignedConstant(Val: C0, DL, VT));
16583 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: New1, N2: DAG.getSignedConstant(Val: CB, DL, VT));
16584}
16585
16586// add (zext, zext) -> zext (add (zext, zext))
16587// sub (zext, zext) -> sext (sub (zext, zext))
16588// mul (zext, zext) -> zext (mul (zext, zext))
16589// sdiv (zext, zext) -> zext (sdiv (zext, zext))
16590// udiv (zext, zext) -> zext (udiv (zext, zext))
16591// srem (zext, zext) -> zext (srem (zext, zext))
16592// urem (zext, zext) -> zext (urem (zext, zext))
16593//
16594// where the sum of the extend widths match, and the the range of the bin op
16595// fits inside the width of the narrower bin op. (For profitability on rvv, we
16596// use a power of two for both inner and outer extend.)
16597static SDValue combineBinOpOfZExt(SDNode *N, SelectionDAG &DAG) {
16598
16599 EVT VT = N->getValueType(ResNo: 0);
16600 if (!VT.isVector() || !DAG.getTargetLoweringInfo().isTypeLegal(VT))
16601 return SDValue();
16602
16603 SDValue N0 = N->getOperand(Num: 0);
16604 SDValue N1 = N->getOperand(Num: 1);
16605 if (N0.getOpcode() != ISD::ZERO_EXTEND || N1.getOpcode() != ISD::ZERO_EXTEND)
16606 return SDValue();
16607 if (!N0.hasOneUse() || !N1.hasOneUse())
16608 return SDValue();
16609
16610 SDValue Src0 = N0.getOperand(i: 0);
16611 SDValue Src1 = N1.getOperand(i: 0);
16612 EVT SrcVT = Src0.getValueType();
16613 if (!DAG.getTargetLoweringInfo().isTypeLegal(VT: SrcVT) ||
16614 SrcVT != Src1.getValueType() || SrcVT.getScalarSizeInBits() < 8 ||
16615 SrcVT.getScalarSizeInBits() >= VT.getScalarSizeInBits() / 2)
16616 return SDValue();
16617
16618 LLVMContext &C = *DAG.getContext();
16619 EVT ElemVT = VT.getVectorElementType().getHalfSizedIntegerVT(Context&: C);
16620 EVT NarrowVT = EVT::getVectorVT(Context&: C, VT: ElemVT, EC: VT.getVectorElementCount());
16621
16622 Src0 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Src0), VT: NarrowVT, Operand: Src0);
16623 Src1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL: SDLoc(Src1), VT: NarrowVT, Operand: Src1);
16624
16625 // Src0 and Src1 are zero extended, so they're always positive if signed.
16626 //
16627 // sub can produce a negative from two positive operands, so it needs sign
16628 // extended. Other nodes produce a positive from two positive operands, so
16629 // zero extend instead.
16630 unsigned OuterExtend =
16631 N->getOpcode() == ISD::SUB ? ISD::SIGN_EXTEND : ISD::ZERO_EXTEND;
16632
16633 return DAG.getNode(
16634 Opcode: OuterExtend, DL: SDLoc(N), VT,
16635 Operand: DAG.getNode(Opcode: N->getOpcode(), DL: SDLoc(N), VT: NarrowVT, N1: Src0, N2: Src1));
16636}
16637
16638// Try to turn (add (xor bool, 1) -1) into (neg bool).
16639static SDValue combineAddOfBooleanXor(SDNode *N, SelectionDAG &DAG) {
16640 SDValue N0 = N->getOperand(Num: 0);
16641 SDValue N1 = N->getOperand(Num: 1);
16642 EVT VT = N->getValueType(ResNo: 0);
16643 SDLoc DL(N);
16644
16645 // RHS should be -1.
16646 if (!isAllOnesConstant(V: N1))
16647 return SDValue();
16648
16649 // Look for (xor X, 1).
16650 if (N0.getOpcode() != ISD::XOR || !isOneConstant(V: N0.getOperand(i: 1)))
16651 return SDValue();
16652
16653 // First xor input should be 0 or 1.
16654 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
16655 if (!DAG.MaskedValueIsZero(Op: N0.getOperand(i: 0), Mask))
16656 return SDValue();
16657
16658 // Emit a negate of the setcc.
16659 return DAG.getNegative(Val: N0.getOperand(i: 0), DL, VT);
16660}
16661
16662static SDValue performADDCombine(SDNode *N,
16663 TargetLowering::DAGCombinerInfo &DCI,
16664 const RISCVSubtarget &Subtarget) {
16665 SelectionDAG &DAG = DCI.DAG;
16666 if (SDValue V = combineAddOfBooleanXor(N, DAG))
16667 return V;
16668 if (SDValue V = transformAddImmMulImm(N, DAG, Subtarget))
16669 return V;
16670 if (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) {
16671 if (SDValue V = transformAddShlImm(N, DAG, Subtarget))
16672 return V;
16673 if (SDValue V = combineShlAddIAdd(N, DAG, Subtarget))
16674 return V;
16675 }
16676 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
16677 return V;
16678 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
16679 return V;
16680 if (SDValue V = combineBinOpOfZExt(N, DAG))
16681 return V;
16682
16683 // fold (add (select lhs, rhs, cc, 0, y), x) ->
16684 // (select lhs, rhs, cc, x, (add x, y))
16685 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
16686}
16687
16688// Try to turn a sub boolean RHS and constant LHS into an addi.
16689static SDValue combineSubOfBoolean(SDNode *N, SelectionDAG &DAG) {
16690 SDValue N0 = N->getOperand(Num: 0);
16691 SDValue N1 = N->getOperand(Num: 1);
16692 EVT VT = N->getValueType(ResNo: 0);
16693 SDLoc DL(N);
16694
16695 // Require a constant LHS.
16696 auto *N0C = dyn_cast<ConstantSDNode>(Val&: N0);
16697 if (!N0C)
16698 return SDValue();
16699
16700 // All our optimizations involve subtracting 1 from the immediate and forming
16701 // an ADDI. Make sure the new immediate is valid for an ADDI.
16702 APInt ImmValMinus1 = N0C->getAPIntValue() - 1;
16703 if (!ImmValMinus1.isSignedIntN(N: 12))
16704 return SDValue();
16705
16706 SDValue NewLHS;
16707 if (N1.getOpcode() == ISD::SETCC && N1.hasOneUse()) {
16708 // (sub constant, (setcc x, y, eq/neq)) ->
16709 // (add (setcc x, y, neq/eq), constant - 1)
16710 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
16711 EVT SetCCOpVT = N1.getOperand(i: 0).getValueType();
16712 if (!isIntEqualitySetCC(Code: CCVal) || !SetCCOpVT.isInteger())
16713 return SDValue();
16714 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: SetCCOpVT);
16715 NewLHS =
16716 DAG.getSetCC(DL: SDLoc(N1), VT, LHS: N1.getOperand(i: 0), RHS: N1.getOperand(i: 1), Cond: CCVal);
16717 } else if (N1.getOpcode() == ISD::XOR && isOneConstant(V: N1.getOperand(i: 1)) &&
16718 N1.getOperand(i: 0).getOpcode() == ISD::SETCC) {
16719 // (sub C, (xor (setcc), 1)) -> (add (setcc), C-1).
16720 // Since setcc returns a bool the xor is equivalent to 1-setcc.
16721 NewLHS = N1.getOperand(i: 0);
16722 } else
16723 return SDValue();
16724
16725 SDValue NewRHS = DAG.getConstant(Val: ImmValMinus1, DL, VT);
16726 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: NewLHS, N2: NewRHS);
16727}
16728
16729// Looks for (sub (shl X, 8-Y), (shr X, Y)) where the Y-th bit in each byte is
16730// potentially set. It is fine for Y to be 0, meaning that (sub (shl X, 8), X)
16731// is also valid. Replace with (orc.b X). For example, 0b0000_1000_0000_1000 is
16732// valid with Y=3, while 0b0000_1000_0000_0100 is not.
16733static SDValue combineSubShiftToOrcB(SDNode *N, SelectionDAG &DAG,
16734 const RISCVSubtarget &Subtarget) {
16735 if (!Subtarget.hasStdExtZbb())
16736 return SDValue();
16737
16738 EVT VT = N->getValueType(ResNo: 0);
16739
16740 if (VT != Subtarget.getXLenVT() && VT != MVT::i32 && VT != MVT::i16)
16741 return SDValue();
16742
16743 SDValue N0 = N->getOperand(Num: 0);
16744 SDValue N1 = N->getOperand(Num: 1);
16745
16746 if (N0->getOpcode() != ISD::SHL)
16747 return SDValue();
16748
16749 auto *ShAmtCLeft = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 1));
16750 if (!ShAmtCLeft)
16751 return SDValue();
16752 unsigned ShiftedAmount = 8 - ShAmtCLeft->getZExtValue();
16753
16754 if (ShiftedAmount >= 8)
16755 return SDValue();
16756
16757 SDValue LeftShiftOperand = N0->getOperand(Num: 0);
16758 SDValue RightShiftOperand = N1;
16759
16760 if (ShiftedAmount != 0) { // Right operand must be a right shift.
16761 if (N1->getOpcode() != ISD::SRL)
16762 return SDValue();
16763 auto *ShAmtCRight = dyn_cast<ConstantSDNode>(Val: N1.getOperand(i: 1));
16764 if (!ShAmtCRight || ShAmtCRight->getZExtValue() != ShiftedAmount)
16765 return SDValue();
16766 RightShiftOperand = N1.getOperand(i: 0);
16767 }
16768
16769 // At least one shift should have a single use.
16770 if (!N0.hasOneUse() && (ShiftedAmount == 0 || !N1.hasOneUse()))
16771 return SDValue();
16772
16773 if (LeftShiftOperand != RightShiftOperand)
16774 return SDValue();
16775
16776 APInt Mask = APInt::getSplat(NewLen: VT.getSizeInBits(), V: APInt(8, 0x1));
16777 Mask <<= ShiftedAmount;
16778 // Check that X has indeed the right shape (only the Y-th bit can be set in
16779 // every byte).
16780 if (!DAG.MaskedValueIsZero(Op: LeftShiftOperand, Mask: ~Mask))
16781 return SDValue();
16782
16783 return DAG.getNode(Opcode: RISCVISD::ORC_B, DL: SDLoc(N), VT, Operand: LeftShiftOperand);
16784}
16785
16786static SDValue performSUBCombine(SDNode *N, SelectionDAG &DAG,
16787 const RISCVSubtarget &Subtarget) {
16788 if (SDValue V = combineSubOfBoolean(N, DAG))
16789 return V;
16790
16791 EVT VT = N->getValueType(ResNo: 0);
16792 SDValue N0 = N->getOperand(Num: 0);
16793 SDValue N1 = N->getOperand(Num: 1);
16794 // fold (sub 0, (setcc x, 0, setlt)) -> (sra x, xlen - 1)
16795 if (isNullConstant(V: N0) && N1.getOpcode() == ISD::SETCC && N1.hasOneUse() &&
16796 isNullConstant(V: N1.getOperand(i: 1)) &&
16797 N1.getValueType() == N1.getOperand(i: 0).getValueType()) {
16798 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: N1.getOperand(i: 2))->get();
16799 if (CCVal == ISD::SETLT) {
16800 SDLoc DL(N);
16801 unsigned ShAmt = N0.getValueSizeInBits() - 1;
16802 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: N1.getOperand(i: 0),
16803 N2: DAG.getConstant(Val: ShAmt, DL, VT));
16804 }
16805 }
16806
16807 if (SDValue V = combineBinOpOfZExt(N, DAG))
16808 return V;
16809 if (SDValue V = combineSubShiftToOrcB(N, DAG, Subtarget))
16810 return V;
16811
16812 // fold (sub x, (select lhs, rhs, cc, 0, y)) ->
16813 // (select lhs, rhs, cc, x, (sub x, y))
16814 return combineSelectAndUse(N, Slct: N1, OtherOp: N0, DAG, /*AllOnes*/ false, Subtarget);
16815}
16816
16817// Apply DeMorgan's law to (and/or (xor X, 1), (xor Y, 1)) if X and Y are 0/1.
16818// Legalizing setcc can introduce xors like this. Doing this transform reduces
16819// the number of xors and may allow the xor to fold into a branch condition.
16820static SDValue combineDeMorganOfBoolean(SDNode *N, SelectionDAG &DAG) {
16821 SDValue N0 = N->getOperand(Num: 0);
16822 SDValue N1 = N->getOperand(Num: 1);
16823 bool IsAnd = N->getOpcode() == ISD::AND;
16824
16825 if (N0.getOpcode() != ISD::XOR || N1.getOpcode() != ISD::XOR)
16826 return SDValue();
16827
16828 if (!N0.hasOneUse() || !N1.hasOneUse())
16829 return SDValue();
16830
16831 SDValue N01 = N0.getOperand(i: 1);
16832 SDValue N11 = N1.getOperand(i: 1);
16833
16834 // For AND, SimplifyDemandedBits may have turned one of the (xor X, 1) into
16835 // (xor X, -1) based on the upper bits of the other operand being 0. If the
16836 // operation is And, allow one of the Xors to use -1.
16837 if (isOneConstant(V: N01)) {
16838 if (!isOneConstant(V: N11) && !(IsAnd && isAllOnesConstant(V: N11)))
16839 return SDValue();
16840 } else if (isOneConstant(V: N11)) {
16841 // N01 and N11 being 1 was already handled. Handle N11==1 and N01==-1.
16842 if (!(IsAnd && isAllOnesConstant(V: N01)))
16843 return SDValue();
16844 } else
16845 return SDValue();
16846
16847 EVT VT = N->getValueType(ResNo: 0);
16848
16849 SDValue N00 = N0.getOperand(i: 0);
16850 SDValue N10 = N1.getOperand(i: 0);
16851
16852 // The LHS of the xors needs to be 0/1.
16853 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
16854 if (!DAG.MaskedValueIsZero(Op: N00, Mask) || !DAG.MaskedValueIsZero(Op: N10, Mask))
16855 return SDValue();
16856
16857 // Invert the opcode and insert a new xor.
16858 SDLoc DL(N);
16859 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
16860 SDValue Logic = DAG.getNode(Opcode: Opc, DL, VT, N1: N00, N2: N10);
16861 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: Logic, N2: DAG.getConstant(Val: 1, DL, VT));
16862}
16863
16864// Fold (vXi8 (trunc (vselect (setltu, X, 256), X, (sext (setgt X, 0))))) to
16865// (vXi8 (trunc (smin (smax X, 0), 255))). This represents saturating a signed
16866// value to an unsigned value. This will be lowered to vmax and series of
16867// vnclipu instructions later. This can be extended to other truncated types
16868// other than i8 by replacing 256 and 255 with the equivalent constants for the
16869// type.
16870static SDValue combineTruncSelectToSMaxUSat(SDNode *N, SelectionDAG &DAG) {
16871 EVT VT = N->getValueType(ResNo: 0);
16872 SDValue N0 = N->getOperand(Num: 0);
16873 EVT SrcVT = N0.getValueType();
16874
16875 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
16876 if (!VT.isVector() || !TLI.isTypeLegal(VT) || !TLI.isTypeLegal(VT: SrcVT))
16877 return SDValue();
16878
16879 if (N0.getOpcode() != ISD::VSELECT || !N0.hasOneUse())
16880 return SDValue();
16881
16882 SDValue Cond = N0.getOperand(i: 0);
16883 SDValue True = N0.getOperand(i: 1);
16884 SDValue False = N0.getOperand(i: 2);
16885
16886 if (Cond.getOpcode() != ISD::SETCC)
16887 return SDValue();
16888
16889 // FIXME: Support the version of this pattern with the select operands
16890 // swapped.
16891 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
16892 if (CCVal != ISD::SETULT)
16893 return SDValue();
16894
16895 SDValue CondLHS = Cond.getOperand(i: 0);
16896 SDValue CondRHS = Cond.getOperand(i: 1);
16897
16898 if (CondLHS != True)
16899 return SDValue();
16900
16901 unsigned ScalarBits = VT.getScalarSizeInBits();
16902
16903 // FIXME: Support other constants.
16904 ConstantSDNode *CondRHSC = isConstOrConstSplat(N: CondRHS);
16905 if (!CondRHSC || CondRHSC->getAPIntValue() != (1ULL << ScalarBits))
16906 return SDValue();
16907
16908 if (False.getOpcode() != ISD::SIGN_EXTEND)
16909 return SDValue();
16910
16911 False = False.getOperand(i: 0);
16912
16913 if (False.getOpcode() != ISD::SETCC || False.getOperand(i: 0) != True)
16914 return SDValue();
16915
16916 ConstantSDNode *FalseRHSC = isConstOrConstSplat(N: False.getOperand(i: 1));
16917 if (!FalseRHSC || !FalseRHSC->isZero())
16918 return SDValue();
16919
16920 ISD::CondCode CCVal2 = cast<CondCodeSDNode>(Val: False.getOperand(i: 2))->get();
16921 if (CCVal2 != ISD::SETGT)
16922 return SDValue();
16923
16924 // Emit the signed to unsigned saturation pattern.
16925 SDLoc DL(N);
16926 SDValue Max =
16927 DAG.getNode(Opcode: ISD::SMAX, DL, VT: SrcVT, N1: True, N2: DAG.getConstant(Val: 0, DL, VT: SrcVT));
16928 SDValue Min =
16929 DAG.getNode(Opcode: ISD::SMIN, DL, VT: SrcVT, N1: Max,
16930 N2: DAG.getConstant(Val: (1ULL << ScalarBits) - 1, DL, VT: SrcVT));
16931 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: Min);
16932}
16933
16934// Handle P extension truncate patterns:
16935// ASUB/ASUBU: (trunc (srl (sub ([s|z]ext a), ([s|z]ext b)), 1))
16936// MULHSU: (trunc (srl (mul (sext a), (zext b)), EltBits))
16937// MULHR*: (trunc (srl (add (mul (sext a), (zext b)), round_const), EltBits))
16938static SDValue combinePExtTruncate(SDNode *N, SelectionDAG &DAG,
16939 const RISCVSubtarget &Subtarget) {
16940 SDValue N0 = N->getOperand(Num: 0);
16941 EVT VT = N->getValueType(ResNo: 0);
16942 if (N0.getOpcode() != ISD::SRL)
16943 return SDValue();
16944
16945 MVT VecVT = VT.getSimpleVT();
16946 if (VecVT != MVT::v4i16 && VecVT != MVT::v2i16 && VecVT != MVT::v8i8 &&
16947 VecVT != MVT::v4i8 && VecVT != MVT::v2i32)
16948 return SDValue();
16949
16950 // Check if shift amount is a splat constant
16951 SDValue ShAmt = N0.getOperand(i: 1);
16952 if (ShAmt.getOpcode() != ISD::BUILD_VECTOR)
16953 return SDValue();
16954
16955 BuildVectorSDNode *BV = dyn_cast<BuildVectorSDNode>(Val: ShAmt.getNode());
16956 if (!BV)
16957 return SDValue();
16958 SDValue Splat = BV->getSplatValue();
16959 if (!Splat)
16960 return SDValue();
16961 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val&: Splat);
16962 if (!C)
16963 return SDValue();
16964
16965 SDValue Op = N0.getOperand(i: 0);
16966 unsigned ShAmtVal = C->getZExtValue();
16967 unsigned EltBits = VecVT.getScalarSizeInBits();
16968
16969 // Check for rounding pattern: (add (mul ...), round_const)
16970 bool IsRounding = false;
16971 if (Op.getOpcode() == ISD::ADD && (EltBits == 16 || EltBits == 32)) {
16972 SDValue AddRHS = Op.getOperand(i: 1);
16973 if (auto *RndBV = dyn_cast<BuildVectorSDNode>(Val: AddRHS.getNode())) {
16974 if (auto *RndC =
16975 dyn_cast_or_null<ConstantSDNode>(Val: RndBV->getSplatValue())) {
16976 uint64_t ExpectedRnd = 1ULL << (EltBits - 1);
16977 if (RndC->getZExtValue() == ExpectedRnd &&
16978 Op.getOperand(i: 0).getOpcode() == ISD::MUL) {
16979 Op = Op.getOperand(i: 0);
16980 IsRounding = true;
16981 }
16982 }
16983 }
16984 }
16985
16986 // Ensure Op is a binary operation before accessing its operands.
16987 if (Op.getNumOperands() != 2)
16988 return SDValue();
16989
16990 SDValue LHS = Op.getOperand(i: 0);
16991 SDValue RHS = Op.getOperand(i: 1);
16992
16993 bool LHSIsSExt = LHS.getOpcode() == ISD::SIGN_EXTEND;
16994 bool LHSIsZExt = LHS.getOpcode() == ISD::ZERO_EXTEND;
16995 bool RHSIsSExt = RHS.getOpcode() == ISD::SIGN_EXTEND;
16996 bool RHSIsZExt = RHS.getOpcode() == ISD::ZERO_EXTEND;
16997
16998 if (!(LHSIsSExt || LHSIsZExt) || !(RHSIsSExt || RHSIsZExt))
16999 return SDValue();
17000
17001 SDValue A = LHS.getOperand(i: 0);
17002 SDValue B = RHS.getOperand(i: 0);
17003
17004 if (A.getValueType() != VT || B.getValueType() != VT)
17005 return SDValue();
17006
17007 unsigned Opc;
17008 switch (Op.getOpcode()) {
17009 default:
17010 return SDValue();
17011 case ISD::SUB:
17012 // PASUB/PASUBU: shift amount must be 1
17013 if (ShAmtVal != 1)
17014 return SDValue();
17015 if (LHSIsSExt && RHSIsSExt)
17016 Opc = RISCVISD::ASUB;
17017 else if (LHSIsZExt && RHSIsZExt)
17018 Opc = RISCVISD::ASUBU;
17019 else
17020 return SDValue();
17021 break;
17022 case ISD::MUL:
17023 // MULH*/MULHR*: shift amount must be element size, only for i16/i32
17024 if (ShAmtVal != EltBits || (EltBits != 16 && EltBits != 32))
17025 return SDValue();
17026 if (IsRounding) {
17027 if (LHSIsSExt && RHSIsSExt) {
17028 Opc = RISCVISD::MULHR;
17029 } else if (LHSIsZExt && RHSIsZExt) {
17030 Opc = RISCVISD::MULHRU;
17031 } else if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
17032 Opc = RISCVISD::MULHRSU;
17033 // commuted case
17034 if (LHSIsZExt && RHSIsSExt)
17035 std::swap(a&: A, b&: B);
17036 } else {
17037 return SDValue();
17038 }
17039 } else {
17040 if ((LHSIsSExt && RHSIsZExt) || (LHSIsZExt && RHSIsSExt)) {
17041 Opc = RISCVISD::MULHSU;
17042 // commuted case
17043 if (LHSIsZExt && RHSIsSExt)
17044 std::swap(a&: A, b&: B);
17045 } else
17046 return SDValue();
17047 }
17048 break;
17049 }
17050
17051 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT, Ops: {A, B});
17052}
17053
17054static SDValue performTRUNCATECombine(SDNode *N, SelectionDAG &DAG,
17055 const RISCVSubtarget &Subtarget) {
17056 SDValue N0 = N->getOperand(Num: 0);
17057 EVT VT = N->getValueType(ResNo: 0);
17058
17059 if (VT.isFixedLengthVector() && Subtarget.hasStdExtP())
17060 return combinePExtTruncate(N, DAG, Subtarget);
17061
17062 // Pre-promote (i1 (truncate (srl X, Y))) on RV64 with Zbs without zero
17063 // extending X. This is safe since we only need the LSB after the shift and
17064 // shift amounts larger than 31 would produce poison. If we wait until
17065 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
17066 // to use a BEXT instruction.
17067 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() && VT == MVT::i1 &&
17068 N0.getValueType() == MVT::i32 && N0.getOpcode() == ISD::SRL &&
17069 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) && N0.hasOneUse()) {
17070 SDLoc DL(N0);
17071 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
17072 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
17073 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
17074 return DAG.getNode(Opcode: ISD::TRUNCATE, DL: SDLoc(N), VT, Operand: Srl);
17075 }
17076
17077 return combineTruncSelectToSMaxUSat(N, DAG);
17078}
17079
17080// InstCombinerImpl::transformZExtICmp will narrow a zext of an icmp with a
17081// truncation. But RVV doesn't have truncation instructions for more than twice
17082// the bitwidth.
17083//
17084// E.g. trunc <vscale x 1 x i64> %x to <vscale x 1 x i8> will generate:
17085//
17086// vsetvli a0, zero, e32, m2, ta, ma
17087// vnsrl.wi v12, v8, 0
17088// vsetvli zero, zero, e16, m1, ta, ma
17089// vnsrl.wi v8, v12, 0
17090// vsetvli zero, zero, e8, mf2, ta, ma
17091// vnsrl.wi v8, v8, 0
17092//
17093// So reverse the combine so we generate an vmseq/vmsne again:
17094//
17095// and (lshr (trunc X), ShAmt), 1
17096// -->
17097// zext (icmp ne (and X, (1 << ShAmt)), 0)
17098//
17099// and (lshr (not (trunc X)), ShAmt), 1
17100// -->
17101// zext (icmp eq (and X, (1 << ShAmt)), 0)
17102static SDValue reverseZExtICmpCombine(SDNode *N, SelectionDAG &DAG,
17103 const RISCVSubtarget &Subtarget) {
17104 using namespace SDPatternMatch;
17105 SDLoc DL(N);
17106
17107 if (!Subtarget.hasVInstructions())
17108 return SDValue();
17109
17110 EVT VT = N->getValueType(ResNo: 0);
17111 if (!VT.isVector())
17112 return SDValue();
17113
17114 APInt ShAmt;
17115 SDValue Inner;
17116 if (!sd_match(N, P: m_And(L: m_OneUse(P: m_Srl(L: m_Value(N&: Inner), R: m_ConstInt(V&: ShAmt))),
17117 R: m_One())))
17118 return SDValue();
17119
17120 SDValue X;
17121 bool IsNot;
17122 if (sd_match(N: Inner, P: m_Not(V: m_Trunc(Op: m_Value(N&: X)))))
17123 IsNot = true;
17124 else if (sd_match(N: Inner, P: m_Trunc(Op: m_Value(N&: X))))
17125 IsNot = false;
17126 else
17127 return SDValue();
17128
17129 EVT WideVT = X.getValueType();
17130 if (VT.getScalarSizeInBits() >= WideVT.getScalarSizeInBits() / 2)
17131 return SDValue();
17132
17133 SDValue Res =
17134 DAG.getNode(Opcode: ISD::AND, DL, VT: WideVT, N1: X,
17135 N2: DAG.getConstant(Val: 1ULL << ShAmt.getZExtValue(), DL, VT: WideVT));
17136 Res = DAG.getSetCC(DL,
17137 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
17138 EC: WideVT.getVectorElementCount()),
17139 LHS: Res, RHS: DAG.getConstant(Val: 0, DL, VT: WideVT),
17140 Cond: IsNot ? ISD::SETEQ : ISD::SETNE);
17141 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT, Operand: Res);
17142}
17143
17144// (and (i1) f, (setcc c, 0, ne)) -> (czero.nez f, c)
17145// (and (i1) f, (setcc c, 0, eq)) -> (czero.eqz f, c)
17146// (and (setcc c, 0, ne), (i1) g) -> (czero.nez g, c)
17147// (and (setcc c, 0, eq), (i1) g) -> (czero.eqz g, c)
17148static SDValue combineANDOfSETCCToCZERO(SDNode *N, SelectionDAG &DAG,
17149 const RISCVSubtarget &Subtarget) {
17150 if (!Subtarget.hasCZEROLike())
17151 return SDValue();
17152
17153 SDValue N0 = N->getOperand(Num: 0);
17154 SDValue N1 = N->getOperand(Num: 1);
17155
17156 auto IsEqualCompZero = [](SDValue &V) -> bool {
17157 if (V.getOpcode() == ISD::SETCC && isNullConstant(V: V.getOperand(i: 1))) {
17158 ISD::CondCode CC = cast<CondCodeSDNode>(Val: V.getOperand(i: 2))->get();
17159 if (ISD::isIntEqualitySetCC(Code: CC))
17160 return true;
17161 }
17162 return false;
17163 };
17164
17165 if (!IsEqualCompZero(N0) || !N0.hasOneUse())
17166 std::swap(a&: N0, b&: N1);
17167 if (!IsEqualCompZero(N0) || !N0.hasOneUse())
17168 return SDValue();
17169
17170 KnownBits Known = DAG.computeKnownBits(Op: N1);
17171 if (Known.getMaxValue().ugt(RHS: 1))
17172 return SDValue();
17173
17174 unsigned CzeroOpcode =
17175 (cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get() == ISD::SETNE)
17176 ? RISCVISD::CZERO_EQZ
17177 : RISCVISD::CZERO_NEZ;
17178
17179 EVT VT = N->getValueType(ResNo: 0);
17180 SDLoc DL(N);
17181 return DAG.getNode(Opcode: CzeroOpcode, DL, VT, N1, N2: N0.getOperand(i: 0));
17182}
17183
17184static SDValue reduceANDOfAtomicLoad(SDNode *N,
17185 TargetLowering::DAGCombinerInfo &DCI) {
17186 SelectionDAG &DAG = DCI.DAG;
17187 if (N->getOpcode() != ISD::AND)
17188 return SDValue();
17189
17190 SDValue N0 = N->getOperand(Num: 0);
17191 if (N0.getOpcode() != ISD::ATOMIC_LOAD)
17192 return SDValue();
17193 if (!N0.hasOneUse())
17194 return SDValue();
17195
17196 AtomicSDNode *ALoad = cast<AtomicSDNode>(Val: N0.getNode());
17197 if (isStrongerThanMonotonic(AO: ALoad->getSuccessOrdering()))
17198 return SDValue();
17199
17200 EVT LoadedVT = ALoad->getMemoryVT();
17201 ConstantSDNode *MaskConst = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17202 if (!MaskConst)
17203 return SDValue();
17204 uint64_t Mask = MaskConst->getZExtValue();
17205 uint64_t ExpectedMask = maskTrailingOnes<uint64_t>(N: LoadedVT.getSizeInBits());
17206 if (Mask != ExpectedMask)
17207 return SDValue();
17208
17209 SDValue ZextLoad = DAG.getAtomicLoad(
17210 ExtType: ISD::ZEXTLOAD, dl: SDLoc(N), MemVT: ALoad->getMemoryVT(), VT: N->getValueType(ResNo: 0),
17211 Chain: ALoad->getChain(), Ptr: ALoad->getBasePtr(), MMO: ALoad->getMemOperand());
17212 DCI.CombineTo(N, Res: ZextLoad);
17213 DAG.ReplaceAllUsesOfValueWith(From: SDValue(N0.getNode(), 1), To: ZextLoad.getValue(R: 1));
17214 DCI.recursivelyDeleteUnusedNodes(N: N0.getNode());
17215 return SDValue(N, 0);
17216}
17217
17218// Sometimes a mask is applied after a shift. If that shift was fed by a
17219// load, there is sometimes the opportunity to narrow the load, which is
17220// hidden by the intermediate shift. Detect that case and commute the
17221// shift/and in order to enable load narrowing.
17222static SDValue combineNarrowableShiftedLoad(SDNode *N, SelectionDAG &DAG) {
17223 EVT VT = N->getValueType(ResNo: 0);
17224 if (!VT.isScalarInteger())
17225 return SDValue();
17226
17227 using namespace SDPatternMatch;
17228 SDValue LoadNode;
17229 APInt MaskVal, ShiftVal;
17230 // (and (shl (load ...), ShiftAmt), Mask)
17231 if (!sd_match(
17232 N, P: m_And(L: m_OneUse(P: m_Shl(L: m_AllOf(preds: m_Opc(Opcode: ISD::LOAD), preds: m_Value(N&: LoadNode)),
17233 R: m_ConstInt(V&: ShiftVal))),
17234 R: m_ConstInt(V&: MaskVal)))) {
17235 return SDValue();
17236 }
17237
17238 uint64_t ShiftAmt = ShiftVal.getZExtValue();
17239
17240 if (ShiftAmt >= VT.getSizeInBits())
17241 return SDValue();
17242
17243 // Calculate the appropriate mask if it were applied before the shift.
17244 APInt InnerMask = MaskVal.lshr(shiftAmt: ShiftAmt);
17245 bool IsNarrowable =
17246 InnerMask == 0xff || InnerMask == 0xffff || InnerMask == 0xffffffff;
17247
17248 if (!IsNarrowable)
17249 return SDValue();
17250
17251 // AND the loaded value and change the shift appropriately, allowing
17252 // the load to be narrowed.
17253 SDLoc DL(N);
17254 SDValue InnerAnd = DAG.getNode(Opcode: ISD::AND, DL, VT, N1: LoadNode,
17255 N2: DAG.getConstant(Val: InnerMask, DL, VT));
17256 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: InnerAnd,
17257 N2: DAG.getShiftAmountConstant(Val: ShiftAmt, VT, DL));
17258}
17259
17260// Combines two comparison operation and logic operation to one selection
17261// operation(min, max) and logic operation. Returns new constructed Node if
17262// conditions for optimization are satisfied.
17263static SDValue performANDCombine(SDNode *N,
17264 TargetLowering::DAGCombinerInfo &DCI,
17265 const RISCVSubtarget &Subtarget) {
17266 SelectionDAG &DAG = DCI.DAG;
17267 SDValue N0 = N->getOperand(Num: 0);
17268
17269 // Pre-promote (i32 (and (srl X, Y), 1)) on RV64 with Zbs without zero
17270 // extending X. This is safe since we only need the LSB after the shift and
17271 // shift amounts larger than 31 would produce poison. If we wait until
17272 // type legalization, we'll create RISCVISD::SRLW and we can't recover it
17273 // to use a BEXT instruction.
17274 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
17275 N->getValueType(ResNo: 0) == MVT::i32 && isOneConstant(V: N->getOperand(Num: 1)) &&
17276 N0.getOpcode() == ISD::SRL && !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) &&
17277 N0.hasOneUse()) {
17278 SDLoc DL(N);
17279 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
17280 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
17281 SDValue Srl = DAG.getNode(Opcode: ISD::SRL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
17282 SDValue And = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i64, N1: Srl,
17283 N2: DAG.getConstant(Val: 1, DL, VT: MVT::i64));
17284 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: And);
17285 }
17286
17287 if (SDValue V = combineNarrowableShiftedLoad(N, DAG))
17288 return V;
17289 if (SDValue V = reverseZExtICmpCombine(N, DAG, Subtarget))
17290 return V;
17291 if (DCI.isAfterLegalizeDAG())
17292 if (SDValue V = combineANDOfSETCCToCZERO(N, DAG, Subtarget))
17293 return V;
17294 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17295 return V;
17296 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17297 return V;
17298 if (SDValue V = reduceANDOfAtomicLoad(N, DCI))
17299 return V;
17300
17301 if (DCI.isAfterLegalizeDAG())
17302 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
17303 return V;
17304
17305 // fold (and (select lhs, rhs, cc, -1, y), x) ->
17306 // (select lhs, rhs, cc, x, (and x, y))
17307 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ true, Subtarget);
17308}
17309
17310// Try to pull an xor with 1 through a select idiom that uses czero_eqz/nez.
17311// FIXME: Generalize to other binary operators with same operand.
17312static SDValue combineOrOfCZERO(SDNode *N, SDValue N0, SDValue N1,
17313 SelectionDAG &DAG) {
17314 assert(N->getOpcode() == ISD::OR && "Unexpected opcode");
17315
17316 if (N0.getOpcode() != RISCVISD::CZERO_EQZ ||
17317 N1.getOpcode() != RISCVISD::CZERO_NEZ ||
17318 !N0.hasOneUse() || !N1.hasOneUse())
17319 return SDValue();
17320
17321 // Should have the same condition.
17322 SDValue Cond = N0.getOperand(i: 1);
17323 if (Cond != N1.getOperand(i: 1))
17324 return SDValue();
17325
17326 SDValue TrueV = N0.getOperand(i: 0);
17327 SDValue FalseV = N1.getOperand(i: 0);
17328
17329 if (TrueV.getOpcode() != ISD::XOR || FalseV.getOpcode() != ISD::XOR ||
17330 TrueV.getOperand(i: 1) != FalseV.getOperand(i: 1) ||
17331 !isOneConstant(V: TrueV.getOperand(i: 1)) ||
17332 !TrueV.hasOneUse() || !FalseV.hasOneUse())
17333 return SDValue();
17334
17335 EVT VT = N->getValueType(ResNo: 0);
17336 SDLoc DL(N);
17337
17338 SDValue NewN0 = DAG.getNode(Opcode: RISCVISD::CZERO_EQZ, DL, VT, N1: TrueV.getOperand(i: 0),
17339 N2: Cond);
17340 SDValue NewN1 =
17341 DAG.getNode(Opcode: RISCVISD::CZERO_NEZ, DL, VT, N1: FalseV.getOperand(i: 0), N2: Cond);
17342 SDValue NewOr =
17343 DAG.getNode(Opcode: ISD::OR, DL, VT, N1: NewN0, N2: NewN1, Flags: SDNodeFlags::Disjoint);
17344 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewOr, N2: TrueV.getOperand(i: 1));
17345}
17346
17347// (xor X, (xor (and X, C2), Y))
17348// ->(qc_insb X, (sra Y, ShAmt), Width, ShAmt)
17349// where C2 is a shifted mask with width = Width and shift = ShAmt
17350// qc_insb might become qc.insb or qc.insbi depending on the operands.
17351static SDValue combineXorToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17352 const RISCVSubtarget &Subtarget) {
17353 if (!Subtarget.hasVendorXqcibm())
17354 return SDValue();
17355
17356 using namespace SDPatternMatch;
17357 SDValue Base, Inserted;
17358 APInt CMask;
17359 if (!sd_match(N, P: m_Xor(L: m_Value(N&: Base),
17360 R: m_OneUse(P: m_Xor(L: m_OneUse(P: m_And(L: m_Deferred(V&: Base),
17361 R: m_ConstInt(V&: CMask))),
17362 R: m_Value(N&: Inserted))))))
17363 return SDValue();
17364
17365 if (N->getValueType(ResNo: 0) != MVT::i32)
17366 return SDValue();
17367 unsigned Width, ShAmt;
17368 if (!CMask.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width))
17369 return SDValue();
17370
17371 // Check if all zero bits in CMask are also zero in Inserted
17372 if (!DAG.MaskedValueIsZero(Op: Inserted, Mask: ~CMask))
17373 return SDValue();
17374
17375 SDLoc DL(N);
17376
17377 // `Inserted` needs to be right shifted before it is put into the
17378 // instruction.
17379 Inserted = DAG.getNode(Opcode: ISD::SRA, DL, VT: MVT::i32, N1: Inserted,
17380 N2: DAG.getShiftAmountConstant(Val: ShAmt, VT: MVT::i32, DL));
17381
17382 SDValue Ops[] = {Base, Inserted, DAG.getConstant(Val: Width, DL, VT: MVT::i32),
17383 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
17384 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
17385}
17386
17387static SDValue combineOrToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17388 const RISCVSubtarget &Subtarget) {
17389 if (!Subtarget.hasVendorXqcibm())
17390 return SDValue();
17391
17392 using namespace SDPatternMatch;
17393
17394 SDValue X;
17395 APInt MaskImm;
17396 if (!sd_match(N, P: m_Or(L: m_OneUse(P: m_Value(N&: X)), R: m_ConstInt(V&: MaskImm))))
17397 return SDValue();
17398
17399 unsigned ShAmt, Width;
17400 if (!MaskImm.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width) || MaskImm.isSignedIntN(N: 12))
17401 return SDValue();
17402
17403 if (N->getValueType(ResNo: 0) != MVT::i32)
17404 return SDValue();
17405
17406 // If Zbs is enabled and it is a single bit set we can use BSETI which
17407 // can be compressed to C_BSETI when Xqcibm in enabled.
17408 if (Width == 1 && Subtarget.hasStdExtZbs())
17409 return SDValue();
17410
17411 // If C1 is a shifted mask (but can't be formed as an ORI),
17412 // use a bitfield insert of -1.
17413 // Transform (or x, C1)
17414 // -> (qc.insbi x, -1, width, shift)
17415 SDLoc DL(N);
17416
17417 SDValue Ops[] = {X, DAG.getSignedConstant(Val: -1, DL, VT: MVT::i32),
17418 DAG.getConstant(Val: Width, DL, VT: MVT::i32),
17419 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
17420 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
17421}
17422
17423// Generate a QC_INSB/QC_INSBI from 'or (and X, MaskImm), OrImm' iff the value
17424// being inserted only sets known zero bits.
17425static SDValue combineOrAndToBitfieldInsert(SDNode *N, SelectionDAG &DAG,
17426 const RISCVSubtarget &Subtarget) {
17427 // Supported only in Xqcibm for now.
17428 if (!Subtarget.hasVendorXqcibm())
17429 return SDValue();
17430
17431 using namespace SDPatternMatch;
17432
17433 SDValue Inserted;
17434 APInt MaskImm, OrImm;
17435 if (!sd_match(
17436 N, P: m_SpecificVT(RefVT: MVT::i32, P: m_Or(L: m_OneUse(P: m_And(L: m_Value(N&: Inserted),
17437 R: m_ConstInt(V&: MaskImm))),
17438 R: m_ConstInt(V&: OrImm)))))
17439 return SDValue();
17440
17441 // Compute the Known Zero for the AND as this allows us to catch more general
17442 // cases than just looking for AND with imm.
17443 KnownBits Known = DAG.computeKnownBits(Op: N->getOperand(Num: 0));
17444
17445 // The bits being inserted must only set those bits that are known to be
17446 // zero.
17447 if (!OrImm.isSubsetOf(RHS: Known.Zero)) {
17448 // FIXME: It's okay if the OrImm sets NotKnownZero bits to 1, but we don't
17449 // currently handle this case.
17450 return SDValue();
17451 }
17452
17453 unsigned ShAmt, Width;
17454 // The KnownZero mask must be a shifted mask (e.g., 1110..011, 11100..00).
17455 if (!Known.Zero.isShiftedMask(MaskIdx&: ShAmt, MaskLen&: Width))
17456 return SDValue();
17457
17458 // QC_INSB(I) dst, src, #width, #shamt.
17459 SDLoc DL(N);
17460
17461 SDValue ImmNode =
17462 DAG.getSignedConstant(Val: OrImm.getSExtValue() >> ShAmt, DL, VT: MVT::i32);
17463
17464 SDValue Ops[] = {Inserted, ImmNode, DAG.getConstant(Val: Width, DL, VT: MVT::i32),
17465 DAG.getConstant(Val: ShAmt, DL, VT: MVT::i32)};
17466 return DAG.getNode(Opcode: RISCVISD::QC_INSB, DL, VT: MVT::i32, Ops);
17467}
17468
17469static SDValue performORCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
17470 const RISCVSubtarget &Subtarget) {
17471 SelectionDAG &DAG = DCI.DAG;
17472
17473 if (SDValue V = combineOrAndToBitfieldInsert(N, DAG, Subtarget))
17474 return V;
17475 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17476 return V;
17477 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17478 return V;
17479
17480 if (DCI.isAfterLegalizeDAG()) {
17481 if (SDValue V = combineOrToBitfieldInsert(N, DAG, Subtarget))
17482 return V;
17483 if (SDValue V = combineDeMorganOfBoolean(N, DAG))
17484 return V;
17485 }
17486
17487 // Look for Or of CZERO_EQZ/NEZ with same condition which is the select idiom.
17488 // We may be able to pull a common operation out of the true and false value.
17489 SDValue N0 = N->getOperand(Num: 0);
17490 SDValue N1 = N->getOperand(Num: 1);
17491 if (SDValue V = combineOrOfCZERO(N, N0, N1, DAG))
17492 return V;
17493 if (SDValue V = combineOrOfCZERO(N, N0: N1, N1: N0, DAG))
17494 return V;
17495
17496 // fold (or (select cond, 0, y), x) ->
17497 // (select cond, x, (or x, y))
17498 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
17499}
17500
17501static SDValue performXORCombine(SDNode *N, SelectionDAG &DAG,
17502 const RISCVSubtarget &Subtarget) {
17503 SDValue N0 = N->getOperand(Num: 0);
17504 SDValue N1 = N->getOperand(Num: 1);
17505
17506 // Pre-promote (i32 (xor (shl -1, X), ~0)) on RV64 with Zbs so we can use
17507 // (ADDI (BSET X0, X), -1). If we wait until type legalization, we'll create
17508 // RISCVISD:::SLLW and we can't recover it to use a BSET instruction.
17509 if (Subtarget.is64Bit() && Subtarget.hasStdExtZbs() &&
17510 N->getValueType(ResNo: 0) == MVT::i32 && isAllOnesConstant(V: N1) &&
17511 N0.getOpcode() == ISD::SHL && isAllOnesConstant(V: N0.getOperand(i: 0)) &&
17512 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) && N0.hasOneUse()) {
17513 SDLoc DL(N);
17514 SDValue Op0 = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 0));
17515 SDValue Op1 = DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT: MVT::i64, Operand: N0.getOperand(i: 1));
17516 SDValue Shl = DAG.getNode(Opcode: ISD::SHL, DL, VT: MVT::i64, N1: Op0, N2: Op1);
17517 SDValue Not = DAG.getNOT(DL, Val: Shl, VT: MVT::i64);
17518 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i32, Operand: Not);
17519 }
17520
17521 // fold (xor (sllw 1, x), -1) -> (rolw ~1, x)
17522 // NOTE: Assumes ROL being legal means ROLW is legal.
17523 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17524 if (N0.getOpcode() == RISCVISD::SLLW &&
17525 isAllOnesConstant(V: N1) && isOneConstant(V: N0.getOperand(i: 0)) &&
17526 TLI.isOperationLegal(Op: ISD::ROTL, VT: MVT::i64)) {
17527 SDLoc DL(N);
17528 return DAG.getNode(Opcode: RISCVISD::ROLW, DL, VT: MVT::i64,
17529 N1: DAG.getConstant(Val: ~1, DL, VT: MVT::i64), N2: N0.getOperand(i: 1));
17530 }
17531
17532 // Fold (xor (setcc constant, y, setlt), 1) -> (setcc y, constant + 1, setlt)
17533 if (N0.getOpcode() == ISD::SETCC && isOneConstant(V: N1) && N0.hasOneUse()) {
17534 auto *ConstN00 = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: 0));
17535 ISD::CondCode CC = cast<CondCodeSDNode>(Val: N0.getOperand(i: 2))->get();
17536 if (ConstN00 && CC == ISD::SETLT) {
17537 EVT VT = N0.getValueType();
17538 SDLoc DL(N0);
17539 const APInt &Imm = ConstN00->getAPIntValue();
17540 if ((Imm + 1).isSignedIntN(N: 12))
17541 return DAG.getSetCC(DL, VT, LHS: N0.getOperand(i: 1),
17542 RHS: DAG.getConstant(Val: Imm + 1, DL, VT), Cond: CC);
17543 }
17544 }
17545
17546 if (SDValue V = combineXorToBitfieldInsert(N, DAG, Subtarget))
17547 return V;
17548
17549 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
17550 return V;
17551 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
17552 return V;
17553
17554 // fold (xor (select cond, 0, y), x) ->
17555 // (select cond, x, (xor x, y))
17556 return combineSelectAndUseCommutative(N, DAG, /*AllOnes*/ false, Subtarget);
17557}
17558
17559// Try to expand a multiply to a sequence of shifts and add/subs,
17560// for a machine without native mul instruction.
17561static SDValue expandMulToNAFSequence(SDNode *N, SelectionDAG &DAG,
17562 uint64_t MulAmt) {
17563 SDLoc DL(N);
17564 EVT VT = N->getValueType(ResNo: 0);
17565 const uint64_t BitWidth = VT.getFixedSizeInBits();
17566
17567 SDValue Result = DAG.getConstant(Val: 0, DL, VT: N->getValueType(ResNo: 0));
17568 SDValue N0 = N->getOperand(Num: 0);
17569
17570 // Find the Non-adjacent form of the multiplier.
17571 for (uint64_t E = MulAmt, I = 0; E && I < BitWidth; ++I, E >>= 1) {
17572 if (E & 1) {
17573 bool IsAdd = (E & 3) == 1;
17574 E -= IsAdd ? 1 : -1;
17575 SDValue ShiftVal = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: N0,
17576 N2: DAG.getShiftAmountConstant(Val: I, VT, DL));
17577 ISD::NodeType AddSubOp = IsAdd ? ISD::ADD : ISD::SUB;
17578 Result = DAG.getNode(Opcode: AddSubOp, DL, VT, N1: Result, N2: ShiftVal);
17579 }
17580 }
17581
17582 return Result;
17583}
17584
17585// X * (2^N +/- 2^M) -> (add/sub (shl X, C1), (shl X, C2))
17586static SDValue expandMulToAddOrSubOfShl(SDNode *N, SelectionDAG &DAG,
17587 uint64_t MulAmt) {
17588 uint64_t MulAmtLowBit = MulAmt & (-MulAmt);
17589 SDValue X = N->getOperand(Num: 0);
17590 ISD::NodeType Op;
17591 uint64_t ShiftAmt1;
17592 bool CanSub = isPowerOf2_64(Value: MulAmt + MulAmtLowBit);
17593 auto PreferSub = [X, MulAmtLowBit]() {
17594 // For MulAmt == 3 << M both (X << M + 2) - (X << M)
17595 // and (X << M + 1) + (X << M) are valid expansions.
17596 // Prefer SUB if we can get (X << M + 2) for free,
17597 // because X is exact (Y >> M + 2).
17598 uint64_t ShAmt = Log2_64(Value: MulAmtLowBit) + 2;
17599 using namespace SDPatternMatch;
17600 return sd_match(N: X, P: m_ExactSr(L: m_Value(), R: m_SpecificInt(V: ShAmt)));
17601 };
17602 if (isPowerOf2_64(Value: MulAmt - MulAmtLowBit) && !(CanSub && PreferSub())) {
17603 Op = ISD::ADD;
17604 ShiftAmt1 = MulAmt - MulAmtLowBit;
17605 } else if (CanSub) {
17606 Op = ISD::SUB;
17607 ShiftAmt1 = MulAmt + MulAmtLowBit;
17608 } else {
17609 return SDValue();
17610 }
17611 EVT VT = N->getValueType(ResNo: 0);
17612 SDLoc DL(N);
17613 SDValue Shift1 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X,
17614 N2: DAG.getConstant(Val: Log2_64(Value: ShiftAmt1), DL, VT));
17615 SDValue Shift2 = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X,
17616 N2: DAG.getConstant(Val: Log2_64(Value: MulAmtLowBit), DL, VT));
17617 return DAG.getNode(Opcode: Op, DL, VT, N1: Shift1, N2: Shift2);
17618}
17619
17620static SDValue getShlAddShlAdd(SDNode *N, SelectionDAG &DAG, unsigned ShX,
17621 unsigned ShY, bool AddX, unsigned Shift) {
17622 SDLoc DL(N);
17623 EVT VT = N->getValueType(ResNo: 0);
17624 SDValue X = N->getOperand(Num: 0);
17625 // Put the shift first if we can fold:
17626 // a. a zext into the shift forming a slli.uw
17627 // b. an exact shift right forming one shorter shift or no shift at all
17628 using namespace SDPatternMatch;
17629 if (Shift != 0 &&
17630 sd_match(N: X, P: m_AnyOf(preds: m_And(L: m_Value(), R: m_SpecificInt(UINT64_C(0xffffffff))),
17631 preds: m_ExactSr(L: m_Value(), R: m_ConstInt())))) {
17632 X = DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: Shift, DL, VT));
17633 Shift = 0;
17634 }
17635 SDValue ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
17636 N2: DAG.getTargetConstant(Val: ShY, DL, VT), N3: X);
17637 if (ShX != 0)
17638 ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: ShlAdd,
17639 N2: DAG.getTargetConstant(Val: ShX, DL, VT), N3: AddX ? X : ShlAdd);
17640 if (Shift == 0)
17641 return ShlAdd;
17642 // Otherwise, put the shl last so that it can fold with following instructions
17643 // (e.g. sext or add).
17644 return DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: ShlAdd, N2: DAG.getConstant(Val: Shift, DL, VT));
17645}
17646
17647static SDValue expandMulToShlAddShlAdd(SDNode *N, SelectionDAG &DAG,
17648 uint64_t MulAmt, unsigned Shift) {
17649 switch (MulAmt) {
17650 // 3/5/9 -> (shYadd X, X)
17651 case 3:
17652 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 1, /*AddX=*/false, Shift);
17653 case 5:
17654 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 2, /*AddX=*/false, Shift);
17655 case 9:
17656 return getShlAddShlAdd(N, DAG, ShX: 0, ShY: 3, /*AddX=*/false, Shift);
17657 // 3/5/9 * 3/5/9 -> (shXadd (shYadd X, X), (shYadd X, X))
17658 case 5 * 3:
17659 return getShlAddShlAdd(N, DAG, ShX: 2, ShY: 1, /*AddX=*/false, Shift);
17660 case 9 * 3:
17661 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 1, /*AddX=*/false, Shift);
17662 case 5 * 5:
17663 return getShlAddShlAdd(N, DAG, ShX: 2, ShY: 2, /*AddX=*/false, Shift);
17664 case 9 * 5:
17665 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 2, /*AddX=*/false, Shift);
17666 case 9 * 9:
17667 return getShlAddShlAdd(N, DAG, ShX: 3, ShY: 3, /*AddX=*/false, Shift);
17668 default:
17669 break;
17670 }
17671
17672 int ShX;
17673 if (int ShY = isShifted359(Value: MulAmt - 1, Shift&: ShX)) {
17674 assert(ShX != 0 && "MulAmt=4,6,10 handled before");
17675 // 2/4/8 * 3/5/9 + 1 -> (shXadd (shYadd X, X), X)
17676 if (ShX <= 3)
17677 return getShlAddShlAdd(N, DAG, ShX, ShY, /*AddX=*/true, Shift);
17678 // 2^N * 3/5/9 + 1 -> (add (shYadd (shl X, N), (shl X, N)), X)
17679 if (Shift == 0) {
17680 SDLoc DL(N);
17681 EVT VT = N->getValueType(ResNo: 0);
17682 SDValue X = N->getOperand(Num: 0);
17683 SDValue Shl =
17684 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShX, DL, VT));
17685 SDValue ShlAdd = DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: Shl,
17686 N2: DAG.getTargetConstant(Val: ShY, DL, VT), N3: Shl);
17687 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: ShlAdd, N2: X);
17688 }
17689 }
17690 return SDValue();
17691}
17692
17693// Try to expand a scalar multiply to a faster sequence.
17694static SDValue expandMul(SDNode *N, SelectionDAG &DAG,
17695 TargetLowering::DAGCombinerInfo &DCI,
17696 const RISCVSubtarget &Subtarget) {
17697
17698 EVT VT = N->getValueType(ResNo: 0);
17699
17700 // LI + MUL is usually smaller than the alternative sequence.
17701 if (DAG.getMachineFunction().getFunction().hasMinSize())
17702 return SDValue();
17703
17704 if (VT != Subtarget.getXLenVT())
17705 return SDValue();
17706
17707 bool ShouldExpandMul =
17708 (!DCI.isBeforeLegalize() && !DCI.isCalledByLegalizer()) ||
17709 !Subtarget.hasStdExtZmmul();
17710 if (!ShouldExpandMul)
17711 return SDValue();
17712
17713 ConstantSDNode *CNode = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
17714 if (!CNode)
17715 return SDValue();
17716 uint64_t MulAmt = CNode->getZExtValue();
17717
17718 // Don't do this if the Xqciac extension is enabled and the MulAmt in simm12.
17719 if (Subtarget.hasVendorXqciac() && isInt<12>(x: CNode->getSExtValue()))
17720 return SDValue();
17721
17722 // WARNING: The code below is knowingly incorrect with regards to undef
17723 // semantics. We're adding additional uses of X here, and in principle, we
17724 // should be freezing X before doing so. However, adding freeze here causes
17725 // real regressions, and no other target properly freezes X in these cases
17726 // either.
17727 if (Subtarget.hasShlAdd(ShAmt: 3)) {
17728 // 3/5/9 * 2^N -> (shl (shXadd X, X), N)
17729 // 3/5/9 * 3/5/9 * 2^N - In particular, this covers multiples
17730 // of 25 which happen to be quite common.
17731 // (2/4/8 * 3/5/9 + 1) * 2^N
17732 unsigned Shift = llvm::countr_zero(Val: MulAmt);
17733 if (SDValue V = expandMulToShlAddShlAdd(N, DAG, MulAmt: MulAmt >> Shift, Shift))
17734 return V;
17735
17736 // If this is a power 2 + 2/4/8, we can use a shift followed by a single
17737 // shXadd. First check if this a sum of two power of 2s because that's
17738 // easy. Then count how many zeros are up to the first bit.
17739 SDValue X = N->getOperand(Num: 0);
17740 if (Shift >= 1 && Shift <= 3 && isPowerOf2_64(Value: MulAmt & (MulAmt - 1))) {
17741 unsigned ShiftAmt = llvm::countr_zero(Val: (MulAmt & (MulAmt - 1)));
17742 SDLoc DL(N);
17743 SDValue Shift1 =
17744 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShiftAmt, DL, VT));
17745 return DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
17746 N2: DAG.getTargetConstant(Val: Shift, DL, VT), N3: Shift1);
17747 }
17748
17749 // TODO: 2^(C1>3) * 3/5/9 - 1
17750
17751 // 2^n + 2/4/8 + 1 -> (add (shl X, C1), (shXadd X, X))
17752 if (MulAmt > 2 && isPowerOf2_64(Value: (MulAmt - 1) & (MulAmt - 2))) {
17753 unsigned ScaleShift = llvm::countr_zero(Val: MulAmt - 1);
17754 if (ScaleShift >= 1 && ScaleShift < 4) {
17755 unsigned ShiftAmt = llvm::countr_zero(Val: (MulAmt - 1) & (MulAmt - 2));
17756 SDLoc DL(N);
17757 SDValue Shift1 =
17758 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShiftAmt, DL, VT));
17759 return DAG.getNode(
17760 Opcode: ISD::ADD, DL, VT, N1: Shift1,
17761 N2: DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
17762 N2: DAG.getTargetConstant(Val: ScaleShift, DL, VT), N3: X));
17763 }
17764 }
17765
17766 // 2^N - 3/5/9 --> (sub (shl X, C1), (shXadd X, x))
17767 for (uint64_t Offset : {3, 5, 9}) {
17768 if (isPowerOf2_64(Value: MulAmt + Offset)) {
17769 unsigned ShAmt = llvm::countr_zero(Val: MulAmt + Offset);
17770 if (ShAmt >= VT.getSizeInBits())
17771 continue;
17772 SDLoc DL(N);
17773 SDValue Shift1 =
17774 DAG.getNode(Opcode: ISD::SHL, DL, VT, N1: X, N2: DAG.getConstant(Val: ShAmt, DL, VT));
17775 SDValue Mul359 =
17776 DAG.getNode(Opcode: RISCVISD::SHL_ADD, DL, VT, N1: X,
17777 N2: DAG.getTargetConstant(Val: Log2_64(Value: Offset - 1), DL, VT), N3: X);
17778 return DAG.getNode(Opcode: ISD::SUB, DL, VT, N1: Shift1, N2: Mul359);
17779 }
17780 }
17781 }
17782
17783 if (SDValue V = expandMulToAddOrSubOfShl(N, DAG, MulAmt))
17784 return V;
17785
17786 if (!Subtarget.hasStdExtZmmul())
17787 return expandMulToNAFSequence(N, DAG, MulAmt);
17788
17789 return SDValue();
17790}
17791
17792// Combine vXi32 (mul (and (lshr X, 15), 0x10001), 0xffff) ->
17793// (bitcast (sra (v2Xi16 (bitcast X)), 15))
17794// Same for other equivalent types with other equivalent constants.
17795static SDValue combineVectorMulToSraBitcast(SDNode *N, SelectionDAG &DAG) {
17796 EVT VT = N->getValueType(ResNo: 0);
17797 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
17798
17799 // Do this for legal vectors unless they are i1 or i8 vectors.
17800 if (!VT.isVector() || !TLI.isTypeLegal(VT) || VT.getScalarSizeInBits() < 16)
17801 return SDValue();
17802
17803 if (N->getOperand(Num: 0).getOpcode() != ISD::AND ||
17804 N->getOperand(Num: 0).getOperand(i: 0).getOpcode() != ISD::SRL)
17805 return SDValue();
17806
17807 SDValue And = N->getOperand(Num: 0);
17808 SDValue Srl = And.getOperand(i: 0);
17809
17810 APInt V1, V2, V3;
17811 if (!ISD::isConstantSplatVector(N: N->getOperand(Num: 1).getNode(), SplatValue&: V1) ||
17812 !ISD::isConstantSplatVector(N: And.getOperand(i: 1).getNode(), SplatValue&: V2) ||
17813 !ISD::isConstantSplatVector(N: Srl.getOperand(i: 1).getNode(), SplatValue&: V3))
17814 return SDValue();
17815
17816 unsigned HalfSize = VT.getScalarSizeInBits() / 2;
17817 if (!V1.isMask(numBits: HalfSize) || V2 != (1ULL | 1ULL << HalfSize) ||
17818 V3 != (HalfSize - 1))
17819 return SDValue();
17820
17821 EVT HalfVT = EVT::getVectorVT(Context&: *DAG.getContext(),
17822 VT: EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: HalfSize),
17823 EC: VT.getVectorElementCount() * 2);
17824 SDLoc DL(N);
17825 SDValue Cast = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: HalfVT, Operand: Srl.getOperand(i: 0));
17826 SDValue Sra = DAG.getNode(Opcode: ISD::SRA, DL, VT: HalfVT, N1: Cast,
17827 N2: DAG.getConstant(Val: HalfSize - 1, DL, VT: HalfVT));
17828 return DAG.getNode(Opcode: ISD::BITCAST, DL, VT, Operand: Sra);
17829}
17830
17831static SDValue performMULCombine(SDNode *N, SelectionDAG &DAG,
17832 TargetLowering::DAGCombinerInfo &DCI,
17833 const RISCVSubtarget &Subtarget) {
17834 EVT VT = N->getValueType(ResNo: 0);
17835 if (!VT.isVector())
17836 return expandMul(N, DAG, DCI, Subtarget);
17837
17838 SDLoc DL(N);
17839 SDValue N0 = N->getOperand(Num: 0);
17840 SDValue N1 = N->getOperand(Num: 1);
17841 SDValue MulOper;
17842 unsigned AddSubOpc;
17843
17844 // vmadd: (mul (add x, 1), y) -> (add (mul x, y), y)
17845 // (mul x, add (y, 1)) -> (add x, (mul x, y))
17846 // vnmsub: (mul (sub 1, x), y) -> (sub y, (mul x, y))
17847 // (mul x, (sub 1, y)) -> (sub x, (mul x, y))
17848 auto IsAddSubWith1 = [&](SDValue V) -> bool {
17849 AddSubOpc = V->getOpcode();
17850 if ((AddSubOpc == ISD::ADD || AddSubOpc == ISD::SUB) && V->hasOneUse()) {
17851 SDValue Opnd = V->getOperand(Num: 1);
17852 MulOper = V->getOperand(Num: 0);
17853 if (AddSubOpc == ISD::SUB)
17854 std::swap(a&: Opnd, b&: MulOper);
17855 if (isOneOrOneSplat(V: Opnd))
17856 return true;
17857 }
17858 return false;
17859 };
17860
17861 if (IsAddSubWith1(N0)) {
17862 SDValue MulVal = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1, N2: MulOper);
17863 return DAG.getNode(Opcode: AddSubOpc, DL, VT, N1, N2: MulVal);
17864 }
17865
17866 if (IsAddSubWith1(N1)) {
17867 SDValue MulVal = DAG.getNode(Opcode: ISD::MUL, DL, VT, N1: N0, N2: MulOper);
17868 return DAG.getNode(Opcode: AddSubOpc, DL, VT, N1: N0, N2: MulVal);
17869 }
17870
17871 if (SDValue V = combineBinOpOfZExt(N, DAG))
17872 return V;
17873
17874 if (SDValue V = combineVectorMulToSraBitcast(N, DAG))
17875 return V;
17876
17877 return SDValue();
17878}
17879
17880/// According to the property that indexed load/store instructions zero-extend
17881/// their indices, try to narrow the type of index operand.
17882static bool narrowIndex(SDValue &N, ISD::MemIndexType IndexType, SelectionDAG &DAG) {
17883 if (isIndexTypeSigned(IndexType))
17884 return false;
17885
17886 if (!N->hasOneUse())
17887 return false;
17888
17889 EVT VT = N.getValueType();
17890 SDLoc DL(N);
17891
17892 // In general, what we're doing here is seeing if we can sink a truncate to
17893 // a smaller element type into the expression tree building our index.
17894 // TODO: We can generalize this and handle a bunch more cases if useful.
17895
17896 // Narrow a buildvector to the narrowest element type. This requires less
17897 // work and less register pressure at high LMUL, and creates smaller constants
17898 // which may be cheaper to materialize.
17899 if (ISD::isBuildVectorOfConstantSDNodes(N: N.getNode())) {
17900 KnownBits Known = DAG.computeKnownBits(Op: N);
17901 unsigned ActiveBits = std::max(a: 8u, b: Known.countMaxActiveBits());
17902 LLVMContext &C = *DAG.getContext();
17903 EVT ResultVT = EVT::getIntegerVT(Context&: C, BitWidth: ActiveBits).getRoundIntegerType(Context&: C);
17904 if (ResultVT.bitsLT(VT: VT.getVectorElementType())) {
17905 N = DAG.getNode(Opcode: ISD::TRUNCATE, DL,
17906 VT: VT.changeVectorElementType(Context&: C, EltVT: ResultVT), Operand: N);
17907 return true;
17908 }
17909 }
17910
17911 // Handle the pattern (shl (zext x to ty), C) and bits(x) + C < bits(ty).
17912 if (N.getOpcode() != ISD::SHL)
17913 return false;
17914
17915 SDValue N0 = N.getOperand(i: 0);
17916 if (N0.getOpcode() != ISD::ZERO_EXTEND &&
17917 N0.getOpcode() != RISCVISD::VZEXT_VL)
17918 return false;
17919 if (!N0->hasOneUse())
17920 return false;
17921
17922 APInt ShAmt;
17923 SDValue N1 = N.getOperand(i: 1);
17924 if (!ISD::isConstantSplatVector(N: N1.getNode(), SplatValue&: ShAmt))
17925 return false;
17926
17927 SDValue Src = N0.getOperand(i: 0);
17928 EVT SrcVT = Src.getValueType();
17929 unsigned SrcElen = SrcVT.getScalarSizeInBits();
17930 unsigned ShAmtV = ShAmt.getZExtValue();
17931 unsigned NewElen = PowerOf2Ceil(A: SrcElen + ShAmtV);
17932 NewElen = std::max(a: NewElen, b: 8U);
17933
17934 // Skip if NewElen is not narrower than the original extended type.
17935 if (NewElen >= N0.getValueType().getScalarSizeInBits())
17936 return false;
17937
17938 EVT NewEltVT = EVT::getIntegerVT(Context&: *DAG.getContext(), BitWidth: NewElen);
17939 EVT NewVT = SrcVT.changeVectorElementType(Context&: *DAG.getContext(), EltVT: NewEltVT);
17940
17941 SDValue NewExt = DAG.getNode(Opcode: N0->getOpcode(), DL, VT: NewVT, Ops: N0->ops());
17942 SDValue NewShAmtVec = DAG.getConstant(Val: ShAmtV, DL, VT: NewVT);
17943 N = DAG.getNode(Opcode: ISD::SHL, DL, VT: NewVT, N1: NewExt, N2: NewShAmtVec);
17944 return true;
17945}
17946
17947/// Try to map an integer comparison with size > XLEN to vector instructions
17948/// before type legalization splits it up into chunks.
17949static SDValue
17950combineVectorSizedSetCCEquality(EVT VT, SDValue X, SDValue Y, ISD::CondCode CC,
17951 const SDLoc &DL, SelectionDAG &DAG,
17952 const RISCVSubtarget &Subtarget) {
17953 assert(ISD::isIntEqualitySetCC(CC) && "Bad comparison predicate");
17954
17955 if (!Subtarget.hasVInstructions())
17956 return SDValue();
17957
17958 MVT XLenVT = Subtarget.getXLenVT();
17959 EVT OpVT = X.getValueType();
17960 // We're looking for an oversized integer equality comparison.
17961 if (!OpVT.isScalarInteger())
17962 return SDValue();
17963
17964 unsigned OpSize = OpVT.getSizeInBits();
17965 // The size should be larger than XLen and smaller than the maximum vector
17966 // size.
17967 if (OpSize <= Subtarget.getXLen() ||
17968 OpSize > Subtarget.getRealMinVLen() *
17969 Subtarget.getMaxLMULForFixedLengthVectors())
17970 return SDValue();
17971
17972 // Don't perform this combine if constructing the vector will be expensive.
17973 auto IsVectorBitCastCheap = [](SDValue X) {
17974 X = peekThroughBitcasts(V: X);
17975 return isa<ConstantSDNode>(Val: X) || X.getValueType().isVector() ||
17976 X.getOpcode() == ISD::LOAD;
17977 };
17978 if (!IsVectorBitCastCheap(X) || !IsVectorBitCastCheap(Y))
17979 return SDValue();
17980
17981 if (DAG.getMachineFunction().getFunction().hasFnAttribute(
17982 Kind: Attribute::NoImplicitFloat))
17983 return SDValue();
17984
17985 // Bail out for non-byte-sized types.
17986 if (!OpVT.isByteSized())
17987 return SDValue();
17988
17989 unsigned VecSize = OpSize / 8;
17990 EVT VecVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i8, NumElements: VecSize);
17991 EVT CmpVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: VecSize);
17992
17993 SDValue VecX = DAG.getBitcast(VT: VecVT, V: X);
17994 SDValue VecY = DAG.getBitcast(VT: VecVT, V: Y);
17995 SDValue Mask = DAG.getAllOnesConstant(DL, VT: CmpVT);
17996 SDValue VL = DAG.getConstant(Val: VecSize, DL, VT: XLenVT);
17997
17998 SDValue Cmp = DAG.getSetCC(DL, VT: CmpVT, LHS: VecX, RHS: VecY, Cond: ISD::SETNE);
17999 return DAG.getSetCC(DL, VT,
18000 LHS: DAG.getNode(Opcode: ISD::VP_REDUCE_OR, DL, VT: XLenVT,
18001 N1: DAG.getConstant(Val: 0, DL, VT: XLenVT), N2: Cmp, N3: Mask,
18002 N4: VL),
18003 RHS: DAG.getConstant(Val: 0, DL, VT: XLenVT), Cond: CC);
18004}
18005
18006static SDValue performSETCCCombine(SDNode *N,
18007 TargetLowering::DAGCombinerInfo &DCI,
18008 const RISCVSubtarget &Subtarget) {
18009 SelectionDAG &DAG = DCI.DAG;
18010 SDLoc dl(N);
18011 SDValue N0 = N->getOperand(Num: 0);
18012 SDValue N1 = N->getOperand(Num: 1);
18013 EVT VT = N->getValueType(ResNo: 0);
18014 EVT OpVT = N0.getValueType();
18015
18016 ISD::CondCode Cond = cast<CondCodeSDNode>(Val: N->getOperand(Num: 2))->get();
18017 // Looking for an equality compare.
18018 if (!isIntEqualitySetCC(Code: Cond))
18019 return SDValue();
18020
18021 if (SDValue V =
18022 combineVectorSizedSetCCEquality(VT, X: N0, Y: N1, CC: Cond, DL: dl, DAG, Subtarget))
18023 return V;
18024
18025 if (DCI.isAfterLegalizeDAG() && isa<ConstantSDNode>(Val: N1) &&
18026 N0.getOpcode() == ISD::AND && N0.hasOneUse() &&
18027 isa<ConstantSDNode>(Val: N0.getOperand(i: 1))) {
18028 const APInt &AndRHSC = N0.getConstantOperandAPInt(i: 1);
18029 // (X & -(1 << C)) == 0 -> (X >> C) == 0 if the AND constant can't use ANDI.
18030 if (isNullConstant(V: N1) && !isInt<12>(x: AndRHSC.getSExtValue()) &&
18031 AndRHSC.isNegatedPowerOf2()) {
18032 unsigned ShiftBits = AndRHSC.countr_zero();
18033 SDValue Shift = DAG.getNode(Opcode: ISD::SRL, DL: dl, VT: OpVT, N1: N0.getOperand(i: 0),
18034 N2: DAG.getConstant(Val: ShiftBits, DL: dl, VT: OpVT));
18035 return DAG.getSetCC(DL: dl, VT, LHS: Shift, RHS: N1, Cond);
18036 }
18037
18038 // Similar to above but handling the lower 32 bits by using sraiw. Allow
18039 // comparing with constants other than 0 if the constant can be folded into
18040 // addi or xori after shifting.
18041 uint64_t N1Int = cast<ConstantSDNode>(Val&: N1)->getZExtValue();
18042 uint64_t AndRHSInt = AndRHSC.getZExtValue();
18043 if (OpVT == MVT::i64 && isUInt<32>(x: AndRHSInt) &&
18044 isPowerOf2_32(Value: -uint32_t(AndRHSInt)) && (N1Int & AndRHSInt) == N1Int) {
18045 unsigned ShiftBits = llvm::countr_zero(Val: AndRHSInt);
18046 int64_t NewC = SignExtend64<32>(x: N1Int) >> ShiftBits;
18047 if (NewC >= -2048 && NewC <= 2048) {
18048 SDValue SExt =
18049 DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: dl, VT: OpVT, N1: N0.getOperand(i: 0),
18050 N2: DAG.getValueType(MVT::i32));
18051 SDValue Shift = DAG.getNode(Opcode: ISD::SRA, DL: dl, VT: OpVT, N1: SExt,
18052 N2: DAG.getConstant(Val: ShiftBits, DL: dl, VT: OpVT));
18053 return DAG.getSetCC(DL: dl, VT, LHS: Shift,
18054 RHS: DAG.getSignedConstant(Val: NewC, DL: dl, VT: OpVT), Cond);
18055 }
18056 }
18057
18058 // Fold (and X, Mask) ==/!= C -> X ==/!= sext(C, countr_one(Mask)) if the
18059 // Mask is only clearing redundant sign bits.
18060 if (isMask_64(Value: AndRHSInt)) {
18061 unsigned TrailingOnes = llvm::countr_one(Value: AndRHSInt);
18062 unsigned N1Width = llvm::bit_width(Value: N1Int);
18063 int64_t N1SExt = SignExtend64(X: N1Int, B: TrailingOnes);
18064 if (N1Width <= TrailingOnes && isInt<12>(x: N1SExt) &&
18065 DAG.ComputeMaxSignificantBits(Op: N0.getOperand(i: 0)) <= TrailingOnes)
18066 return DAG.getSetCC(DL: dl, VT, LHS: N0.getOperand(i: 0),
18067 RHS: DAG.getSignedConstant(Val: N1SExt, DL: dl, VT: OpVT), Cond);
18068 }
18069 }
18070
18071 // Replace (seteq (i64 (and X, 0xffffffff)), C1) with
18072 // (seteq (i64 (sext_inreg (X, i32)), C1')) where C1' is C1 sign extended from
18073 // bit 31. Same for setne. C1' may be cheaper to materialize and the
18074 // sext_inreg can become a sext.w instead of a shift pair.
18075 if (OpVT != MVT::i64 || !Subtarget.is64Bit())
18076 return SDValue();
18077
18078 // RHS needs to be a constant.
18079 auto *N1C = dyn_cast<ConstantSDNode>(Val&: N1);
18080 if (!N1C)
18081 return SDValue();
18082
18083 // LHS needs to be (and X, 0xffffffff).
18084 if (N0.getOpcode() != ISD::AND || !N0.hasOneUse() ||
18085 !isa<ConstantSDNode>(Val: N0.getOperand(i: 1)) ||
18086 N0.getConstantOperandVal(i: 1) != UINT64_C(0xffffffff))
18087 return SDValue();
18088
18089 // Don't do this if the sign bit is provably zero, it will be turned back into
18090 // an AND.
18091 APInt SignMask = APInt::getOneBitSet(numBits: 64, BitNo: 31);
18092 if (DAG.MaskedValueIsZero(Op: N0.getOperand(i: 0), Mask: SignMask))
18093 return SDValue();
18094
18095 const APInt &C1 = N1C->getAPIntValue();
18096
18097 // If the constant is larger than 2^32 - 1 it is impossible for both sides
18098 // to be equal.
18099 if (C1.getActiveBits() > 32)
18100 return DAG.getBoolConstant(V: Cond == ISD::SETNE, DL: dl, VT, OpVT);
18101
18102 SDValue SExtOp = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL: N, VT: OpVT,
18103 N1: N0.getOperand(i: 0), N2: DAG.getValueType(MVT::i32));
18104 return DAG.getSetCC(DL: dl, VT, LHS: SExtOp, RHS: DAG.getConstant(Val: C1.trunc(width: 32).sext(width: 64),
18105 DL: dl, VT: OpVT), Cond);
18106}
18107
18108static SDValue
18109performSIGN_EXTEND_INREGCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
18110 const RISCVSubtarget &Subtarget) {
18111 SelectionDAG &DAG = DCI.DAG;
18112 SDValue Src = N->getOperand(Num: 0);
18113 EVT VT = N->getValueType(ResNo: 0);
18114 EVT SrcVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
18115 unsigned Opc = Src.getOpcode();
18116 SDLoc DL(N);
18117
18118 // Fold (sext_inreg (fmv_x_anyexth X), i16) -> (fmv_x_signexth X)
18119 // Don't do this with Zhinx. We need to explicitly sign extend the GPR.
18120 if (Opc == RISCVISD::FMV_X_ANYEXTH && SrcVT.bitsGE(VT: MVT::i16) &&
18121 Subtarget.hasStdExtZfhmin())
18122 return DAG.getNode(Opcode: RISCVISD::FMV_X_SIGNEXTH, DL, VT, Operand: Src.getOperand(i: 0));
18123
18124 // Fold (sext_inreg (shl X, Y), i32) -> (sllw X, Y) iff Y u< 32
18125 if (Opc == ISD::SHL && Subtarget.is64Bit() && SrcVT == MVT::i32 &&
18126 VT == MVT::i64 && !isa<ConstantSDNode>(Val: Src.getOperand(i: 1)) &&
18127 DAG.computeKnownBits(Op: Src.getOperand(i: 1)).countMaxActiveBits() <= 5)
18128 return DAG.getNode(Opcode: RISCVISD::SLLW, DL, VT, N1: Src.getOperand(i: 0),
18129 N2: Src.getOperand(i: 1));
18130
18131 // Fold (sext_inreg (setcc), i1) -> (sub 0, (setcc))
18132 if (Opc == ISD::SETCC && SrcVT == MVT::i1 && DCI.isAfterLegalizeDAG())
18133 return DAG.getNegative(Val: Src, DL, VT);
18134
18135 // Fold (sext_inreg (xor (setcc), -1), i1) -> (add (setcc), -1)
18136 if (Opc == ISD::XOR && SrcVT == MVT::i1 &&
18137 isAllOnesConstant(V: Src.getOperand(i: 1)) &&
18138 Src.getOperand(i: 0).getOpcode() == ISD::SETCC && DCI.isAfterLegalizeDAG())
18139 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: Src.getOperand(i: 0),
18140 N2: DAG.getAllOnesConstant(DL, VT));
18141
18142 return SDValue();
18143}
18144
18145namespace {
18146// Forward declaration of the structure holding the necessary information to
18147// apply a combine.
18148struct CombineResult;
18149
18150enum ExtKind : uint8_t {
18151 ZExt = 1 << 0,
18152 SExt = 1 << 1,
18153 FPExt = 1 << 2,
18154 BF16Ext = 1 << 3
18155};
18156/// Helper class for folding sign/zero extensions.
18157/// In particular, this class is used for the following combines:
18158/// add | add_vl | or disjoint -> vwadd(u) | vwadd(u)_w
18159/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
18160/// mul | mul_vl -> vwmul(u) | vwmul_su
18161/// shl | shl_vl -> vwsll
18162/// fadd -> vfwadd | vfwadd_w
18163/// fsub -> vfwsub | vfwsub_w
18164/// fmul -> vfwmul
18165/// An object of this class represents an operand of the operation we want to
18166/// combine.
18167/// E.g., when trying to combine `mul_vl a, b`, we will have one instance of
18168/// NodeExtensionHelper for `a` and one for `b`.
18169///
18170/// This class abstracts away how the extension is materialized and
18171/// how its number of users affect the combines.
18172///
18173/// In particular:
18174/// - VWADD_W is conceptually == add(op0, sext(op1))
18175/// - VWADDU_W == add(op0, zext(op1))
18176/// - VWSUB_W == sub(op0, sext(op1))
18177/// - VWSUBU_W == sub(op0, zext(op1))
18178/// - VFWADD_W == fadd(op0, fpext(op1))
18179/// - VFWSUB_W == fsub(op0, fpext(op1))
18180/// And VMV_V_X_VL, depending on the value, is conceptually equivalent to
18181/// zext|sext(smaller_value).
18182struct NodeExtensionHelper {
18183 /// Records if this operand is like being zero extended.
18184 bool SupportsZExt;
18185 /// Records if this operand is like being sign extended.
18186 /// Note: SupportsZExt and SupportsSExt are not mutually exclusive. For
18187 /// instance, a splat constant (e.g., 3), would support being both sign and
18188 /// zero extended.
18189 bool SupportsSExt;
18190 /// Records if this operand is like being floating point extended.
18191 bool SupportsFPExt;
18192 /// Records if this operand is extended from bf16.
18193 bool SupportsBF16Ext;
18194 /// This boolean captures whether we care if this operand would still be
18195 /// around after the folding happens.
18196 bool EnforceOneUse;
18197 /// Original value that this NodeExtensionHelper represents.
18198 SDValue OrigOperand;
18199
18200 /// Get the value feeding the extension or the value itself.
18201 /// E.g., for zext(a), this would return a.
18202 SDValue getSource() const {
18203 switch (OrigOperand.getOpcode()) {
18204 case ISD::ZERO_EXTEND:
18205 case ISD::SIGN_EXTEND:
18206 case RISCVISD::VSEXT_VL:
18207 case RISCVISD::VZEXT_VL:
18208 case RISCVISD::FP_EXTEND_VL:
18209 return OrigOperand.getOperand(i: 0);
18210 default:
18211 return OrigOperand;
18212 }
18213 }
18214
18215 /// Check if this instance represents a splat.
18216 bool isSplat() const {
18217 return OrigOperand.getOpcode() == RISCVISD::VMV_V_X_VL ||
18218 OrigOperand.getOpcode() == ISD::SPLAT_VECTOR;
18219 }
18220
18221 /// Get the extended opcode.
18222 unsigned getExtOpc(ExtKind SupportsExt) const {
18223 switch (SupportsExt) {
18224 case ExtKind::SExt:
18225 return RISCVISD::VSEXT_VL;
18226 case ExtKind::ZExt:
18227 return RISCVISD::VZEXT_VL;
18228 case ExtKind::FPExt:
18229 case ExtKind::BF16Ext:
18230 return RISCVISD::FP_EXTEND_VL;
18231 }
18232 llvm_unreachable("Unknown ExtKind enum");
18233 }
18234
18235 /// Get or create a value that can feed \p Root with the given extension \p
18236 /// SupportsExt. If \p SExt is std::nullopt, this returns the source of this
18237 /// operand. \see ::getSource().
18238 SDValue getOrCreateExtendedOp(SDNode *Root, SelectionDAG &DAG,
18239 const RISCVSubtarget &Subtarget,
18240 std::optional<ExtKind> SupportsExt) const {
18241 if (!SupportsExt.has_value())
18242 return OrigOperand;
18243
18244 MVT NarrowVT = getNarrowType(Root, SupportsExt: *SupportsExt);
18245
18246 SDValue Source = getSource();
18247 assert(Subtarget.getTargetLowering()->isTypeLegal(Source.getValueType()));
18248 if (Source.getValueType() == NarrowVT)
18249 return Source;
18250
18251 unsigned ExtOpc = getExtOpc(SupportsExt: *SupportsExt);
18252
18253 // If we need an extension, we should be changing the type.
18254 SDLoc DL(OrigOperand);
18255 auto [Mask, VL] = getMaskAndVL(Root, DAG, Subtarget);
18256 switch (OrigOperand.getOpcode()) {
18257 case ISD::ZERO_EXTEND:
18258 case ISD::SIGN_EXTEND:
18259 case RISCVISD::VSEXT_VL:
18260 case RISCVISD::VZEXT_VL:
18261 case RISCVISD::FP_EXTEND_VL:
18262 return DAG.getNode(Opcode: ExtOpc, DL, VT: NarrowVT, N1: Source, N2: Mask, N3: VL);
18263 case ISD::SPLAT_VECTOR:
18264 return DAG.getSplat(VT: NarrowVT, DL, Op: Source.getOperand(i: 0));
18265 case RISCVISD::VMV_V_X_VL:
18266 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT: NarrowVT,
18267 N1: DAG.getUNDEF(VT: NarrowVT), N2: Source.getOperand(i: 1), N3: VL);
18268 case RISCVISD::VFMV_V_F_VL:
18269 Source = Source.getOperand(i: 1);
18270 assert(Source.getOpcode() == ISD::FP_EXTEND && "Unexpected source");
18271 Source = Source.getOperand(i: 0);
18272 assert(Source.getValueType() == NarrowVT.getVectorElementType());
18273 return DAG.getNode(Opcode: RISCVISD::VFMV_V_F_VL, DL, VT: NarrowVT,
18274 N1: DAG.getUNDEF(VT: NarrowVT), N2: Source, N3: VL);
18275 default:
18276 // Other opcodes can only come from the original LHS of VW(ADD|SUB)_W_VL
18277 // and that operand should already have the right NarrowVT so no
18278 // extension should be required at this point.
18279 llvm_unreachable("Unsupported opcode");
18280 }
18281 }
18282
18283 /// Helper function to get the narrow type for \p Root.
18284 /// The narrow type is the type of \p Root where we divided the size of each
18285 /// element by 2. E.g., if Root's type <2xi16> -> narrow type <2xi8>.
18286 /// \pre Both the narrow type and the original type should be legal.
18287 static MVT getNarrowType(const SDNode *Root, ExtKind SupportsExt) {
18288 MVT VT = Root->getSimpleValueType(ResNo: 0);
18289
18290 // Determine the narrow size.
18291 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
18292
18293 MVT EltVT = SupportsExt == ExtKind::BF16Ext ? MVT::bf16
18294 : SupportsExt == ExtKind::FPExt
18295 ? MVT::getFloatingPointVT(BitWidth: NarrowSize)
18296 : MVT::getIntegerVT(BitWidth: NarrowSize);
18297
18298 assert((int)NarrowSize >= (SupportsExt == ExtKind::FPExt ? 16 : 8) &&
18299 "Trying to extend something we can't represent");
18300 MVT NarrowVT = MVT::getVectorVT(VT: EltVT, EC: VT.getVectorElementCount());
18301 return NarrowVT;
18302 }
18303
18304 /// Get the opcode to materialize:
18305 /// Opcode(sext(a), sext(b)) -> newOpcode(a, b)
18306 static unsigned getSExtOpcode(unsigned Opcode) {
18307 switch (Opcode) {
18308 case ISD::ADD:
18309 case RISCVISD::ADD_VL:
18310 case RISCVISD::VWADD_W_VL:
18311 case RISCVISD::VWADDU_W_VL:
18312 case ISD::OR:
18313 case RISCVISD::OR_VL:
18314 return RISCVISD::VWADD_VL;
18315 case ISD::SUB:
18316 case RISCVISD::SUB_VL:
18317 case RISCVISD::VWSUB_W_VL:
18318 case RISCVISD::VWSUBU_W_VL:
18319 return RISCVISD::VWSUB_VL;
18320 case ISD::MUL:
18321 case RISCVISD::MUL_VL:
18322 return RISCVISD::VWMUL_VL;
18323 default:
18324 llvm_unreachable("Unexpected opcode");
18325 }
18326 }
18327
18328 /// Get the opcode to materialize:
18329 /// Opcode(zext(a), zext(b)) -> newOpcode(a, b)
18330 static unsigned getZExtOpcode(unsigned Opcode) {
18331 switch (Opcode) {
18332 case ISD::ADD:
18333 case RISCVISD::ADD_VL:
18334 case RISCVISD::VWADD_W_VL:
18335 case RISCVISD::VWADDU_W_VL:
18336 case ISD::OR:
18337 case RISCVISD::OR_VL:
18338 return RISCVISD::VWADDU_VL;
18339 case ISD::SUB:
18340 case RISCVISD::SUB_VL:
18341 case RISCVISD::VWSUB_W_VL:
18342 case RISCVISD::VWSUBU_W_VL:
18343 return RISCVISD::VWSUBU_VL;
18344 case ISD::MUL:
18345 case RISCVISD::MUL_VL:
18346 return RISCVISD::VWMULU_VL;
18347 case ISD::SHL:
18348 case RISCVISD::SHL_VL:
18349 return RISCVISD::VWSLL_VL;
18350 default:
18351 llvm_unreachable("Unexpected opcode");
18352 }
18353 }
18354
18355 /// Get the opcode to materialize:
18356 /// Opcode(fpext(a), fpext(b)) -> newOpcode(a, b)
18357 static unsigned getFPExtOpcode(unsigned Opcode) {
18358 switch (Opcode) {
18359 case RISCVISD::FADD_VL:
18360 case RISCVISD::VFWADD_W_VL:
18361 return RISCVISD::VFWADD_VL;
18362 case RISCVISD::FSUB_VL:
18363 case RISCVISD::VFWSUB_W_VL:
18364 return RISCVISD::VFWSUB_VL;
18365 case RISCVISD::FMUL_VL:
18366 return RISCVISD::VFWMUL_VL;
18367 case RISCVISD::VFMADD_VL:
18368 return RISCVISD::VFWMADD_VL;
18369 case RISCVISD::VFMSUB_VL:
18370 return RISCVISD::VFWMSUB_VL;
18371 case RISCVISD::VFNMADD_VL:
18372 return RISCVISD::VFWNMADD_VL;
18373 case RISCVISD::VFNMSUB_VL:
18374 return RISCVISD::VFWNMSUB_VL;
18375 default:
18376 llvm_unreachable("Unexpected opcode");
18377 }
18378 }
18379
18380 /// Get the opcode to materialize \p Opcode(sext(a), zext(b)) ->
18381 /// newOpcode(a, b).
18382 static unsigned getSUOpcode(unsigned Opcode) {
18383 assert((Opcode == RISCVISD::MUL_VL || Opcode == ISD::MUL) &&
18384 "SU is only supported for MUL");
18385 return RISCVISD::VWMULSU_VL;
18386 }
18387
18388 /// Get the opcode to materialize
18389 /// \p Opcode(a, s|z|fpext(b)) -> newOpcode(a, b).
18390 static unsigned getWOpcode(unsigned Opcode, ExtKind SupportsExt) {
18391 switch (Opcode) {
18392 case ISD::ADD:
18393 case RISCVISD::ADD_VL:
18394 case ISD::OR:
18395 case RISCVISD::OR_VL:
18396 return SupportsExt == ExtKind::SExt ? RISCVISD::VWADD_W_VL
18397 : RISCVISD::VWADDU_W_VL;
18398 case ISD::SUB:
18399 case RISCVISD::SUB_VL:
18400 return SupportsExt == ExtKind::SExt ? RISCVISD::VWSUB_W_VL
18401 : RISCVISD::VWSUBU_W_VL;
18402 case RISCVISD::FADD_VL:
18403 return RISCVISD::VFWADD_W_VL;
18404 case RISCVISD::FSUB_VL:
18405 return RISCVISD::VFWSUB_W_VL;
18406 default:
18407 llvm_unreachable("Unexpected opcode");
18408 }
18409 }
18410
18411 using CombineToTry = std::function<std::optional<CombineResult>(
18412 SDNode * /*Root*/, const NodeExtensionHelper & /*LHS*/,
18413 const NodeExtensionHelper & /*RHS*/, SelectionDAG &,
18414 const RISCVSubtarget &)>;
18415
18416 /// Check if this node needs to be fully folded or extended for all users.
18417 bool needToPromoteOtherUsers() const { return EnforceOneUse; }
18418
18419 void fillUpExtensionSupportForSplat(SDNode *Root, SelectionDAG &DAG,
18420 const RISCVSubtarget &Subtarget) {
18421 unsigned Opc = OrigOperand.getOpcode();
18422 MVT VT = OrigOperand.getSimpleValueType();
18423
18424 assert((Opc == ISD::SPLAT_VECTOR || Opc == RISCVISD::VMV_V_X_VL) &&
18425 "Unexpected Opcode");
18426
18427 // The pasthru must be undef for tail agnostic.
18428 if (Opc == RISCVISD::VMV_V_X_VL && !OrigOperand.getOperand(i: 0).isUndef())
18429 return;
18430
18431 // Get the scalar value.
18432 SDValue Op = Opc == ISD::SPLAT_VECTOR ? OrigOperand.getOperand(i: 0)
18433 : OrigOperand.getOperand(i: 1);
18434
18435 // See if we have enough sign bits or zero bits in the scalar to use a
18436 // widening opcode by splatting to smaller element size.
18437 unsigned EltBits = VT.getScalarSizeInBits();
18438 unsigned ScalarBits = Op.getValueSizeInBits();
18439 // If we're not getting all bits from the element, we need special handling.
18440 if (ScalarBits < EltBits) {
18441 // This should only occur on RV32.
18442 assert(Opc == RISCVISD::VMV_V_X_VL && EltBits == 64 && ScalarBits == 32 &&
18443 !Subtarget.is64Bit() && "Unexpected splat");
18444 // vmv.v.x sign extends narrow inputs.
18445 SupportsSExt = true;
18446
18447 // If the input is positive, then sign extend is also zero extend.
18448 if (DAG.SignBitIsZero(Op))
18449 SupportsZExt = true;
18450
18451 EnforceOneUse = false;
18452 return;
18453 }
18454
18455 unsigned NarrowSize = EltBits / 2;
18456 // If the narrow type cannot be expressed with a legal VMV,
18457 // this is not a valid candidate.
18458 if (NarrowSize < 8)
18459 return;
18460
18461 if (DAG.ComputeMaxSignificantBits(Op) <= NarrowSize)
18462 SupportsSExt = true;
18463
18464 if (DAG.MaskedValueIsZero(Op,
18465 Mask: APInt::getBitsSetFrom(numBits: ScalarBits, loBit: NarrowSize)))
18466 SupportsZExt = true;
18467
18468 EnforceOneUse = false;
18469 }
18470
18471 bool isSupportedFPExtend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
18472 return (NarrowEltVT == MVT::f32 ||
18473 (NarrowEltVT == MVT::f16 && Subtarget.hasVInstructionsF16()));
18474 }
18475
18476 bool isSupportedBF16Extend(MVT NarrowEltVT, const RISCVSubtarget &Subtarget) {
18477 return NarrowEltVT == MVT::bf16 &&
18478 (Subtarget.hasStdExtZvfbfwma() || Subtarget.hasVInstructionsBF16());
18479 }
18480
18481 /// Helper method to set the various fields of this struct based on the
18482 /// type of \p Root.
18483 void fillUpExtensionSupport(SDNode *Root, SelectionDAG &DAG,
18484 const RISCVSubtarget &Subtarget) {
18485 SupportsZExt = false;
18486 SupportsSExt = false;
18487 SupportsFPExt = false;
18488 SupportsBF16Ext = false;
18489 EnforceOneUse = true;
18490 unsigned Opc = OrigOperand.getOpcode();
18491 // For the nodes we handle below, we end up using their inputs directly: see
18492 // getSource(). However since they either don't have a passthru or we check
18493 // that their passthru is undef, we can safely ignore their mask and VL.
18494 switch (Opc) {
18495 case ISD::ZERO_EXTEND:
18496 case ISD::SIGN_EXTEND: {
18497 MVT VT = OrigOperand.getSimpleValueType();
18498 if (!VT.isVector())
18499 break;
18500
18501 SDValue NarrowElt = OrigOperand.getOperand(i: 0);
18502 MVT NarrowVT = NarrowElt.getSimpleValueType();
18503 // i1 types are legal but we can't select V{S,Z}EXT_VLs with them.
18504 if (NarrowVT.getVectorElementType() == MVT::i1)
18505 break;
18506
18507 SupportsZExt = Opc == ISD::ZERO_EXTEND;
18508 SupportsSExt = Opc == ISD::SIGN_EXTEND;
18509 break;
18510 }
18511 case RISCVISD::VZEXT_VL:
18512 SupportsZExt = true;
18513 break;
18514 case RISCVISD::VSEXT_VL:
18515 SupportsSExt = true;
18516 break;
18517 case RISCVISD::FP_EXTEND_VL: {
18518 MVT NarrowEltVT =
18519 OrigOperand.getOperand(i: 0).getSimpleValueType().getVectorElementType();
18520 if (isSupportedFPExtend(NarrowEltVT, Subtarget))
18521 SupportsFPExt = true;
18522 if (isSupportedBF16Extend(NarrowEltVT, Subtarget))
18523 SupportsBF16Ext = true;
18524
18525 break;
18526 }
18527 case ISD::SPLAT_VECTOR:
18528 case RISCVISD::VMV_V_X_VL:
18529 fillUpExtensionSupportForSplat(Root, DAG, Subtarget);
18530 break;
18531 case RISCVISD::VFMV_V_F_VL: {
18532 MVT VT = OrigOperand.getSimpleValueType();
18533
18534 if (!OrigOperand.getOperand(i: 0).isUndef())
18535 break;
18536
18537 SDValue Op = OrigOperand.getOperand(i: 1);
18538 if (Op.getOpcode() != ISD::FP_EXTEND)
18539 break;
18540
18541 unsigned NarrowSize = VT.getScalarSizeInBits() / 2;
18542 unsigned ScalarBits = Op.getOperand(i: 0).getValueSizeInBits();
18543 if (NarrowSize != ScalarBits)
18544 break;
18545
18546 if (isSupportedFPExtend(NarrowEltVT: Op.getOperand(i: 0).getSimpleValueType(), Subtarget))
18547 SupportsFPExt = true;
18548 if (isSupportedBF16Extend(NarrowEltVT: Op.getOperand(i: 0).getSimpleValueType(),
18549 Subtarget))
18550 SupportsBF16Ext = true;
18551 break;
18552 }
18553 default:
18554 break;
18555 }
18556 }
18557
18558 /// Check if \p Root supports any extension folding combines.
18559 static bool isSupportedRoot(const SDNode *Root,
18560 const RISCVSubtarget &Subtarget) {
18561 switch (Root->getOpcode()) {
18562 case ISD::ADD:
18563 case ISD::SUB:
18564 case ISD::MUL: {
18565 return Root->getValueType(ResNo: 0).isScalableVector();
18566 }
18567 case ISD::OR: {
18568 return Root->getValueType(ResNo: 0).isScalableVector() &&
18569 Root->getFlags().hasDisjoint();
18570 }
18571 // Vector Widening Integer Add/Sub/Mul Instructions
18572 case RISCVISD::ADD_VL:
18573 case RISCVISD::MUL_VL:
18574 case RISCVISD::VWADD_W_VL:
18575 case RISCVISD::VWADDU_W_VL:
18576 case RISCVISD::SUB_VL:
18577 case RISCVISD::VWSUB_W_VL:
18578 case RISCVISD::VWSUBU_W_VL:
18579 // Vector Widening Floating-Point Add/Sub/Mul Instructions
18580 case RISCVISD::FADD_VL:
18581 case RISCVISD::FSUB_VL:
18582 case RISCVISD::FMUL_VL:
18583 case RISCVISD::VFWADD_W_VL:
18584 case RISCVISD::VFWSUB_W_VL:
18585 return true;
18586 case RISCVISD::OR_VL:
18587 return Root->getFlags().hasDisjoint();
18588 case ISD::SHL:
18589 return Root->getValueType(ResNo: 0).isScalableVector() &&
18590 Subtarget.hasStdExtZvbb();
18591 case RISCVISD::SHL_VL:
18592 return Subtarget.hasStdExtZvbb();
18593 case RISCVISD::VFMADD_VL:
18594 case RISCVISD::VFNMSUB_VL:
18595 case RISCVISD::VFNMADD_VL:
18596 case RISCVISD::VFMSUB_VL:
18597 return true;
18598 default:
18599 return false;
18600 }
18601 }
18602
18603 /// Build a NodeExtensionHelper for \p Root.getOperand(\p OperandIdx).
18604 NodeExtensionHelper(SDNode *Root, unsigned OperandIdx, SelectionDAG &DAG,
18605 const RISCVSubtarget &Subtarget) {
18606 assert(isSupportedRoot(Root, Subtarget) &&
18607 "Trying to build an helper with an "
18608 "unsupported root");
18609 assert(OperandIdx < 2 && "Requesting something else than LHS or RHS");
18610 assert(DAG.getTargetLoweringInfo().isTypeLegal(Root->getValueType(0)));
18611 OrigOperand = Root->getOperand(Num: OperandIdx);
18612
18613 unsigned Opc = Root->getOpcode();
18614 switch (Opc) {
18615 // We consider
18616 // VW<ADD|SUB>_W(LHS, RHS) -> <ADD|SUB>(LHS, SEXT(RHS))
18617 // VW<ADD|SUB>U_W(LHS, RHS) -> <ADD|SUB>(LHS, ZEXT(RHS))
18618 // VFW<ADD|SUB>_W(LHS, RHS) -> F<ADD|SUB>(LHS, FPEXT(RHS))
18619 case RISCVISD::VWADD_W_VL:
18620 case RISCVISD::VWADDU_W_VL:
18621 case RISCVISD::VWSUB_W_VL:
18622 case RISCVISD::VWSUBU_W_VL:
18623 case RISCVISD::VFWADD_W_VL:
18624 case RISCVISD::VFWSUB_W_VL:
18625 // Operand 1 can't be changed.
18626 if (OperandIdx == 1)
18627 break;
18628 [[fallthrough]];
18629 default:
18630 fillUpExtensionSupport(Root, DAG, Subtarget);
18631 break;
18632 }
18633 }
18634
18635 /// Helper function to get the Mask and VL from \p Root.
18636 static std::pair<SDValue, SDValue>
18637 getMaskAndVL(const SDNode *Root, SelectionDAG &DAG,
18638 const RISCVSubtarget &Subtarget) {
18639 assert(isSupportedRoot(Root, Subtarget) && "Unexpected root");
18640 switch (Root->getOpcode()) {
18641 case ISD::ADD:
18642 case ISD::SUB:
18643 case ISD::MUL:
18644 case ISD::OR:
18645 case ISD::SHL: {
18646 SDLoc DL(Root);
18647 MVT VT = Root->getSimpleValueType(ResNo: 0);
18648 return getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
18649 }
18650 default:
18651 return std::make_pair(x: Root->getOperand(Num: 3), y: Root->getOperand(Num: 4));
18652 }
18653 }
18654
18655 /// Helper function to check if \p N is commutative with respect to the
18656 /// foldings that are supported by this class.
18657 static bool isCommutative(const SDNode *N) {
18658 switch (N->getOpcode()) {
18659 case ISD::ADD:
18660 case ISD::MUL:
18661 case ISD::OR:
18662 case RISCVISD::ADD_VL:
18663 case RISCVISD::MUL_VL:
18664 case RISCVISD::OR_VL:
18665 case RISCVISD::FADD_VL:
18666 case RISCVISD::FMUL_VL:
18667 case RISCVISD::VFMADD_VL:
18668 case RISCVISD::VFNMSUB_VL:
18669 case RISCVISD::VFNMADD_VL:
18670 case RISCVISD::VFMSUB_VL:
18671 return true;
18672 case RISCVISD::VWADD_W_VL:
18673 case RISCVISD::VWADDU_W_VL:
18674 case ISD::SUB:
18675 case RISCVISD::SUB_VL:
18676 case RISCVISD::VWSUB_W_VL:
18677 case RISCVISD::VWSUBU_W_VL:
18678 case RISCVISD::VFWADD_W_VL:
18679 case RISCVISD::FSUB_VL:
18680 case RISCVISD::VFWSUB_W_VL:
18681 case ISD::SHL:
18682 case RISCVISD::SHL_VL:
18683 return false;
18684 default:
18685 llvm_unreachable("Unexpected opcode");
18686 }
18687 }
18688
18689 /// Get a list of combine to try for folding extensions in \p Root.
18690 /// Note that each returned CombineToTry function doesn't actually modify
18691 /// anything. Instead they produce an optional CombineResult that if not None,
18692 /// need to be materialized for the combine to be applied.
18693 /// \see CombineResult::materialize.
18694 /// If the related CombineToTry function returns std::nullopt, that means the
18695 /// combine didn't match.
18696 static SmallVector<CombineToTry>
18697 getSupportedFoldings(const SDNode *Root, const RISCVSubtarget &Subtarget);
18698};
18699
18700/// Helper structure that holds all the necessary information to materialize a
18701/// combine that does some extension folding.
18702struct CombineResult {
18703 /// Opcode to be generated when materializing the combine.
18704 unsigned TargetOpcode;
18705 // No value means no extension is needed.
18706 std::optional<ExtKind> LHSExt;
18707 std::optional<ExtKind> RHSExt;
18708 /// Root of the combine.
18709 SDNode *Root;
18710 /// LHS of the TargetOpcode.
18711 NodeExtensionHelper LHS;
18712 /// RHS of the TargetOpcode.
18713 NodeExtensionHelper RHS;
18714
18715 CombineResult(unsigned TargetOpcode, SDNode *Root,
18716 const NodeExtensionHelper &LHS, std::optional<ExtKind> LHSExt,
18717 const NodeExtensionHelper &RHS, std::optional<ExtKind> RHSExt)
18718 : TargetOpcode(TargetOpcode), LHSExt(LHSExt), RHSExt(RHSExt), Root(Root),
18719 LHS(LHS), RHS(RHS) {}
18720
18721 /// Return a value that uses TargetOpcode and that can be used to replace
18722 /// Root.
18723 /// The actual replacement is *not* done in that method.
18724 SDValue materialize(SelectionDAG &DAG,
18725 const RISCVSubtarget &Subtarget) const {
18726 SDValue Mask, VL, Passthru;
18727 std::tie(args&: Mask, args&: VL) =
18728 NodeExtensionHelper::getMaskAndVL(Root, DAG, Subtarget);
18729 switch (Root->getOpcode()) {
18730 default:
18731 Passthru = Root->getOperand(Num: 2);
18732 break;
18733 case ISD::ADD:
18734 case ISD::SUB:
18735 case ISD::MUL:
18736 case ISD::OR:
18737 case ISD::SHL:
18738 Passthru = DAG.getUNDEF(VT: Root->getValueType(ResNo: 0));
18739 break;
18740 }
18741 return DAG.getNode(Opcode: TargetOpcode, DL: SDLoc(Root), VT: Root->getValueType(ResNo: 0),
18742 N1: LHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SupportsExt: LHSExt),
18743 N2: RHS.getOrCreateExtendedOp(Root, DAG, Subtarget, SupportsExt: RHSExt),
18744 N3: Passthru, N4: Mask, N5: VL);
18745 }
18746};
18747
18748/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
18749/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
18750/// are zext) and LHS and RHS can be folded into Root.
18751/// AllowExtMask define which form `ext` can take in this pattern.
18752///
18753/// \note If the pattern can match with both zext and sext, the returned
18754/// CombineResult will feature the zext result.
18755///
18756/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18757/// can be used to apply the pattern.
18758static std::optional<CombineResult>
18759canFoldToVWWithSameExtensionImpl(SDNode *Root, const NodeExtensionHelper &LHS,
18760 const NodeExtensionHelper &RHS,
18761 uint8_t AllowExtMask, SelectionDAG &DAG,
18762 const RISCVSubtarget &Subtarget) {
18763 if ((AllowExtMask & ExtKind::ZExt) && LHS.SupportsZExt && RHS.SupportsZExt)
18764 return CombineResult(NodeExtensionHelper::getZExtOpcode(Opcode: Root->getOpcode()),
18765 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
18766 /*RHSExt=*/{ExtKind::ZExt});
18767 if ((AllowExtMask & ExtKind::SExt) && LHS.SupportsSExt && RHS.SupportsSExt)
18768 return CombineResult(NodeExtensionHelper::getSExtOpcode(Opcode: Root->getOpcode()),
18769 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
18770 /*RHSExt=*/{ExtKind::SExt});
18771 if ((AllowExtMask & ExtKind::FPExt) && LHS.SupportsFPExt && RHS.SupportsFPExt)
18772 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
18773 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
18774 /*RHSExt=*/{ExtKind::FPExt});
18775 if ((AllowExtMask & ExtKind::BF16Ext) && LHS.SupportsBF16Ext &&
18776 RHS.SupportsBF16Ext)
18777 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
18778 Root, LHS, /*LHSExt=*/{ExtKind::BF16Ext}, RHS,
18779 /*RHSExt=*/{ExtKind::BF16Ext});
18780 return std::nullopt;
18781}
18782
18783/// Check if \p Root follows a pattern Root(ext(LHS), ext(RHS))
18784/// where `ext` is the same for both LHS and RHS (i.e., both are sext or both
18785/// are zext) and LHS and RHS can be folded into Root.
18786///
18787/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18788/// can be used to apply the pattern.
18789static std::optional<CombineResult>
18790canFoldToVWWithSameExtension(SDNode *Root, const NodeExtensionHelper &LHS,
18791 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18792 const RISCVSubtarget &Subtarget) {
18793 return canFoldToVWWithSameExtensionImpl(
18794 Root, LHS, RHS, AllowExtMask: ExtKind::ZExt | ExtKind::SExt | ExtKind::FPExt, DAG,
18795 Subtarget);
18796}
18797
18798/// Check if \p Root follows a pattern Root(zext(LHS), zext(RHS))
18799///
18800/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18801/// can be used to apply the pattern.
18802static std::optional<CombineResult>
18803canFoldToVWWithSameExtZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
18804 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18805 const RISCVSubtarget &Subtarget) {
18806 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, AllowExtMask: ExtKind::ZExt, DAG,
18807 Subtarget);
18808}
18809
18810/// Check if \p Root follows a pattern Root(bf16ext(LHS), bf16ext(RHS))
18811///
18812/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18813/// can be used to apply the pattern.
18814static std::optional<CombineResult>
18815canFoldToVWWithSameExtBF16(SDNode *Root, const NodeExtensionHelper &LHS,
18816 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18817 const RISCVSubtarget &Subtarget) {
18818 return canFoldToVWWithSameExtensionImpl(Root, LHS, RHS, AllowExtMask: ExtKind::BF16Ext, DAG,
18819 Subtarget);
18820}
18821
18822/// Check if \p Root follows a pattern Root(LHS, ext(RHS))
18823///
18824/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18825/// can be used to apply the pattern.
18826static std::optional<CombineResult>
18827canFoldToVW_W(SDNode *Root, const NodeExtensionHelper &LHS,
18828 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18829 const RISCVSubtarget &Subtarget) {
18830 if (RHS.SupportsFPExt)
18831 return CombineResult(
18832 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::FPExt),
18833 Root, LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::FPExt});
18834
18835 // FIXME: Is it useful to form a vwadd.wx or vwsub.wx if it removes a scalar
18836 // sext/zext?
18837 // Control this behavior behind an option (AllowSplatInVW_W) for testing
18838 // purposes.
18839 if (RHS.SupportsZExt && (!RHS.isSplat() || AllowSplatInVW_W))
18840 return CombineResult(
18841 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::ZExt), Root,
18842 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::ZExt});
18843 if (RHS.SupportsSExt && (!RHS.isSplat() || AllowSplatInVW_W))
18844 return CombineResult(
18845 NodeExtensionHelper::getWOpcode(Opcode: Root->getOpcode(), SupportsExt: ExtKind::SExt), Root,
18846 LHS, /*LHSExt=*/std::nullopt, RHS, /*RHSExt=*/{ExtKind::SExt});
18847 return std::nullopt;
18848}
18849
18850/// Check if \p Root follows a pattern Root(sext(LHS), RHS)
18851///
18852/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18853/// can be used to apply the pattern.
18854static std::optional<CombineResult>
18855canFoldToVWWithSEXT(SDNode *Root, const NodeExtensionHelper &LHS,
18856 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18857 const RISCVSubtarget &Subtarget) {
18858 if (LHS.SupportsSExt)
18859 return CombineResult(NodeExtensionHelper::getSExtOpcode(Opcode: Root->getOpcode()),
18860 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
18861 /*RHSExt=*/std::nullopt);
18862 return std::nullopt;
18863}
18864
18865/// Check if \p Root follows a pattern Root(zext(LHS), RHS)
18866///
18867/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18868/// can be used to apply the pattern.
18869static std::optional<CombineResult>
18870canFoldToVWWithZEXT(SDNode *Root, const NodeExtensionHelper &LHS,
18871 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18872 const RISCVSubtarget &Subtarget) {
18873 if (LHS.SupportsZExt)
18874 return CombineResult(NodeExtensionHelper::getZExtOpcode(Opcode: Root->getOpcode()),
18875 Root, LHS, /*LHSExt=*/{ExtKind::ZExt}, RHS,
18876 /*RHSExt=*/std::nullopt);
18877 return std::nullopt;
18878}
18879
18880/// Check if \p Root follows a pattern Root(fpext(LHS), RHS)
18881///
18882/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18883/// can be used to apply the pattern.
18884static std::optional<CombineResult>
18885canFoldToVWWithFPEXT(SDNode *Root, const NodeExtensionHelper &LHS,
18886 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18887 const RISCVSubtarget &Subtarget) {
18888 if (LHS.SupportsFPExt)
18889 return CombineResult(NodeExtensionHelper::getFPExtOpcode(Opcode: Root->getOpcode()),
18890 Root, LHS, /*LHSExt=*/{ExtKind::FPExt}, RHS,
18891 /*RHSExt=*/std::nullopt);
18892 return std::nullopt;
18893}
18894
18895/// Check if \p Root follows a pattern Root(sext(LHS), zext(RHS))
18896///
18897/// \returns std::nullopt if the pattern doesn't match or a CombineResult that
18898/// can be used to apply the pattern.
18899static std::optional<CombineResult>
18900canFoldToVW_SU(SDNode *Root, const NodeExtensionHelper &LHS,
18901 const NodeExtensionHelper &RHS, SelectionDAG &DAG,
18902 const RISCVSubtarget &Subtarget) {
18903
18904 if (!LHS.SupportsSExt || !RHS.SupportsZExt)
18905 return std::nullopt;
18906 return CombineResult(NodeExtensionHelper::getSUOpcode(Opcode: Root->getOpcode()),
18907 Root, LHS, /*LHSExt=*/{ExtKind::SExt}, RHS,
18908 /*RHSExt=*/{ExtKind::ZExt});
18909}
18910
18911SmallVector<NodeExtensionHelper::CombineToTry>
18912NodeExtensionHelper::getSupportedFoldings(const SDNode *Root,
18913 const RISCVSubtarget &Subtarget) {
18914 SmallVector<CombineToTry> Strategies;
18915 switch (Root->getOpcode()) {
18916 case ISD::ADD:
18917 case ISD::SUB:
18918 case ISD::OR:
18919 case RISCVISD::ADD_VL:
18920 case RISCVISD::SUB_VL:
18921 case RISCVISD::OR_VL:
18922 case RISCVISD::FADD_VL:
18923 case RISCVISD::FSUB_VL:
18924 // add|sub|fadd|fsub-> vwadd(u)|vwsub(u)|vfwadd|vfwsub
18925 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
18926 // add|sub|fadd|fsub -> vwadd(u)_w|vwsub(u)_w}|vfwadd_w|vfwsub_w
18927 Strategies.push_back(Elt: canFoldToVW_W);
18928 break;
18929 case RISCVISD::FMUL_VL:
18930 case RISCVISD::VFMADD_VL:
18931 case RISCVISD::VFMSUB_VL:
18932 case RISCVISD::VFNMADD_VL:
18933 case RISCVISD::VFNMSUB_VL:
18934 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
18935 if (Subtarget.hasStdExtZvfbfa() && Root->getOpcode() != RISCVISD::FMUL_VL)
18936 // TODO: Once other widen operations are supported we can merge
18937 // canFoldToVWWithSameExtension and canFoldToVWWithSameExtBF16.
18938 Strategies.push_back(Elt: canFoldToVWWithSameExtBF16);
18939 else if (Subtarget.hasStdExtZvfbfwma() &&
18940 Root->getOpcode() == RISCVISD::VFMADD_VL)
18941 Strategies.push_back(Elt: canFoldToVWWithSameExtBF16);
18942 break;
18943 case ISD::MUL:
18944 case RISCVISD::MUL_VL:
18945 // mul -> vwmul(u)
18946 Strategies.push_back(Elt: canFoldToVWWithSameExtension);
18947 // mul -> vwmulsu
18948 Strategies.push_back(Elt: canFoldToVW_SU);
18949 break;
18950 case ISD::SHL:
18951 case RISCVISD::SHL_VL:
18952 // shl -> vwsll
18953 Strategies.push_back(Elt: canFoldToVWWithSameExtZEXT);
18954 break;
18955 case RISCVISD::VWADD_W_VL:
18956 case RISCVISD::VWSUB_W_VL:
18957 // vwadd_w|vwsub_w -> vwadd|vwsub
18958 Strategies.push_back(Elt: canFoldToVWWithSEXT);
18959 break;
18960 case RISCVISD::VWADDU_W_VL:
18961 case RISCVISD::VWSUBU_W_VL:
18962 // vwaddu_w|vwsubu_w -> vwaddu|vwsubu
18963 Strategies.push_back(Elt: canFoldToVWWithZEXT);
18964 break;
18965 case RISCVISD::VFWADD_W_VL:
18966 case RISCVISD::VFWSUB_W_VL:
18967 // vfwadd_w|vfwsub_w -> vfwadd|vfwsub
18968 Strategies.push_back(Elt: canFoldToVWWithFPEXT);
18969 break;
18970 default:
18971 llvm_unreachable("Unexpected opcode");
18972 }
18973 return Strategies;
18974}
18975} // End anonymous namespace.
18976
18977static SDValue simplifyOp_VL(SDNode *N) {
18978 // TODO: Extend this to other binops using generic identity logic
18979 assert(N->getOpcode() == RISCVISD::ADD_VL);
18980 SDValue A = N->getOperand(Num: 0);
18981 SDValue B = N->getOperand(Num: 1);
18982 SDValue Passthru = N->getOperand(Num: 2);
18983 if (!Passthru.isUndef())
18984 // TODO:This could be a vmerge instead
18985 return SDValue();
18986 ;
18987 if (ISD::isConstantSplatVectorAllZeros(N: B.getNode()))
18988 return A;
18989 // Peek through fixed to scalable
18990 if (B.getOpcode() == ISD::INSERT_SUBVECTOR && B.getOperand(i: 0).isUndef() &&
18991 ISD::isConstantSplatVectorAllZeros(N: B.getOperand(i: 1).getNode()))
18992 return A;
18993 return SDValue();
18994}
18995
18996/// Combine a binary or FMA operation to its equivalent VW or VW_W form.
18997/// The supported combines are:
18998/// add | add_vl | or disjoint | or_vl disjoint -> vwadd(u) | vwadd(u)_w
18999/// sub | sub_vl -> vwsub(u) | vwsub(u)_w
19000/// mul | mul_vl -> vwmul(u) | vwmul_su
19001/// shl | shl_vl -> vwsll
19002/// fadd_vl -> vfwadd | vfwadd_w
19003/// fsub_vl -> vfwsub | vfwsub_w
19004/// fmul_vl -> vfwmul
19005/// vwadd_w(u) -> vwadd(u)
19006/// vwsub_w(u) -> vwsub(u)
19007/// vfwadd_w -> vfwadd
19008/// vfwsub_w -> vfwsub
19009static SDValue combineOp_VLToVWOp_VL(SDNode *N,
19010 TargetLowering::DAGCombinerInfo &DCI,
19011 const RISCVSubtarget &Subtarget) {
19012 SelectionDAG &DAG = DCI.DAG;
19013 if (DCI.isBeforeLegalize())
19014 return SDValue();
19015
19016 if (!NodeExtensionHelper::isSupportedRoot(Root: N, Subtarget))
19017 return SDValue();
19018
19019 SmallVector<SDNode *> Worklist;
19020 SmallPtrSet<SDNode *, 8> Inserted;
19021 SmallPtrSet<SDNode *, 8> ExtensionsToRemove;
19022 Worklist.push_back(Elt: N);
19023 Inserted.insert(Ptr: N);
19024 SmallVector<CombineResult> CombinesToApply;
19025
19026 while (!Worklist.empty()) {
19027 SDNode *Root = Worklist.pop_back_val();
19028
19029 NodeExtensionHelper LHS(Root, 0, DAG, Subtarget);
19030 NodeExtensionHelper RHS(Root, 1, DAG, Subtarget);
19031 auto AppendUsersIfNeeded =
19032 [&Worklist, &Subtarget, &Inserted,
19033 &ExtensionsToRemove](const NodeExtensionHelper &Op) {
19034 if (Op.needToPromoteOtherUsers()) {
19035 // Remember that we're supposed to remove this extension.
19036 ExtensionsToRemove.insert(Ptr: Op.OrigOperand.getNode());
19037 for (SDUse &Use : Op.OrigOperand->uses()) {
19038 SDNode *TheUser = Use.getUser();
19039 if (!NodeExtensionHelper::isSupportedRoot(Root: TheUser, Subtarget))
19040 return false;
19041 // We only support the first 2 operands of FMA.
19042 if (Use.getOperandNo() >= 2)
19043 return false;
19044 if (Inserted.insert(Ptr: TheUser).second)
19045 Worklist.push_back(Elt: TheUser);
19046 }
19047 }
19048 return true;
19049 };
19050
19051 // Control the compile time by limiting the number of node we look at in
19052 // total.
19053 if (Inserted.size() > ExtensionMaxWebSize)
19054 return SDValue();
19055
19056 SmallVector<NodeExtensionHelper::CombineToTry> FoldingStrategies =
19057 NodeExtensionHelper::getSupportedFoldings(Root, Subtarget);
19058
19059 assert(!FoldingStrategies.empty() && "Nothing to be folded");
19060 bool Matched = false;
19061 for (int Attempt = 0;
19062 (Attempt != 1 + NodeExtensionHelper::isCommutative(N: Root)) && !Matched;
19063 ++Attempt) {
19064
19065 for (NodeExtensionHelper::CombineToTry FoldingStrategy :
19066 FoldingStrategies) {
19067 std::optional<CombineResult> Res =
19068 FoldingStrategy(Root, LHS, RHS, DAG, Subtarget);
19069 if (Res) {
19070 // If this strategy wouldn't remove an extension we're supposed to
19071 // remove, reject it.
19072 if (!Res->LHSExt.has_value() &&
19073 ExtensionsToRemove.contains(Ptr: LHS.OrigOperand.getNode()))
19074 continue;
19075 if (!Res->RHSExt.has_value() &&
19076 ExtensionsToRemove.contains(Ptr: RHS.OrigOperand.getNode()))
19077 continue;
19078
19079 Matched = true;
19080 CombinesToApply.push_back(Elt: *Res);
19081 // All the inputs that are extended need to be folded, otherwise
19082 // we would be leaving the old input (since it is may still be used),
19083 // and the new one.
19084 if (Res->LHSExt.has_value())
19085 if (!AppendUsersIfNeeded(LHS))
19086 return SDValue();
19087 if (Res->RHSExt.has_value())
19088 if (!AppendUsersIfNeeded(RHS))
19089 return SDValue();
19090 break;
19091 }
19092 }
19093 std::swap(a&: LHS, b&: RHS);
19094 }
19095 // Right now we do an all or nothing approach.
19096 if (!Matched)
19097 return SDValue();
19098 }
19099 // Store the value for the replacement of the input node separately.
19100 SDValue InputRootReplacement;
19101 // We do the RAUW after we materialize all the combines, because some replaced
19102 // nodes may be feeding some of the yet-to-be-replaced nodes. Put differently,
19103 // some of these nodes may appear in the NodeExtensionHelpers of some of the
19104 // yet-to-be-visited CombinesToApply roots.
19105 SmallVector<std::pair<SDValue, SDValue>> ValuesToReplace;
19106 ValuesToReplace.reserve(N: CombinesToApply.size());
19107 for (CombineResult Res : CombinesToApply) {
19108 SDValue NewValue = Res.materialize(DAG, Subtarget);
19109 if (!InputRootReplacement) {
19110 assert(Res.Root == N &&
19111 "First element is expected to be the current node");
19112 InputRootReplacement = NewValue;
19113 } else {
19114 ValuesToReplace.emplace_back(Args: SDValue(Res.Root, 0), Args&: NewValue);
19115 }
19116 }
19117 for (std::pair<SDValue, SDValue> OldNewValues : ValuesToReplace) {
19118 DCI.CombineTo(N: OldNewValues.first.getNode(), Res: OldNewValues.second);
19119 }
19120 return InputRootReplacement;
19121}
19122
19123// Fold (vwadd(u).wv y, (vmerge cond, x, 0)) -> vwadd(u).wv y, x, y, cond
19124// (vwsub(u).wv y, (vmerge cond, x, 0)) -> vwsub(u).wv y, x, y, cond
19125// y will be the Passthru and cond will be the Mask.
19126static SDValue combineVWADDSUBWSelect(SDNode *N, SelectionDAG &DAG) {
19127 unsigned Opc = N->getOpcode();
19128 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
19129 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
19130
19131 SDValue Y = N->getOperand(Num: 0);
19132 SDValue MergeOp = N->getOperand(Num: 1);
19133 unsigned MergeOpc = MergeOp.getOpcode();
19134
19135 if (MergeOpc != RISCVISD::VMERGE_VL && MergeOpc != ISD::VSELECT)
19136 return SDValue();
19137
19138 SDValue X = MergeOp->getOperand(Num: 1);
19139
19140 if (!MergeOp.hasOneUse())
19141 return SDValue();
19142
19143 // Passthru should be undef
19144 SDValue Passthru = N->getOperand(Num: 2);
19145 if (!Passthru.isUndef())
19146 return SDValue();
19147
19148 // Mask should be all ones
19149 SDValue Mask = N->getOperand(Num: 3);
19150 if (Mask.getOpcode() != RISCVISD::VMSET_VL)
19151 return SDValue();
19152
19153 // False value of MergeOp should be all zeros
19154 SDValue Z = MergeOp->getOperand(Num: 2);
19155
19156 if (Z.getOpcode() == ISD::INSERT_SUBVECTOR &&
19157 (isNullOrNullSplat(V: Z.getOperand(i: 0)) || Z.getOperand(i: 0).isUndef()))
19158 Z = Z.getOperand(i: 1);
19159
19160 if (!ISD::isConstantSplatVectorAllZeros(N: Z.getNode()))
19161 return SDValue();
19162
19163 return DAG.getNode(Opcode: Opc, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
19164 Ops: {Y, X, Y, MergeOp->getOperand(Num: 0), N->getOperand(Num: 4)},
19165 Flags: N->getFlags());
19166}
19167
19168// vwaddu C (vabd A B) -> vwabda(A B C)
19169// vwaddu C (vabdu A B) -> vwabdau(A B C)
19170static SDValue performVWABDACombine(SDNode *N, SelectionDAG &DAG,
19171 const RISCVSubtarget &Subtarget) {
19172 if (!Subtarget.hasStdExtZvabd())
19173 return SDValue();
19174
19175 MVT VT = N->getSimpleValueType(ResNo: 0);
19176 if (VT.getVectorElementType() != MVT::i8 &&
19177 VT.getVectorElementType() != MVT::i16)
19178 return SDValue();
19179
19180 SDValue Op0 = N->getOperand(Num: 0);
19181 SDValue Op1 = N->getOperand(Num: 1);
19182 SDValue Passthru = N->getOperand(Num: 2);
19183 if (!Passthru->isUndef())
19184 return SDValue();
19185
19186 SDValue Mask = N->getOperand(Num: 3);
19187 SDValue VL = N->getOperand(Num: 4);
19188 auto IsABD = [](SDValue Op) {
19189 if (Op->getOpcode() != RISCVISD::ABDS_VL &&
19190 Op->getOpcode() != RISCVISD::ABDU_VL)
19191 return SDValue();
19192 return Op;
19193 };
19194
19195 SDValue Diff = IsABD(Op0);
19196 Diff = Diff ? Diff : IsABD(Op1);
19197 if (!Diff)
19198 return SDValue();
19199 SDValue Acc = Diff == Op0 ? Op1 : Op0;
19200
19201 SDLoc DL(N);
19202 Acc = DAG.getNode(Opcode: RISCVISD::VZEXT_VL, DL, VT, N1: Acc, N2: Mask, N3: VL);
19203 SDValue Result = DAG.getNode(
19204 Opcode: Diff.getOpcode() == RISCVISD::ABDS_VL ? RISCVISD::VWABDA_VL
19205 : RISCVISD::VWABDAU_VL,
19206 DL, VT, N1: Diff.getOperand(i: 0), N2: Diff.getOperand(i: 1), N3: Acc, N4: Mask, N5: VL);
19207 return Result;
19208}
19209
19210// vwaddu_wv C (vabd A B) -> vwabda(A B C)
19211// vwaddu_wv C (zext (vabd A B)) -> vwabda(A (sext B) (sext C))
19212// vwaddu_wv C (vabdu A B) -> vwabdau(A B C)
19213// vwaddu_wv C (zext (vabdu A B)) -> vwabdau(A (zext B) (zext C))
19214static SDValue performVWABDACombineWV(SDNode *N, SelectionDAG &DAG,
19215 const RISCVSubtarget &Subtarget) {
19216 if (!Subtarget.hasStdExtZvabd())
19217 return SDValue();
19218
19219 MVT VT = N->getSimpleValueType(ResNo: 0);
19220 // The result is widened, so we can accept i16/i32 here.
19221 if (VT.getVectorElementType() != MVT::i16 &&
19222 VT.getVectorElementType() != MVT::i32)
19223 return SDValue();
19224
19225 SDValue Op0 = N->getOperand(Num: 0);
19226 SDValue Op1 = N->getOperand(Num: 1);
19227 SDValue Passthru = N->getOperand(Num: 2);
19228 if (!Passthru->isUndef())
19229 return SDValue();
19230
19231 SDValue Mask = N->getOperand(Num: 3);
19232 SDValue VL = N->getOperand(Num: 4);
19233 unsigned ExtOpc = 0;
19234 MVT ExtVT;
19235 auto GetDiff = [&](SDValue Op) {
19236 unsigned Opc = Op.getOpcode();
19237 if (Opc == RISCVISD::VZEXT_VL) {
19238 SDValue Src = Op->getOperand(Num: 0);
19239 unsigned SrcOpc = Src.getOpcode();
19240 switch (SrcOpc) {
19241 default:
19242 return SDValue();
19243 case ISD::ABDS:
19244 case RISCVISD::ABDS_VL:
19245 ExtOpc = RISCVISD::VSEXT_VL;
19246 break;
19247 case ISD::ABDU:
19248 case RISCVISD::ABDU_VL:
19249 ExtOpc = RISCVISD::VZEXT_VL;
19250 break;
19251 }
19252 ExtVT = Op->getSimpleValueType(ResNo: 0);
19253 return Src;
19254 }
19255
19256 if (Opc != ISD::ABDS && Opc != ISD::ABDU && Opc != RISCVISD::ABDS_VL &&
19257 Opc != RISCVISD::ABDU_VL)
19258 return SDValue();
19259 return Op;
19260 };
19261
19262 SDValue Diff = GetDiff(Op0);
19263 if (!Diff) {
19264 std::swap(a&: Op0, b&: Op1);
19265 Diff = GetDiff(Op0);
19266 if (!Diff)
19267 return SDValue();
19268 }
19269 SDValue Acc = Op1;
19270
19271 SDLoc DL(N);
19272 SDValue DiffA = Diff.getOperand(i: 0);
19273 SDValue DiffB = Diff.getOperand(i: 1);
19274 if (ExtOpc) {
19275 DiffA = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtVT, N1: DiffA, N2: Mask, N3: VL);
19276 DiffB = DAG.getNode(Opcode: ExtOpc, DL, VT: ExtVT, N1: DiffB, N2: Mask, N3: VL);
19277 }
19278 SDValue Result = DAG.getNode(Opcode: Diff.getOpcode() == ISD::ABDS ||
19279 Diff.getOpcode() == RISCVISD::ABDS_VL
19280 ? RISCVISD::VWABDA_VL
19281 : RISCVISD::VWABDAU_VL,
19282 DL, VT, N1: DiffA, N2: DiffB, N3: Acc, N4: Mask, N5: VL);
19283 return Result;
19284}
19285
19286static SDValue performVWADDSUBW_VLCombine(SDNode *N,
19287 TargetLowering::DAGCombinerInfo &DCI,
19288 const RISCVSubtarget &Subtarget) {
19289 [[maybe_unused]] unsigned Opc = N->getOpcode();
19290 assert(Opc == RISCVISD::VWADD_W_VL || Opc == RISCVISD::VWADDU_W_VL ||
19291 Opc == RISCVISD::VWSUB_W_VL || Opc == RISCVISD::VWSUBU_W_VL);
19292
19293 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
19294 return V;
19295
19296 return combineVWADDSUBWSelect(N, DAG&: DCI.DAG);
19297}
19298
19299// Helper function for performMemPairCombine.
19300// Try to combine the memory loads/stores LSNode1 and LSNode2
19301// into a single memory pair operation.
19302static SDValue tryMemPairCombine(SelectionDAG &DAG, LSBaseSDNode *LSNode1,
19303 LSBaseSDNode *LSNode2, SDValue BasePtr,
19304 uint64_t Imm) {
19305 SmallPtrSet<const SDNode *, 32> Visited;
19306 SmallVector<const SDNode *, 8> Worklist = {LSNode1, LSNode2};
19307
19308 if (SDNode::hasPredecessorHelper(N: LSNode1, Visited, Worklist) ||
19309 SDNode::hasPredecessorHelper(N: LSNode2, Visited, Worklist))
19310 return SDValue();
19311
19312 MachineFunction &MF = DAG.getMachineFunction();
19313 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
19314
19315 // The new operation has twice the width.
19316 MVT XLenVT = Subtarget.getXLenVT();
19317 EVT MemVT = LSNode1->getMemoryVT();
19318 EVT NewMemVT = (MemVT == MVT::i32) ? MVT::i64 : MVT::i128;
19319 MachineMemOperand *MMO = LSNode1->getMemOperand();
19320 MachineMemOperand *NewMMO = MF.getMachineMemOperand(
19321 MMO, PtrInfo: MMO->getPointerInfo(), Size: MemVT == MVT::i32 ? 8 : 16);
19322
19323 if (LSNode1->getOpcode() == ISD::LOAD) {
19324 auto Ext = cast<LoadSDNode>(Val: LSNode1)->getExtensionType();
19325 unsigned Opcode;
19326 if (MemVT == MVT::i32)
19327 Opcode = (Ext == ISD::ZEXTLOAD) ? RISCVISD::TH_LWUD : RISCVISD::TH_LWD;
19328 else
19329 Opcode = RISCVISD::TH_LDD;
19330
19331 SDValue Res = DAG.getMemIntrinsicNode(
19332 Opcode, dl: SDLoc(LSNode1), VTList: DAG.getVTList(VTs: {XLenVT, XLenVT, MVT::Other}),
19333 Ops: {LSNode1->getChain(), BasePtr,
19334 DAG.getConstant(Val: Imm, DL: SDLoc(LSNode1), VT: XLenVT)},
19335 MemVT: NewMemVT, MMO: NewMMO);
19336
19337 SDValue Node1 =
19338 DAG.getMergeValues(Ops: {Res.getValue(R: 0), Res.getValue(R: 2)}, dl: SDLoc(LSNode1));
19339 SDValue Node2 =
19340 DAG.getMergeValues(Ops: {Res.getValue(R: 1), Res.getValue(R: 2)}, dl: SDLoc(LSNode2));
19341
19342 DAG.ReplaceAllUsesWith(From: LSNode2, To: Node2.getNode());
19343 return Node1;
19344 } else {
19345 unsigned Opcode = (MemVT == MVT::i32) ? RISCVISD::TH_SWD : RISCVISD::TH_SDD;
19346
19347 SDValue Res = DAG.getMemIntrinsicNode(
19348 Opcode, dl: SDLoc(LSNode1), VTList: DAG.getVTList(VT: MVT::Other),
19349 Ops: {LSNode1->getChain(), LSNode1->getOperand(Num: 1), LSNode2->getOperand(Num: 1),
19350 BasePtr, DAG.getConstant(Val: Imm, DL: SDLoc(LSNode1), VT: XLenVT)},
19351 MemVT: NewMemVT, MMO: NewMMO);
19352
19353 DAG.ReplaceAllUsesWith(From: LSNode2, To: Res.getNode());
19354 return Res;
19355 }
19356}
19357
19358// Try to combine two adjacent loads/stores to a single pair instruction from
19359// the XTHeadMemPair vendor extension.
19360static SDValue performMemPairCombine(SDNode *N,
19361 TargetLowering::DAGCombinerInfo &DCI) {
19362 SelectionDAG &DAG = DCI.DAG;
19363 MachineFunction &MF = DAG.getMachineFunction();
19364 const RISCVSubtarget &Subtarget = MF.getSubtarget<RISCVSubtarget>();
19365
19366 // Target does not support load/store pair.
19367 if (!Subtarget.hasVendorXTHeadMemPair())
19368 return SDValue();
19369
19370 LSBaseSDNode *LSNode1 = cast<LSBaseSDNode>(Val: N);
19371 EVT MemVT = LSNode1->getMemoryVT();
19372 unsigned OpNum = LSNode1->getOpcode() == ISD::LOAD ? 1 : 2;
19373
19374 // No volatile, indexed or atomic loads/stores.
19375 if (!LSNode1->isSimple() || LSNode1->isIndexed())
19376 return SDValue();
19377
19378 // Function to get a base + constant representation from a memory value.
19379 auto ExtractBaseAndOffset = [](SDValue Ptr) -> std::pair<SDValue, uint64_t> {
19380 if (Ptr->getOpcode() == ISD::ADD)
19381 if (auto *C1 = dyn_cast<ConstantSDNode>(Val: Ptr->getOperand(Num: 1)))
19382 return {Ptr->getOperand(Num: 0), C1->getZExtValue()};
19383 return {Ptr, 0};
19384 };
19385
19386 auto [Base1, Offset1] = ExtractBaseAndOffset(LSNode1->getOperand(Num: OpNum));
19387
19388 SDValue Chain = N->getOperand(Num: 0);
19389 for (SDUse &Use : Chain->uses()) {
19390 if (Use.getUser() != N && Use.getResNo() == 0 &&
19391 Use.getUser()->getOpcode() == N->getOpcode()) {
19392 LSBaseSDNode *LSNode2 = cast<LSBaseSDNode>(Val: Use.getUser());
19393
19394 // No volatile, indexed or atomic loads/stores.
19395 if (!LSNode2->isSimple() || LSNode2->isIndexed())
19396 continue;
19397
19398 // Check if LSNode1 and LSNode2 have the same type and extension.
19399 if (LSNode1->getOpcode() == ISD::LOAD)
19400 if (cast<LoadSDNode>(Val: LSNode2)->getExtensionType() !=
19401 cast<LoadSDNode>(Val: LSNode1)->getExtensionType())
19402 continue;
19403
19404 if (LSNode1->getMemoryVT() != LSNode2->getMemoryVT())
19405 continue;
19406
19407 auto [Base2, Offset2] = ExtractBaseAndOffset(LSNode2->getOperand(Num: OpNum));
19408
19409 // Check if the base pointer is the same for both instruction.
19410 if (Base1 != Base2)
19411 continue;
19412
19413 // Check if the offsets match the XTHeadMemPair encoding constraints.
19414 bool Valid = false;
19415 if (MemVT == MVT::i32) {
19416 // Check for adjacent i32 values and a 2-bit index.
19417 if ((Offset1 + 4 == Offset2) && isShiftedUInt<2, 3>(x: Offset1))
19418 Valid = true;
19419 } else if (MemVT == MVT::i64) {
19420 // Check for adjacent i64 values and a 2-bit index.
19421 if ((Offset1 + 8 == Offset2) && isShiftedUInt<2, 4>(x: Offset1))
19422 Valid = true;
19423 }
19424
19425 if (!Valid)
19426 continue;
19427
19428 // Try to combine.
19429 if (SDValue Res =
19430 tryMemPairCombine(DAG, LSNode1, LSNode2, BasePtr: Base1, Imm: Offset1))
19431 return Res;
19432 }
19433 }
19434
19435 return SDValue();
19436}
19437
19438// Fold
19439// (fp_to_int (froundeven X)) -> fcvt X, rne
19440// (fp_to_int (ftrunc X)) -> fcvt X, rtz
19441// (fp_to_int (ffloor X)) -> fcvt X, rdn
19442// (fp_to_int (fceil X)) -> fcvt X, rup
19443// (fp_to_int (fround X)) -> fcvt X, rmm
19444// (fp_to_int (frint X)) -> fcvt X
19445static SDValue performFP_TO_INTCombine(SDNode *N,
19446 TargetLowering::DAGCombinerInfo &DCI,
19447 const RISCVSubtarget &Subtarget) {
19448 SelectionDAG &DAG = DCI.DAG;
19449 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19450 MVT XLenVT = Subtarget.getXLenVT();
19451
19452 SDValue Src = N->getOperand(Num: 0);
19453
19454 // Don't do this for strict-fp Src.
19455 if (Src->isStrictFPOpcode())
19456 return SDValue();
19457
19458 // Ensure the FP type is legal.
19459 if (!TLI.isTypeLegal(VT: Src.getValueType()))
19460 return SDValue();
19461
19462 // Don't do this for f16 with Zfhmin and not Zfh.
19463 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
19464 return SDValue();
19465
19466 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Src.getOpcode());
19467 // If the result is invalid, we didn't find a foldable instruction.
19468 if (FRM == RISCVFPRndMode::Invalid)
19469 return SDValue();
19470
19471 SDLoc DL(N);
19472 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT;
19473 EVT VT = N->getValueType(ResNo: 0);
19474
19475 if (VT.isVector() && TLI.isTypeLegal(VT)) {
19476 MVT SrcVT = Src.getSimpleValueType();
19477 MVT SrcContainerVT = SrcVT;
19478 MVT ContainerVT = VT.getSimpleVT();
19479 SDValue XVal = Src.getOperand(i: 0);
19480
19481 // For widening and narrowing conversions we just combine it into a
19482 // VFCVT_..._VL node, as there are no specific VFWCVT/VFNCVT VL nodes. They
19483 // end up getting lowered to their appropriate pseudo instructions based on
19484 // their operand types
19485 if (VT.getScalarSizeInBits() > SrcVT.getScalarSizeInBits() * 2 ||
19486 VT.getScalarSizeInBits() * 2 < SrcVT.getScalarSizeInBits())
19487 return SDValue();
19488
19489 // Make fixed-length vectors scalable first
19490 if (SrcVT.isFixedLengthVector()) {
19491 SrcContainerVT = getContainerForFixedLengthVector(DAG, VT: SrcVT, Subtarget);
19492 XVal = convertToScalableVector(VT: SrcContainerVT, V: XVal, DAG, Subtarget);
19493 ContainerVT =
19494 getContainerForFixedLengthVector(DAG, VT: ContainerVT, Subtarget);
19495 }
19496
19497 auto [Mask, VL] =
19498 getDefaultVLOps(VecVT: SrcVT, ContainerVT: SrcContainerVT, DL, DAG, Subtarget);
19499
19500 SDValue FpToInt;
19501 if (FRM == RISCVFPRndMode::RTZ) {
19502 // Use the dedicated trunc static rounding mode if we're truncating so we
19503 // don't need to generate calls to fsrmi/fsrm
19504 unsigned Opc =
19505 IsSigned ? RISCVISD::VFCVT_RTZ_X_F_VL : RISCVISD::VFCVT_RTZ_XU_F_VL;
19506 FpToInt = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: XVal, N2: Mask, N3: VL);
19507 } else {
19508 unsigned Opc =
19509 IsSigned ? RISCVISD::VFCVT_RM_X_F_VL : RISCVISD::VFCVT_RM_XU_F_VL;
19510 FpToInt = DAG.getNode(Opcode: Opc, DL, VT: ContainerVT, N1: XVal, N2: Mask,
19511 N3: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT), N4: VL);
19512 }
19513
19514 // If converted from fixed-length to scalable, convert back
19515 if (VT.isFixedLengthVector())
19516 FpToInt = convertFromScalableVector(VT, V: FpToInt, DAG, Subtarget);
19517
19518 return FpToInt;
19519 }
19520
19521 // Only handle XLen or i32 types. Other types narrower than XLen will
19522 // eventually be legalized to XLenVT.
19523 if (VT != MVT::i32 && VT != XLenVT)
19524 return SDValue();
19525
19526 unsigned Opc;
19527 if (VT == XLenVT)
19528 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
19529 else
19530 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
19531
19532 SDValue FpToInt = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Src.getOperand(i: 0),
19533 N2: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT));
19534 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: FpToInt);
19535}
19536
19537// Fold
19538// (fp_to_int_sat (froundeven X)) -> (select X == nan, 0, (fcvt X, rne))
19539// (fp_to_int_sat (ftrunc X)) -> (select X == nan, 0, (fcvt X, rtz))
19540// (fp_to_int_sat (ffloor X)) -> (select X == nan, 0, (fcvt X, rdn))
19541// (fp_to_int_sat (fceil X)) -> (select X == nan, 0, (fcvt X, rup))
19542// (fp_to_int_sat (fround X)) -> (select X == nan, 0, (fcvt X, rmm))
19543// (fp_to_int_sat (frint X)) -> (select X == nan, 0, (fcvt X, dyn))
19544static SDValue performFP_TO_INT_SATCombine(SDNode *N,
19545 TargetLowering::DAGCombinerInfo &DCI,
19546 const RISCVSubtarget &Subtarget) {
19547 SelectionDAG &DAG = DCI.DAG;
19548 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19549 MVT XLenVT = Subtarget.getXLenVT();
19550
19551 // Only handle XLen types. Other types narrower than XLen will eventually be
19552 // legalized to XLenVT.
19553 EVT DstVT = N->getValueType(ResNo: 0);
19554 if (DstVT != XLenVT)
19555 return SDValue();
19556
19557 SDValue Src = N->getOperand(Num: 0);
19558
19559 // Don't do this for strict-fp Src.
19560 if (Src->isStrictFPOpcode())
19561 return SDValue();
19562
19563 // Ensure the FP type is also legal.
19564 if (!TLI.isTypeLegal(VT: Src.getValueType()))
19565 return SDValue();
19566
19567 // Don't do this for f16 with Zfhmin and not Zfh.
19568 if (Src.getValueType() == MVT::f16 && !Subtarget.hasStdExtZfh())
19569 return SDValue();
19570
19571 EVT SatVT = cast<VTSDNode>(Val: N->getOperand(Num: 1))->getVT();
19572
19573 RISCVFPRndMode::RoundingMode FRM = matchRoundingOp(Opc: Src.getOpcode());
19574 if (FRM == RISCVFPRndMode::Invalid)
19575 return SDValue();
19576
19577 bool IsSigned = N->getOpcode() == ISD::FP_TO_SINT_SAT;
19578
19579 unsigned Opc;
19580 if (SatVT == DstVT)
19581 Opc = IsSigned ? RISCVISD::FCVT_X : RISCVISD::FCVT_XU;
19582 else if (DstVT == MVT::i64 && SatVT == MVT::i32)
19583 Opc = IsSigned ? RISCVISD::FCVT_W_RV64 : RISCVISD::FCVT_WU_RV64;
19584 else
19585 return SDValue();
19586 // FIXME: Support other SatVTs by clamping before or after the conversion.
19587
19588 Src = Src.getOperand(i: 0);
19589
19590 SDLoc DL(N);
19591 SDValue FpToInt = DAG.getNode(Opcode: Opc, DL, VT: XLenVT, N1: Src,
19592 N2: DAG.getTargetConstant(Val: FRM, DL, VT: XLenVT));
19593
19594 // fcvt.wu.* sign extends bit 31 on RV64. FP_TO_UINT_SAT expects to zero
19595 // extend.
19596 if (Opc == RISCVISD::FCVT_WU_RV64)
19597 FpToInt = DAG.getZeroExtendInReg(Op: FpToInt, DL, VT: MVT::i32);
19598
19599 // RISC-V FP-to-int conversions saturate to the destination register size, but
19600 // don't produce 0 for nan.
19601 SDValue ZeroInt = DAG.getConstant(Val: 0, DL, VT: DstVT);
19602 return DAG.getSelectCC(DL, LHS: Src, RHS: Src, True: ZeroInt, False: FpToInt, Cond: ISD::CondCode::SETUO);
19603}
19604
19605// Combine (bitreverse (bswap X)) to the BREV8 GREVI encoding if the type is
19606// smaller than XLenVT.
19607static SDValue performBITREVERSECombine(SDNode *N, SelectionDAG &DAG,
19608 const RISCVSubtarget &Subtarget) {
19609 assert(Subtarget.hasStdExtZbkb() && "Unexpected extension");
19610
19611 SDValue Src = N->getOperand(Num: 0);
19612 if (Src.getOpcode() != ISD::BSWAP)
19613 return SDValue();
19614
19615 EVT VT = N->getValueType(ResNo: 0);
19616 if (!VT.isScalarInteger() || VT.getSizeInBits() >= Subtarget.getXLen() ||
19617 !llvm::has_single_bit<uint32_t>(Value: VT.getSizeInBits()))
19618 return SDValue();
19619
19620 SDLoc DL(N);
19621 return DAG.getNode(Opcode: RISCVISD::BREV8, DL, VT, Operand: Src.getOperand(i: 0));
19622}
19623
19624static SDValue performVP_REVERSECombine(SDNode *N, SelectionDAG &DAG,
19625 const RISCVSubtarget &Subtarget) {
19626 // Fold:
19627 // vp.reverse(vp.load(ADDR, MASK)) -> vp.strided.load(ADDR, -1, MASK)
19628
19629 // Check if its first operand is a vp.load.
19630 auto *VPLoad = dyn_cast<VPLoadSDNode>(Val: N->getOperand(Num: 0));
19631 if (!VPLoad)
19632 return SDValue();
19633
19634 EVT LoadVT = VPLoad->getValueType(ResNo: 0);
19635 // We do not have a strided_load version for masks, and the evl of vp.reverse
19636 // and vp.load should always be the same.
19637 if (!LoadVT.getVectorElementType().isByteSized() ||
19638 N->getOperand(Num: 2) != VPLoad->getVectorLength() ||
19639 !N->getOperand(Num: 0).hasOneUse())
19640 return SDValue();
19641
19642 SDValue LoadMask = VPLoad->getMask();
19643 // If Mask is all ones, then load is unmasked and can be reversed.
19644 if (!isOneOrOneSplat(V: LoadMask)) {
19645 // If the mask is not all ones, we can reverse the load if the mask was also
19646 // reversed by a vp.reverse with the same EVL.
19647 if (LoadMask.getOpcode() != ISD::EXPERIMENTAL_VP_REVERSE ||
19648 LoadMask.getOperand(i: 2) != VPLoad->getVectorLength())
19649 return SDValue();
19650 LoadMask = LoadMask.getOperand(i: 0);
19651 }
19652
19653 // Base = LoadAddr + (NumElem - 1) * ElemWidthByte
19654 SDLoc DL(N);
19655 MVT XLenVT = Subtarget.getXLenVT();
19656 SDValue NumElem = VPLoad->getVectorLength();
19657 uint64_t ElemWidthByte = VPLoad->getValueType(ResNo: 0).getScalarSizeInBits() / 8;
19658
19659 SDValue Temp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: NumElem,
19660 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
19661 SDValue Temp2 = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Temp1,
19662 N2: DAG.getConstant(Val: ElemWidthByte, DL, VT: XLenVT));
19663 SDValue Base = DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: VPLoad->getBasePtr(), N2: Temp2);
19664 SDValue Stride = DAG.getSignedConstant(Val: -ElemWidthByte, DL, VT: XLenVT);
19665
19666 MachineFunction &MF = DAG.getMachineFunction();
19667 MachinePointerInfo PtrInfo(VPLoad->getAddressSpace());
19668 MachineMemOperand *MMO = MF.getMachineMemOperand(
19669 PtrInfo, F: VPLoad->getMemOperand()->getFlags(),
19670 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: VPLoad->getAlign());
19671
19672 SDValue Ret = DAG.getStridedLoadVP(
19673 VT: LoadVT, DL, Chain: VPLoad->getChain(), Ptr: Base, Stride, Mask: LoadMask,
19674 EVL: VPLoad->getVectorLength(), MMO, IsExpanding: VPLoad->isExpandingLoad());
19675
19676 DAG.ReplaceAllUsesOfValueWith(From: SDValue(VPLoad, 1), To: Ret.getValue(R: 1));
19677
19678 return Ret;
19679}
19680
19681static SDValue performVP_STORECombine(SDNode *N, SelectionDAG &DAG,
19682 const RISCVSubtarget &Subtarget) {
19683 // Fold:
19684 // vp.store(vp.reverse(VAL), ADDR, MASK) -> vp.strided.store(VAL, NEW_ADDR,
19685 // -1, MASK)
19686 auto *VPStore = cast<VPStoreSDNode>(Val: N);
19687
19688 if (VPStore->getValue().getOpcode() != ISD::EXPERIMENTAL_VP_REVERSE)
19689 return SDValue();
19690
19691 SDValue VPReverse = VPStore->getValue();
19692 EVT ReverseVT = VPReverse->getValueType(ResNo: 0);
19693
19694 // We do not have a strided_store version for masks, and the evl of vp.reverse
19695 // and vp.store should always be the same.
19696 if (!ReverseVT.getVectorElementType().isByteSized() ||
19697 VPStore->getVectorLength() != VPReverse.getOperand(i: 2) ||
19698 !VPReverse.hasOneUse())
19699 return SDValue();
19700
19701 SDValue StoreMask = VPStore->getMask();
19702 // If Mask is all ones, then load is unmasked and can be reversed.
19703 if (!isOneOrOneSplat(V: StoreMask)) {
19704 // If the mask is not all ones, we can reverse the store if the mask was
19705 // also reversed by a vp.reverse with the same EVL.
19706 if (StoreMask.getOpcode() != ISD::EXPERIMENTAL_VP_REVERSE ||
19707 StoreMask.getOperand(i: 2) != VPStore->getVectorLength())
19708 return SDValue();
19709 StoreMask = StoreMask.getOperand(i: 0);
19710 }
19711
19712 // Base = StoreAddr + (NumElem - 1) * ElemWidthByte
19713 SDLoc DL(N);
19714 MVT XLenVT = Subtarget.getXLenVT();
19715 SDValue NumElem = VPStore->getVectorLength();
19716 uint64_t ElemWidthByte = VPReverse.getValueType().getScalarSizeInBits() / 8;
19717
19718 SDValue Temp1 = DAG.getNode(Opcode: ISD::SUB, DL, VT: XLenVT, N1: NumElem,
19719 N2: DAG.getConstant(Val: 1, DL, VT: XLenVT));
19720 SDValue Temp2 = DAG.getNode(Opcode: ISD::MUL, DL, VT: XLenVT, N1: Temp1,
19721 N2: DAG.getConstant(Val: ElemWidthByte, DL, VT: XLenVT));
19722 SDValue Base =
19723 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: VPStore->getBasePtr(), N2: Temp2);
19724 SDValue Stride = DAG.getSignedConstant(Val: -ElemWidthByte, DL, VT: XLenVT);
19725
19726 MachineFunction &MF = DAG.getMachineFunction();
19727 MachinePointerInfo PtrInfo(VPStore->getAddressSpace());
19728 MachineMemOperand *MMO = MF.getMachineMemOperand(
19729 PtrInfo, F: VPStore->getMemOperand()->getFlags(),
19730 Size: LocationSize::beforeOrAfterPointer(), BaseAlignment: VPStore->getAlign());
19731
19732 return DAG.getStridedStoreVP(
19733 Chain: VPStore->getChain(), DL, Val: VPReverse.getOperand(i: 0), Ptr: Base,
19734 Offset: VPStore->getOffset(), Stride, Mask: StoreMask, EVL: VPStore->getVectorLength(),
19735 MemVT: VPStore->getMemoryVT(), MMO, AM: VPStore->getAddressingMode(),
19736 IsTruncating: VPStore->isTruncatingStore(), IsCompressing: VPStore->isCompressingStore());
19737}
19738
19739// Peephole avgceil pattern.
19740// %1 = zext <N x i8> %a to <N x i32>
19741// %2 = zext <N x i8> %b to <N x i32>
19742// %3 = add nuw nsw <N x i32> %1, splat (i32 1)
19743// %4 = add nuw nsw <N x i32> %3, %2
19744// %5 = lshr <N x i32> %4, splat (i32 1)
19745// %6 = trunc <N x i32> %5 to <N x i8>
19746static SDValue performVP_TRUNCATECombine(SDNode *N, SelectionDAG &DAG,
19747 const RISCVSubtarget &Subtarget) {
19748 EVT VT = N->getValueType(ResNo: 0);
19749
19750 // Ignore fixed vectors.
19751 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
19752 if (!VT.isScalableVector() || !TLI.isTypeLegal(VT))
19753 return SDValue();
19754
19755 SDValue In = N->getOperand(Num: 0);
19756 SDValue Mask = N->getOperand(Num: 1);
19757 SDValue VL = N->getOperand(Num: 2);
19758
19759 // Input should be a vp_srl with same mask and VL.
19760 if (In.getOpcode() != ISD::VP_SRL || In.getOperand(i: 2) != Mask ||
19761 In.getOperand(i: 3) != VL)
19762 return SDValue();
19763
19764 // Shift amount should be 1.
19765 if (!isOneOrOneSplat(V: In.getOperand(i: 1)))
19766 return SDValue();
19767
19768 // Shifted value should be a vp_add with same mask and VL.
19769 SDValue LHS = In.getOperand(i: 0);
19770 if (LHS.getOpcode() != ISD::VP_ADD || LHS.getOperand(i: 2) != Mask ||
19771 LHS.getOperand(i: 3) != VL)
19772 return SDValue();
19773
19774 SDValue Operands[3];
19775
19776 // Matches another VP_ADD with same VL and Mask.
19777 auto FindAdd = [&](SDValue V, SDValue Other) {
19778 if (V.getOpcode() != ISD::VP_ADD || V.getOperand(i: 2) != Mask ||
19779 V.getOperand(i: 3) != VL)
19780 return false;
19781
19782 Operands[0] = Other;
19783 Operands[1] = V.getOperand(i: 1);
19784 Operands[2] = V.getOperand(i: 0);
19785 return true;
19786 };
19787
19788 // We need to find another VP_ADD in one of the operands.
19789 SDValue LHS0 = LHS.getOperand(i: 0);
19790 SDValue LHS1 = LHS.getOperand(i: 1);
19791 if (!FindAdd(LHS0, LHS1) && !FindAdd(LHS1, LHS0))
19792 return SDValue();
19793
19794 // Now we have three operands of two additions. Check that one of them is a
19795 // constant vector with ones.
19796 auto I = llvm::find_if(Range&: Operands,
19797 P: [](const SDValue &Op) { return isOneOrOneSplat(V: Op); });
19798 if (I == std::end(arr&: Operands))
19799 return SDValue();
19800 // We found a vector with ones, move if it to the end of the Operands array.
19801 std::swap(a&: *I, b&: Operands[2]);
19802
19803 // Make sure the other 2 operands can be promoted from the result type.
19804 for (SDValue Op : drop_end(RangeOrContainer&: Operands)) {
19805 if (Op.getOpcode() != ISD::VP_ZERO_EXTEND || Op.getOperand(i: 1) != Mask ||
19806 Op.getOperand(i: 2) != VL)
19807 return SDValue();
19808 // Input must be the same size or smaller than our result.
19809 if (Op.getOperand(i: 0).getScalarValueSizeInBits() > VT.getScalarSizeInBits())
19810 return SDValue();
19811 }
19812
19813 // Pattern is detected.
19814 // Rebuild the zero extends in case the inputs are smaller than our result.
19815 SDValue NewOp0 = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: SDLoc(Operands[0]), VT,
19816 N1: Operands[0].getOperand(i: 0), N2: Mask, N3: VL);
19817 SDValue NewOp1 = DAG.getNode(Opcode: ISD::VP_ZERO_EXTEND, DL: SDLoc(Operands[1]), VT,
19818 N1: Operands[1].getOperand(i: 0), N2: Mask, N3: VL);
19819 // Build a AVGCEILU_VL which will be selected as a VAADDU with RNU rounding
19820 // mode.
19821 SDLoc DL(N);
19822 return DAG.getNode(Opcode: RISCVISD::AVGCEILU_VL, DL, VT,
19823 Ops: {NewOp0, NewOp1, DAG.getUNDEF(VT), Mask, VL});
19824}
19825
19826// Convert from one FMA opcode to another based on whether we are negating the
19827// multiply result and/or the accumulator.
19828// NOTE: Only supports RVV operations with VL.
19829static unsigned negateFMAOpcode(unsigned Opcode, bool NegMul, bool NegAcc) {
19830 // Negating the multiply result changes ADD<->SUB and toggles 'N'.
19831 if (NegMul) {
19832 // clang-format off
19833 switch (Opcode) {
19834 default: llvm_unreachable("Unexpected opcode");
19835 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
19836 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
19837 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
19838 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
19839 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
19840 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
19841 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
19842 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
19843 }
19844 // clang-format on
19845 }
19846
19847 // Negating the accumulator changes ADD<->SUB.
19848 if (NegAcc) {
19849 // clang-format off
19850 switch (Opcode) {
19851 default: llvm_unreachable("Unexpected opcode");
19852 case RISCVISD::VFMADD_VL: Opcode = RISCVISD::VFMSUB_VL; break;
19853 case RISCVISD::VFMSUB_VL: Opcode = RISCVISD::VFMADD_VL; break;
19854 case RISCVISD::VFNMADD_VL: Opcode = RISCVISD::VFNMSUB_VL; break;
19855 case RISCVISD::VFNMSUB_VL: Opcode = RISCVISD::VFNMADD_VL; break;
19856 case RISCVISD::STRICT_VFMADD_VL: Opcode = RISCVISD::STRICT_VFMSUB_VL; break;
19857 case RISCVISD::STRICT_VFMSUB_VL: Opcode = RISCVISD::STRICT_VFMADD_VL; break;
19858 case RISCVISD::STRICT_VFNMADD_VL: Opcode = RISCVISD::STRICT_VFNMSUB_VL; break;
19859 case RISCVISD::STRICT_VFNMSUB_VL: Opcode = RISCVISD::STRICT_VFNMADD_VL; break;
19860 }
19861 // clang-format on
19862 }
19863
19864 return Opcode;
19865}
19866
19867static SDValue combineVFMADD_VLWithVFNEG_VL(SDNode *N, SelectionDAG &DAG) {
19868 // Fold FNEG_VL into FMA opcodes.
19869 // The first operand of strict-fp is chain.
19870 bool IsStrict =
19871 DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(Opcode: N->getOpcode());
19872 unsigned Offset = IsStrict ? 1 : 0;
19873 SDValue A = N->getOperand(Num: 0 + Offset);
19874 SDValue B = N->getOperand(Num: 1 + Offset);
19875 SDValue C = N->getOperand(Num: 2 + Offset);
19876 SDValue Mask = N->getOperand(Num: 3 + Offset);
19877 SDValue VL = N->getOperand(Num: 4 + Offset);
19878
19879 auto invertIfNegative = [&Mask, &VL](SDValue &V) {
19880 if (V.getOpcode() == RISCVISD::FNEG_VL && V.getOperand(i: 1) == Mask &&
19881 V.getOperand(i: 2) == VL) {
19882 // Return the negated input.
19883 V = V.getOperand(i: 0);
19884 return true;
19885 }
19886
19887 return false;
19888 };
19889
19890 bool NegA = invertIfNegative(A);
19891 bool NegB = invertIfNegative(B);
19892 bool NegC = invertIfNegative(C);
19893
19894 // If no operands are negated, we're done.
19895 if (!NegA && !NegB && !NegC)
19896 return SDValue();
19897
19898 unsigned NewOpcode = negateFMAOpcode(Opcode: N->getOpcode(), NegMul: NegA != NegB, NegAcc: NegC);
19899 if (IsStrict)
19900 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(N), VTList: N->getVTList(),
19901 Ops: {N->getOperand(Num: 0), A, B, C, Mask, VL});
19902 return DAG.getNode(Opcode: NewOpcode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: A, N2: B, N3: C, N4: Mask,
19903 N5: VL);
19904}
19905
19906static SDValue performVFMADD_VLCombine(SDNode *N,
19907 TargetLowering::DAGCombinerInfo &DCI,
19908 const RISCVSubtarget &Subtarget) {
19909 SelectionDAG &DAG = DCI.DAG;
19910
19911 if (SDValue V = combineVFMADD_VLWithVFNEG_VL(N, DAG))
19912 return V;
19913
19914 // FIXME: Ignore strict opcodes for now.
19915 if (DAG.getSelectionDAGInfo().isTargetStrictFPOpcode(Opcode: N->getOpcode()))
19916 return SDValue();
19917
19918 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
19919}
19920
19921static SDValue performSRACombine(SDNode *N, SelectionDAG &DAG,
19922 const RISCVSubtarget &Subtarget) {
19923 assert(N->getOpcode() == ISD::SRA && "Unexpected opcode");
19924
19925 EVT VT = N->getValueType(ResNo: 0);
19926
19927 if (VT != Subtarget.getXLenVT())
19928 return SDValue();
19929
19930 if (!isa<ConstantSDNode>(Val: N->getOperand(Num: 1)))
19931 return SDValue();
19932 uint64_t ShAmt = N->getConstantOperandVal(Num: 1);
19933
19934 SDValue N0 = N->getOperand(Num: 0);
19935
19936 // Combine (sra (sext_inreg (shl X, C1), iX), C2) ->
19937 // (sra (shl X, C1+(XLen-iX)), C2+(XLen-iX)) so it gets selected as SLLI+SRAI.
19938 if (N0.getOpcode() == ISD::SIGN_EXTEND_INREG && N0.hasOneUse()) {
19939 unsigned ExtSize =
19940 cast<VTSDNode>(Val: N0.getOperand(i: 1))->getVT().getSizeInBits();
19941 if (ShAmt < ExtSize && N0.getOperand(i: 0).getOpcode() == ISD::SHL &&
19942 N0.getOperand(i: 0).hasOneUse() &&
19943 isa<ConstantSDNode>(Val: N0.getOperand(i: 0).getOperand(i: 1))) {
19944 uint64_t LShAmt = N0.getOperand(i: 0).getConstantOperandVal(i: 1);
19945 if (LShAmt < ExtSize) {
19946 unsigned Size = VT.getSizeInBits();
19947 SDLoc ShlDL(N0.getOperand(i: 0));
19948 SDValue Shl =
19949 DAG.getNode(Opcode: ISD::SHL, DL: ShlDL, VT, N1: N0.getOperand(i: 0).getOperand(i: 0),
19950 N2: DAG.getConstant(Val: LShAmt + (Size - ExtSize), DL: ShlDL, VT));
19951 SDLoc DL(N);
19952 return DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: Shl,
19953 N2: DAG.getConstant(Val: ShAmt + (Size - ExtSize), DL, VT));
19954 }
19955 }
19956 }
19957
19958 if (ShAmt > 32 || VT != MVT::i64)
19959 return SDValue();
19960
19961 // Combine (sra (shl X, 32), 32 - C) -> (shl (sext_inreg X, i32), C)
19962 // FIXME: Should this be a generic combine? There's a similar combine on X86.
19963 //
19964 // Also try these folds where an add or sub is in the middle.
19965 // (sra (add (shl X, 32), C1), 32 - C) -> (shl (sext_inreg (add X, C1), C)
19966 // (sra (sub C1, (shl X, 32)), 32 - C) -> (shl (sext_inreg (sub C1, X), C)
19967 SDValue Shl;
19968 ConstantSDNode *AddC = nullptr;
19969
19970 // We might have an ADD or SUB between the SRA and SHL.
19971 bool IsAdd = N0.getOpcode() == ISD::ADD;
19972 if ((IsAdd || N0.getOpcode() == ISD::SUB)) {
19973 // Other operand needs to be a constant we can modify.
19974 AddC = dyn_cast<ConstantSDNode>(Val: N0.getOperand(i: IsAdd ? 1 : 0));
19975 if (!AddC)
19976 return SDValue();
19977
19978 // AddC needs to have at least 32 trailing zeros.
19979 if (llvm::countr_zero(Val: AddC->getZExtValue()) < 32)
19980 return SDValue();
19981
19982 // All users should be a shift by constant less than or equal to 32. This
19983 // ensures we'll do this optimization for each of them to produce an
19984 // add/sub+sext_inreg they can all share.
19985 for (SDNode *U : N0->users()) {
19986 if (U->getOpcode() != ISD::SRA ||
19987 !isa<ConstantSDNode>(Val: U->getOperand(Num: 1)) ||
19988 U->getConstantOperandVal(Num: 1) > 32)
19989 return SDValue();
19990 }
19991
19992 Shl = N0.getOperand(i: IsAdd ? 0 : 1);
19993 } else {
19994 // Not an ADD or SUB.
19995 Shl = N0;
19996 }
19997
19998 // Look for a shift left by 32.
19999 if (Shl.getOpcode() != ISD::SHL || !isa<ConstantSDNode>(Val: Shl.getOperand(i: 1)) ||
20000 Shl.getConstantOperandVal(i: 1) != 32)
20001 return SDValue();
20002
20003 // We if we didn't look through an add/sub, then the shl should have one use.
20004 // If we did look through an add/sub, the sext_inreg we create is free so
20005 // we're only creating 2 new instructions. It's enough to only remove the
20006 // original sra+add/sub.
20007 if (!AddC && !Shl.hasOneUse())
20008 return SDValue();
20009
20010 SDLoc DL(N);
20011 SDValue In = Shl.getOperand(i: 0);
20012
20013 // If we looked through an ADD or SUB, we need to rebuild it with the shifted
20014 // constant.
20015 if (AddC) {
20016 SDValue ShiftedAddC =
20017 DAG.getConstant(Val: AddC->getZExtValue() >> 32, DL, VT: MVT::i64);
20018 if (IsAdd)
20019 In = DAG.getNode(Opcode: ISD::ADD, DL, VT: MVT::i64, N1: In, N2: ShiftedAddC);
20020 else
20021 In = DAG.getNode(Opcode: ISD::SUB, DL, VT: MVT::i64, N1: ShiftedAddC, N2: In);
20022 }
20023
20024 SDValue SExt = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: MVT::i64, N1: In,
20025 N2: DAG.getValueType(MVT::i32));
20026 if (ShAmt == 32)
20027 return SExt;
20028
20029 return DAG.getNode(
20030 Opcode: ISD::SHL, DL, VT: MVT::i64, N1: SExt,
20031 N2: DAG.getConstant(Val: 32 - ShAmt, DL, VT: MVT::i64));
20032}
20033
20034// Invert (and/or (set cc X, Y), (xor Z, 1)) to (or/and (set !cc X, Y)), Z) if
20035// the result is used as the condition of a br_cc or select_cc we can invert,
20036// inverting the setcc is free, and Z is 0/1. Caller will invert the
20037// br_cc/select_cc.
20038static SDValue tryDemorganOfBooleanCondition(SDValue Cond, SelectionDAG &DAG) {
20039 bool IsAnd = Cond.getOpcode() == ISD::AND;
20040 if (!IsAnd && Cond.getOpcode() != ISD::OR)
20041 return SDValue();
20042
20043 if (!Cond.hasOneUse())
20044 return SDValue();
20045
20046 SDValue Setcc = Cond.getOperand(i: 0);
20047 SDValue Xor = Cond.getOperand(i: 1);
20048 // Canonicalize setcc to LHS.
20049 if (Setcc.getOpcode() != ISD::SETCC)
20050 std::swap(a&: Setcc, b&: Xor);
20051 // LHS should be a setcc and RHS should be an xor.
20052 if (Setcc.getOpcode() != ISD::SETCC || !Setcc.hasOneUse() ||
20053 Xor.getOpcode() != ISD::XOR || !Xor.hasOneUse())
20054 return SDValue();
20055
20056 // If the condition is an And, SimplifyDemandedBits may have changed
20057 // (xor Z, 1) to (not Z).
20058 SDValue Xor1 = Xor.getOperand(i: 1);
20059 if (!isOneConstant(V: Xor1) && !(IsAnd && isAllOnesConstant(V: Xor1)))
20060 return SDValue();
20061
20062 EVT VT = Cond.getValueType();
20063 SDValue Xor0 = Xor.getOperand(i: 0);
20064
20065 // The LHS of the xor needs to be 0/1.
20066 APInt Mask = APInt::getBitsSetFrom(numBits: VT.getSizeInBits(), loBit: 1);
20067 if (!DAG.MaskedValueIsZero(Op: Xor0, Mask))
20068 return SDValue();
20069
20070 // We can only invert integer setccs.
20071 EVT SetCCOpVT = Setcc.getOperand(i: 0).getValueType();
20072 if (!SetCCOpVT.isScalarInteger())
20073 return SDValue();
20074
20075 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Setcc.getOperand(i: 2))->get();
20076 if (ISD::isIntEqualitySetCC(Code: CCVal)) {
20077 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: SetCCOpVT);
20078 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT, LHS: Setcc.getOperand(i: 0),
20079 RHS: Setcc.getOperand(i: 1), Cond: CCVal);
20080 } else if (CCVal == ISD::SETLT && isNullConstant(V: Setcc.getOperand(i: 0))) {
20081 // Invert (setlt 0, X) by converting to (setlt X, 1).
20082 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT, LHS: Setcc.getOperand(i: 1),
20083 RHS: DAG.getConstant(Val: 1, DL: SDLoc(Setcc), VT), Cond: CCVal);
20084 } else if (CCVal == ISD::SETLT && isOneConstant(V: Setcc.getOperand(i: 1))) {
20085 // (setlt X, 1) by converting to (setlt 0, X).
20086 Setcc = DAG.getSetCC(DL: SDLoc(Setcc), VT,
20087 LHS: DAG.getConstant(Val: 0, DL: SDLoc(Setcc), VT),
20088 RHS: Setcc.getOperand(i: 0), Cond: CCVal);
20089 } else
20090 return SDValue();
20091
20092 unsigned Opc = IsAnd ? ISD::OR : ISD::AND;
20093 return DAG.getNode(Opcode: Opc, DL: SDLoc(Cond), VT, N1: Setcc, N2: Xor.getOperand(i: 0));
20094}
20095
20096// Perform common combines for BR_CC and SELECT_CC conditions.
20097static bool combine_CC(SDValue &LHS, SDValue &RHS, SDValue &CC, const SDLoc &DL,
20098 SelectionDAG &DAG, const RISCVSubtarget &Subtarget) {
20099 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
20100
20101 // As far as arithmetic right shift always saves the sign,
20102 // shift can be omitted.
20103 // Fold setlt (sra X, N), 0 -> setlt X, 0 and
20104 // setge (sra X, N), 0 -> setge X, 0
20105 if (isNullConstant(V: RHS) && (CCVal == ISD::SETGE || CCVal == ISD::SETLT) &&
20106 LHS.getOpcode() == ISD::SRA) {
20107 LHS = LHS.getOperand(i: 0);
20108 return true;
20109 }
20110
20111 if (!ISD::isIntEqualitySetCC(Code: CCVal))
20112 return false;
20113
20114 // Fold ((setlt X, Y), 0, ne) -> (X, Y, lt)
20115 // Sometimes the setcc is introduced after br_cc/select_cc has been formed.
20116 if (LHS.getOpcode() == ISD::SETCC && isNullConstant(V: RHS) &&
20117 LHS.getOperand(i: 0).getValueType() == Subtarget.getXLenVT()) {
20118 // If we're looking for eq 0 instead of ne 0, we need to invert the
20119 // condition.
20120 bool Invert = CCVal == ISD::SETEQ;
20121 CCVal = cast<CondCodeSDNode>(Val: LHS.getOperand(i: 2))->get();
20122 if (Invert)
20123 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
20124
20125 RHS = LHS.getOperand(i: 1);
20126 LHS = LHS.getOperand(i: 0);
20127 translateSetCCForBranch(DL, LHS, RHS, CC&: CCVal, DAG, Subtarget);
20128
20129 CC = DAG.getCondCode(Cond: CCVal);
20130 return true;
20131 }
20132
20133 // If XOR is reused and has an immediate that will fit in XORI,
20134 // do not fold.
20135 auto isXorImmediate = [](const SDValue &Op) -> bool {
20136 if (const auto *XorCnst = dyn_cast<ConstantSDNode>(Val: Op))
20137 return isInt<12>(x: XorCnst->getSExtValue());
20138 return false;
20139 };
20140 // Fold (X(i1) ^ 1) == 0 -> X != 0
20141 auto singleBitOp = [&DAG](const SDValue &VarOp,
20142 const SDValue &ConstOp) -> bool {
20143 if (const auto *XorCnst = dyn_cast<ConstantSDNode>(Val: ConstOp)) {
20144 const APInt Mask = APInt::getBitsSetFrom(numBits: VarOp.getValueSizeInBits(), loBit: 1);
20145 return (XorCnst->getSExtValue() == 1) &&
20146 DAG.MaskedValueIsZero(Op: VarOp, Mask);
20147 }
20148 return false;
20149 };
20150 auto onlyUsedBySelectOrBR = [](const SDValue &Op) -> bool {
20151 for (const SDNode *UserNode : Op->users()) {
20152 const unsigned Opcode = UserNode->getOpcode();
20153 if (Opcode != RISCVISD::SELECT_CC && Opcode != RISCVISD::BR_CC)
20154 return false;
20155 }
20156 return true;
20157 };
20158 auto isFoldableXorEq = [isXorImmediate, singleBitOp, onlyUsedBySelectOrBR](
20159 const SDValue &LHS, const SDValue &RHS) -> bool {
20160 return LHS.getOpcode() == ISD::XOR && isNullConstant(V: RHS) &&
20161 (!isXorImmediate(LHS.getOperand(i: 1)) ||
20162 singleBitOp(LHS.getOperand(i: 0), LHS.getOperand(i: 1)) ||
20163 onlyUsedBySelectOrBR(LHS));
20164 };
20165 // Fold ((xor X, Y), 0, eq/ne) -> (X, Y, eq/ne)
20166 if (isFoldableXorEq(LHS, RHS)) {
20167 RHS = LHS.getOperand(i: 1);
20168 LHS = LHS.getOperand(i: 0);
20169 return true;
20170 }
20171 // Fold ((sext (xor X, C)), 0, eq/ne) -> ((sext(X), C, eq/ne)
20172 if (LHS.getOpcode() == ISD::SIGN_EXTEND_INREG) {
20173 const SDValue LHS0 = LHS.getOperand(i: 0);
20174 if (isFoldableXorEq(LHS0, RHS) && isa<ConstantSDNode>(Val: LHS0.getOperand(i: 1))) {
20175 // SEXT(XOR(X, Y)) -> XOR(SEXT(X), SEXT(Y)))
20176 RHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: LHS.getValueType(),
20177 N1: LHS0.getOperand(i: 1), N2: LHS.getOperand(i: 1));
20178 LHS = DAG.getNode(Opcode: ISD::SIGN_EXTEND_INREG, DL, VT: LHS.getValueType(),
20179 N1: LHS0.getOperand(i: 0), N2: LHS.getOperand(i: 1));
20180 return true;
20181 }
20182 }
20183
20184 // Fold ((srl (and X, 1<<C), C), 0, eq/ne) -> ((shl X, XLen-1-C), 0, ge/lt)
20185 if (isNullConstant(V: RHS) && LHS.getOpcode() == ISD::SRL && LHS.hasOneUse() &&
20186 LHS.getOperand(i: 1).getOpcode() == ISD::Constant) {
20187 SDValue LHS0 = LHS.getOperand(i: 0);
20188 if (LHS0.getOpcode() == ISD::AND &&
20189 LHS0.getOperand(i: 1).getOpcode() == ISD::Constant) {
20190 uint64_t Mask = LHS0.getConstantOperandVal(i: 1);
20191 uint64_t ShAmt = LHS.getConstantOperandVal(i: 1);
20192 if (isPowerOf2_64(Value: Mask) && Log2_64(Value: Mask) == ShAmt) {
20193 // XAndesPerf supports branch on test bit.
20194 if (Subtarget.hasVendorXAndesPerf()) {
20195 LHS =
20196 DAG.getNode(Opcode: ISD::AND, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
20197 N2: DAG.getConstant(Val: Mask, DL, VT: LHS.getValueType()));
20198 return true;
20199 }
20200
20201 CCVal = CCVal == ISD::SETEQ ? ISD::SETGE : ISD::SETLT;
20202 CC = DAG.getCondCode(Cond: CCVal);
20203
20204 ShAmt = LHS.getValueSizeInBits() - 1 - ShAmt;
20205 LHS = LHS0.getOperand(i: 0);
20206 if (ShAmt != 0)
20207 LHS =
20208 DAG.getNode(Opcode: ISD::SHL, DL, VT: LHS.getValueType(), N1: LHS0.getOperand(i: 0),
20209 N2: DAG.getConstant(Val: ShAmt, DL, VT: LHS.getValueType()));
20210 return true;
20211 }
20212 }
20213 }
20214
20215 // (X, 1, setne) -> // (X, 0, seteq) if we can prove X is 0/1.
20216 // This can occur when legalizing some floating point comparisons.
20217 APInt Mask = APInt::getBitsSetFrom(numBits: LHS.getValueSizeInBits(), loBit: 1);
20218 if (isOneConstant(V: RHS) && DAG.MaskedValueIsZero(Op: LHS, Mask)) {
20219 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
20220 CC = DAG.getCondCode(Cond: CCVal);
20221 RHS = DAG.getConstant(Val: 0, DL, VT: LHS.getValueType());
20222 return true;
20223 }
20224
20225 if (isNullConstant(V: RHS)) {
20226 if (SDValue NewCond = tryDemorganOfBooleanCondition(Cond: LHS, DAG)) {
20227 CCVal = ISD::getSetCCInverse(Operation: CCVal, Type: LHS.getValueType());
20228 CC = DAG.getCondCode(Cond: CCVal);
20229 LHS = NewCond;
20230 return true;
20231 }
20232 }
20233
20234 return false;
20235}
20236
20237// Fold
20238// (select C, (add Y, X), Y) -> (add Y, (select C, X, 0)).
20239// (select C, (sub Y, X), Y) -> (sub Y, (select C, X, 0)).
20240// (select C, (or Y, X), Y) -> (or Y, (select C, X, 0)).
20241// (select C, (xor Y, X), Y) -> (xor Y, (select C, X, 0)).
20242// (select C, (rotl Y, X), Y) -> (rotl Y, (select C, X, 0)).
20243// (select C, (rotr Y, X), Y) -> (rotr Y, (select C, X, 0)).
20244static SDValue tryFoldSelectIntoOp(SDNode *N, SelectionDAG &DAG,
20245 SDValue TrueVal, SDValue FalseVal,
20246 bool Swapped) {
20247 bool Commutative = true;
20248 unsigned Opc = TrueVal.getOpcode();
20249 switch (Opc) {
20250 default:
20251 return SDValue();
20252 case ISD::SHL:
20253 case ISD::SRA:
20254 case ISD::SRL:
20255 case ISD::SUB:
20256 case ISD::ROTL:
20257 case ISD::ROTR:
20258 Commutative = false;
20259 break;
20260 case ISD::ADD:
20261 case ISD::OR:
20262 case ISD::XOR:
20263 case ISD::UMIN:
20264 case ISD::UMAX:
20265 break;
20266 }
20267
20268 if (!TrueVal.hasOneUse())
20269 return SDValue();
20270
20271 unsigned OpToFold;
20272 if (FalseVal == TrueVal.getOperand(i: 0))
20273 OpToFold = 0;
20274 else if (Commutative && FalseVal == TrueVal.getOperand(i: 1))
20275 OpToFold = 1;
20276 else
20277 return SDValue();
20278
20279 EVT VT = N->getValueType(ResNo: 0);
20280 SDLoc DL(N);
20281 SDValue OtherOp = TrueVal.getOperand(i: 1 - OpToFold);
20282 EVT OtherOpVT = OtherOp.getValueType();
20283 SDValue IdentityOperand =
20284 DAG.getNeutralElement(Opcode: Opc, DL, VT: OtherOpVT, Flags: N->getFlags());
20285 if (!Commutative)
20286 IdentityOperand = DAG.getConstant(Val: 0, DL, VT: OtherOpVT);
20287 assert(IdentityOperand && "No identity operand!");
20288
20289 if (Swapped)
20290 std::swap(a&: OtherOp, b&: IdentityOperand);
20291 SDValue NewSel =
20292 DAG.getSelect(DL, VT: OtherOpVT, Cond: N->getOperand(Num: 0), LHS: OtherOp, RHS: IdentityOperand);
20293 return DAG.getNode(Opcode: TrueVal.getOpcode(), DL, VT, N1: FalseVal, N2: NewSel);
20294}
20295
20296// This tries to get rid of `select` and `icmp` that are being used to handle
20297// `Targets` that do not support `cttz(0)`/`ctlz(0)`.
20298static SDValue foldSelectOfCTTZOrCTLZ(SDNode *N, SelectionDAG &DAG) {
20299 SDValue Cond = N->getOperand(Num: 0);
20300
20301 // This represents either CTTZ or CTLZ instruction.
20302 SDValue CountZeroes;
20303
20304 SDValue ValOnZero;
20305
20306 if (Cond.getOpcode() != ISD::SETCC)
20307 return SDValue();
20308
20309 if (!isNullConstant(V: Cond->getOperand(Num: 1)))
20310 return SDValue();
20311
20312 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond->getOperand(Num: 2))->get();
20313 if (CCVal == ISD::CondCode::SETEQ) {
20314 CountZeroes = N->getOperand(Num: 2);
20315 ValOnZero = N->getOperand(Num: 1);
20316 } else if (CCVal == ISD::CondCode::SETNE) {
20317 CountZeroes = N->getOperand(Num: 1);
20318 ValOnZero = N->getOperand(Num: 2);
20319 } else {
20320 return SDValue();
20321 }
20322
20323 if (CountZeroes.getOpcode() == ISD::TRUNCATE ||
20324 CountZeroes.getOpcode() == ISD::ZERO_EXTEND)
20325 CountZeroes = CountZeroes.getOperand(i: 0);
20326
20327 if (CountZeroes.getOpcode() != ISD::CTTZ &&
20328 CountZeroes.getOpcode() != ISD::CTTZ_ZERO_UNDEF &&
20329 CountZeroes.getOpcode() != ISD::CTLZ &&
20330 CountZeroes.getOpcode() != ISD::CTLZ_ZERO_UNDEF)
20331 return SDValue();
20332
20333 if (!isNullConstant(V: ValOnZero))
20334 return SDValue();
20335
20336 SDValue CountZeroesArgument = CountZeroes->getOperand(Num: 0);
20337 if (Cond->getOperand(Num: 0) != CountZeroesArgument)
20338 return SDValue();
20339
20340 unsigned BitWidth = CountZeroes.getValueSizeInBits();
20341 if (!isPowerOf2_32(Value: BitWidth))
20342 return SDValue();
20343
20344 if (CountZeroes.getOpcode() == ISD::CTTZ_ZERO_UNDEF) {
20345 CountZeroes = DAG.getNode(Opcode: ISD::CTTZ, DL: SDLoc(CountZeroes),
20346 VT: CountZeroes.getValueType(), Operand: CountZeroesArgument);
20347 } else if (CountZeroes.getOpcode() == ISD::CTLZ_ZERO_UNDEF) {
20348 CountZeroes = DAG.getNode(Opcode: ISD::CTLZ, DL: SDLoc(CountZeroes),
20349 VT: CountZeroes.getValueType(), Operand: CountZeroesArgument);
20350 }
20351
20352 SDValue BitWidthMinusOne =
20353 DAG.getConstant(Val: BitWidth - 1, DL: SDLoc(N), VT: CountZeroes.getValueType());
20354
20355 auto AndNode = DAG.getNode(Opcode: ISD::AND, DL: SDLoc(N), VT: CountZeroes.getValueType(),
20356 N1: CountZeroes, N2: BitWidthMinusOne);
20357 return DAG.getZExtOrTrunc(Op: AndNode, DL: SDLoc(N), VT: N->getValueType(ResNo: 0));
20358}
20359
20360static SDValue useInversedSetcc(SDNode *N, SelectionDAG &DAG,
20361 const RISCVSubtarget &Subtarget) {
20362 SDValue Cond = N->getOperand(Num: 0);
20363 SDValue True = N->getOperand(Num: 1);
20364 SDValue False = N->getOperand(Num: 2);
20365 SDLoc DL(N);
20366 EVT VT = N->getValueType(ResNo: 0);
20367 EVT CondVT = Cond.getValueType();
20368
20369 if (Cond.getOpcode() != ISD::SETCC || !Cond.hasOneUse())
20370 return SDValue();
20371
20372 // Replace (setcc eq (and x, C)) with (setcc ne (and x, C))) to generate
20373 // BEXTI, where C is power of 2.
20374 if (Subtarget.hasBEXTILike() && VT.isScalarInteger() &&
20375 (Subtarget.hasCZEROLike() || Subtarget.hasVendorXTHeadCondMov())) {
20376 SDValue LHS = Cond.getOperand(i: 0);
20377 SDValue RHS = Cond.getOperand(i: 1);
20378 ISD::CondCode CC = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
20379 if (CC == ISD::SETEQ && LHS.getOpcode() == ISD::AND &&
20380 isa<ConstantSDNode>(Val: LHS.getOperand(i: 1)) && isNullConstant(V: RHS)) {
20381 const APInt &MaskVal = LHS.getConstantOperandAPInt(i: 1);
20382 if (MaskVal.isPowerOf2() && !MaskVal.isSignedIntN(N: 12))
20383 return DAG.getSelect(DL, VT,
20384 Cond: DAG.getSetCC(DL, VT: CondVT, LHS, RHS, Cond: ISD::SETNE),
20385 LHS: False, RHS: True);
20386 }
20387 }
20388 return SDValue();
20389}
20390
20391static bool matchSelectAddSub(SDValue TrueVal, SDValue FalseVal, bool &SwapCC) {
20392 if (!TrueVal.hasOneUse() || !FalseVal.hasOneUse())
20393 return false;
20394
20395 SwapCC = false;
20396 if (TrueVal.getOpcode() == ISD::SUB && FalseVal.getOpcode() == ISD::ADD) {
20397 std::swap(a&: TrueVal, b&: FalseVal);
20398 SwapCC = true;
20399 }
20400
20401 if (TrueVal.getOpcode() != ISD::ADD || FalseVal.getOpcode() != ISD::SUB)
20402 return false;
20403
20404 SDValue A = FalseVal.getOperand(i: 0);
20405 SDValue B = FalseVal.getOperand(i: 1);
20406 // Add is commutative, so check both orders
20407 return ((TrueVal.getOperand(i: 0) == A && TrueVal.getOperand(i: 1) == B) ||
20408 (TrueVal.getOperand(i: 1) == A && TrueVal.getOperand(i: 0) == B));
20409}
20410
20411/// Convert vselect CC, (add a, b), (sub a, b) to add a, (vselect CC, -b, b).
20412/// This allows us match a vadd.vv fed by a masked vrsub, which reduces
20413/// register pressure over the add followed by masked vsub sequence.
20414static SDValue performVSELECTCombine(SDNode *N, SelectionDAG &DAG) {
20415 SDLoc DL(N);
20416 EVT VT = N->getValueType(ResNo: 0);
20417 SDValue CC = N->getOperand(Num: 0);
20418 SDValue TrueVal = N->getOperand(Num: 1);
20419 SDValue FalseVal = N->getOperand(Num: 2);
20420
20421 bool SwapCC;
20422 if (!matchSelectAddSub(TrueVal, FalseVal, SwapCC))
20423 return SDValue();
20424
20425 SDValue Sub = SwapCC ? TrueVal : FalseVal;
20426 SDValue A = Sub.getOperand(i: 0);
20427 SDValue B = Sub.getOperand(i: 1);
20428
20429 // Arrange the select such that we can match a masked
20430 // vrsub.vi to perform the conditional negate
20431 SDValue NegB = DAG.getNegative(Val: B, DL, VT);
20432 if (!SwapCC)
20433 CC = DAG.getLogicalNOT(DL, Val: CC, VT: CC->getValueType(ResNo: 0));
20434 SDValue NewB = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CC, N2: NegB, N3: B);
20435 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: NewB);
20436}
20437
20438static SDValue performSELECTCombine(SDNode *N, SelectionDAG &DAG,
20439 const RISCVSubtarget &Subtarget) {
20440 if (SDValue Folded = foldSelectOfCTTZOrCTLZ(N, DAG))
20441 return Folded;
20442
20443 if (SDValue V = useInversedSetcc(N, DAG, Subtarget))
20444 return V;
20445
20446 if (Subtarget.hasConditionalMoveFusion())
20447 return SDValue();
20448
20449 SDValue TrueVal = N->getOperand(Num: 1);
20450 SDValue FalseVal = N->getOperand(Num: 2);
20451 if (SDValue V = tryFoldSelectIntoOp(N, DAG, TrueVal, FalseVal, /*Swapped*/false))
20452 return V;
20453 return tryFoldSelectIntoOp(N, DAG, TrueVal: FalseVal, FalseVal: TrueVal, /*Swapped*/true);
20454}
20455
20456/// If we have a build_vector where each lane is binop X, C, where C
20457/// is a constant (but not necessarily the same constant on all lanes),
20458/// form binop (build_vector x1, x2, ...), (build_vector c1, c2, c3, ..).
20459/// We assume that materializing a constant build vector will be no more
20460/// expensive that performing O(n) binops.
20461static SDValue performBUILD_VECTORCombine(SDNode *N, SelectionDAG &DAG,
20462 const RISCVSubtarget &Subtarget,
20463 const RISCVTargetLowering &TLI) {
20464 SDLoc DL(N);
20465 EVT VT = N->getValueType(ResNo: 0);
20466
20467 assert(!VT.isScalableVector() && "unexpected build vector");
20468
20469 if (VT.getVectorNumElements() == 1)
20470 return SDValue();
20471
20472 const unsigned Opcode = N->op_begin()->getNode()->getOpcode();
20473 if (!TLI.isBinOp(Opcode))
20474 return SDValue();
20475
20476 if (!TLI.isOperationLegalOrCustom(Op: Opcode, VT) || !TLI.isTypeLegal(VT))
20477 return SDValue();
20478
20479 // This BUILD_VECTOR involves an implicit truncation, and sinking
20480 // truncates through binops is non-trivial.
20481 if (N->op_begin()->getValueType() != VT.getVectorElementType())
20482 return SDValue();
20483
20484 SmallVector<SDValue> LHSOps;
20485 SmallVector<SDValue> RHSOps;
20486 for (SDValue Op : N->ops()) {
20487 if (Op.isUndef()) {
20488 // We can't form a divide or remainder from undef.
20489 if (!DAG.isSafeToSpeculativelyExecute(Opcode))
20490 return SDValue();
20491
20492 LHSOps.push_back(Elt: Op);
20493 RHSOps.push_back(Elt: Op);
20494 continue;
20495 }
20496
20497 // TODO: We can handle operations which have an neutral rhs value
20498 // (e.g. x + 0, a * 1 or a << 0), but we then have to keep track
20499 // of profit in a more explicit manner.
20500 if (Op.getOpcode() != Opcode || !Op.hasOneUse())
20501 return SDValue();
20502
20503 LHSOps.push_back(Elt: Op.getOperand(i: 0));
20504 if (!isa<ConstantSDNode>(Val: Op.getOperand(i: 1)) &&
20505 !isa<ConstantFPSDNode>(Val: Op.getOperand(i: 1)))
20506 return SDValue();
20507 // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
20508 // have different LHS and RHS types.
20509 if (Op.getOperand(i: 0).getValueType() != Op.getOperand(i: 1).getValueType())
20510 return SDValue();
20511
20512 RHSOps.push_back(Elt: Op.getOperand(i: 1));
20513 }
20514
20515 return DAG.getNode(Opcode, DL, VT, N1: DAG.getBuildVector(VT, DL, Ops: LHSOps),
20516 N2: DAG.getBuildVector(VT, DL, Ops: RHSOps));
20517}
20518
20519static MVT getQDOTXResultType(MVT OpVT) {
20520 ElementCount OpEC = OpVT.getVectorElementCount();
20521 assert(OpEC.isKnownMultipleOf(4) && OpVT.getVectorElementType() == MVT::i8);
20522 return MVT::getVectorVT(VT: MVT::i32, EC: OpEC.divideCoefficientBy(RHS: 4));
20523}
20524
20525/// Given fixed length vectors A and B with equal element types, but possibly
20526/// different number of elements, return A + B where either A or B is zero
20527/// padded to the larger number of elements.
20528static SDValue getZeroPaddedAdd(const SDLoc &DL, SDValue A, SDValue B,
20529 SelectionDAG &DAG) {
20530 // NOTE: Manually doing the extract/add/insert scheme produces
20531 // significantly better codegen than the naive pad with zeros
20532 // and add scheme.
20533 EVT AVT = A.getValueType();
20534 EVT BVT = B.getValueType();
20535 assert(AVT.getVectorElementType() == BVT.getVectorElementType());
20536 if (AVT.getVectorMinNumElements() > BVT.getVectorMinNumElements()) {
20537 std::swap(a&: A, b&: B);
20538 std::swap(a&: AVT, b&: BVT);
20539 }
20540
20541 SDValue BPart = DAG.getExtractSubvector(DL, VT: AVT, Vec: B, Idx: 0);
20542 SDValue Res = DAG.getNode(Opcode: ISD::ADD, DL, VT: AVT, N1: A, N2: BPart);
20543 return DAG.getInsertSubvector(DL, Vec: B, SubVec: Res, Idx: 0);
20544}
20545
20546static SDValue foldReduceOperandViaVDOTA4(SDValue InVec, const SDLoc &DL,
20547 SelectionDAG &DAG,
20548 const RISCVSubtarget &Subtarget,
20549 const RISCVTargetLowering &TLI) {
20550 using namespace SDPatternMatch;
20551 // Note: We intentionally do not check the legality of the reduction type.
20552 // We want to handle the m4/m8 *src* types, and thus need to let illegal
20553 // intermediate types flow through here.
20554 if (InVec.getValueType().getVectorElementType() != MVT::i32 ||
20555 !InVec.getValueType().getVectorElementCount().isKnownMultipleOf(RHS: 4))
20556 return SDValue();
20557
20558 // Recurse through adds/disjoint ors (since generic dag canonicalizes to that
20559 // form).
20560 SDValue A, B;
20561 if (sd_match(N: InVec, P: m_AddLike(L: m_Value(N&: A), R: m_Value(N&: B)))) {
20562 SDValue AOpt = foldReduceOperandViaVDOTA4(InVec: A, DL, DAG, Subtarget, TLI);
20563 SDValue BOpt = foldReduceOperandViaVDOTA4(InVec: B, DL, DAG, Subtarget, TLI);
20564 if (AOpt || BOpt) {
20565 if (AOpt)
20566 A = AOpt;
20567 if (BOpt)
20568 B = BOpt;
20569 // From here, we're doing A + B with mixed types, implicitly zero
20570 // padded to the wider type. Note that we *don't* need the result
20571 // type to be the original VT, and in fact prefer narrower ones
20572 // if possible.
20573 return getZeroPaddedAdd(DL, A, B, DAG);
20574 }
20575 }
20576
20577 // zext a <--> partial_reduce_umla 0, a, 1
20578 // sext a <--> partial_reduce_smla 0, a, 1
20579 if (InVec.getOpcode() == ISD::ZERO_EXTEND ||
20580 InVec.getOpcode() == ISD::SIGN_EXTEND) {
20581 SDValue A = InVec.getOperand(i: 0);
20582 EVT OpVT = A.getValueType();
20583 if (OpVT.getVectorElementType() != MVT::i8 || !TLI.isTypeLegal(VT: OpVT))
20584 return SDValue();
20585
20586 MVT ResVT = getQDOTXResultType(OpVT: A.getSimpleValueType());
20587 SDValue B = DAG.getConstant(Val: 0x1, DL, VT: OpVT);
20588 bool IsSigned = InVec.getOpcode() == ISD::SIGN_EXTEND;
20589 unsigned Opc =
20590 IsSigned ? ISD::PARTIAL_REDUCE_SMLA : ISD::PARTIAL_REDUCE_UMLA;
20591 return DAG.getNode(Opcode: Opc, DL, VT: ResVT, Ops: {DAG.getConstant(Val: 0, DL, VT: ResVT), A, B});
20592 }
20593
20594 // mul (sext a, sext b) -> partial_reduce_smla 0, a, b
20595 // mul (zext a, zext b) -> partial_reduce_umla 0, a, b
20596 // mul (sext a, zext b) -> partial_reduce_ssmla 0, a, b
20597 // mul (zext a, sext b) -> partial_reduce_smla 0, b, a (swapped)
20598 if (!sd_match(N: InVec, P: m_Mul(L: m_Value(N&: A), R: m_Value(N&: B))))
20599 return SDValue();
20600
20601 if (!ISD::isExtOpcode(Opcode: A.getOpcode()))
20602 return SDValue();
20603
20604 EVT OpVT = A.getOperand(i: 0).getValueType();
20605 if (OpVT.getVectorElementType() != MVT::i8 ||
20606 OpVT != B.getOperand(i: 0).getValueType() ||
20607 !TLI.isTypeLegal(VT: A.getValueType()))
20608 return SDValue();
20609
20610 unsigned Opc;
20611 if (A.getOpcode() == ISD::SIGN_EXTEND && B.getOpcode() == ISD::SIGN_EXTEND)
20612 Opc = ISD::PARTIAL_REDUCE_SMLA;
20613 else if (A.getOpcode() == ISD::ZERO_EXTEND &&
20614 B.getOpcode() == ISD::ZERO_EXTEND)
20615 Opc = ISD::PARTIAL_REDUCE_UMLA;
20616 else if (A.getOpcode() == ISD::SIGN_EXTEND &&
20617 B.getOpcode() == ISD::ZERO_EXTEND)
20618 Opc = ISD::PARTIAL_REDUCE_SUMLA;
20619 else if (A.getOpcode() == ISD::ZERO_EXTEND &&
20620 B.getOpcode() == ISD::SIGN_EXTEND) {
20621 Opc = ISD::PARTIAL_REDUCE_SUMLA;
20622 std::swap(a&: A, b&: B);
20623 } else
20624 return SDValue();
20625
20626 MVT ResVT = getQDOTXResultType(OpVT: OpVT.getSimpleVT());
20627 return DAG.getNode(
20628 Opcode: Opc, DL, VT: ResVT,
20629 Ops: {DAG.getConstant(Val: 0, DL, VT: ResVT), A.getOperand(i: 0), B.getOperand(i: 0)});
20630}
20631
20632static SDValue performVECREDUCECombine(SDNode *N, SelectionDAG &DAG,
20633 const RISCVSubtarget &Subtarget,
20634 const RISCVTargetLowering &TLI) {
20635 if (!Subtarget.hasStdExtZvdot4a8i())
20636 return SDValue();
20637
20638 SDLoc DL(N);
20639 EVT VT = N->getValueType(ResNo: 0);
20640 SDValue InVec = N->getOperand(Num: 0);
20641 if (SDValue V = foldReduceOperandViaVDOTA4(InVec, DL, DAG, Subtarget, TLI))
20642 return DAG.getNode(Opcode: ISD::VECREDUCE_ADD, DL, VT, Operand: V);
20643 return SDValue();
20644}
20645
20646static SDValue performINSERT_VECTOR_ELTCombine(SDNode *N, SelectionDAG &DAG,
20647 const RISCVSubtarget &Subtarget,
20648 const RISCVTargetLowering &TLI) {
20649 SDValue InVec = N->getOperand(Num: 0);
20650 SDValue InVal = N->getOperand(Num: 1);
20651 SDValue EltNo = N->getOperand(Num: 2);
20652 SDLoc DL(N);
20653
20654 EVT VT = InVec.getValueType();
20655 if (VT.isScalableVector())
20656 return SDValue();
20657
20658 if (!InVec.hasOneUse())
20659 return SDValue();
20660
20661 // Given insert_vector_elt (binop a, VecC), (same_binop b, C2), Elt
20662 // move the insert_vector_elts into the arms of the binop. Note that
20663 // the new RHS must be a constant.
20664 const unsigned InVecOpcode = InVec->getOpcode();
20665 if (InVecOpcode == InVal->getOpcode() && TLI.isBinOp(Opcode: InVecOpcode) &&
20666 InVal.hasOneUse()) {
20667 SDValue InVecLHS = InVec->getOperand(Num: 0);
20668 SDValue InVecRHS = InVec->getOperand(Num: 1);
20669 SDValue InValLHS = InVal->getOperand(Num: 0);
20670 SDValue InValRHS = InVal->getOperand(Num: 1);
20671
20672 if (!ISD::isBuildVectorOfConstantSDNodes(N: InVecRHS.getNode()))
20673 return SDValue();
20674 if (!isa<ConstantSDNode>(Val: InValRHS) && !isa<ConstantFPSDNode>(Val: InValRHS))
20675 return SDValue();
20676 // FIXME: Return failure if the RHS type doesn't match the LHS. Shifts may
20677 // have different LHS and RHS types.
20678 if (InVec.getOperand(i: 0).getValueType() != InVec.getOperand(i: 1).getValueType())
20679 return SDValue();
20680 SDValue LHS = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT,
20681 N1: InVecLHS, N2: InValLHS, N3: EltNo);
20682 SDValue RHS = DAG.getNode(Opcode: ISD::INSERT_VECTOR_ELT, DL, VT,
20683 N1: InVecRHS, N2: InValRHS, N3: EltNo);
20684 return DAG.getNode(Opcode: InVecOpcode, DL, VT, N1: LHS, N2: RHS);
20685 }
20686
20687 // Given insert_vector_elt (concat_vectors ...), InVal, Elt
20688 // move the insert_vector_elt to the source operand of the concat_vector.
20689 if (InVec.getOpcode() != ISD::CONCAT_VECTORS)
20690 return SDValue();
20691
20692 auto *IndexC = dyn_cast<ConstantSDNode>(Val&: EltNo);
20693 if (!IndexC)
20694 return SDValue();
20695 unsigned Elt = IndexC->getZExtValue();
20696
20697 EVT ConcatVT = InVec.getOperand(i: 0).getValueType();
20698 if (ConcatVT.getVectorElementType() != InVal.getValueType())
20699 return SDValue();
20700 unsigned ConcatNumElts = ConcatVT.getVectorNumElements();
20701 unsigned NewIdx = Elt % ConcatNumElts;
20702
20703 unsigned ConcatOpIdx = Elt / ConcatNumElts;
20704 SDValue ConcatOp = InVec.getOperand(i: ConcatOpIdx);
20705 ConcatOp = DAG.getInsertVectorElt(DL, Vec: ConcatOp, Elt: InVal, Idx: NewIdx);
20706
20707 SmallVector<SDValue> ConcatOps(InVec->ops());
20708 ConcatOps[ConcatOpIdx] = ConcatOp;
20709 return DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT, Ops: ConcatOps);
20710}
20711
20712// If we're concatenating a series of vector loads like
20713// concat_vectors (load v4i8, p+0), (load v4i8, p+n), (load v4i8, p+n*2) ...
20714// Then we can turn this into a strided load by widening the vector elements
20715// vlse32 p, stride=n
20716static SDValue performCONCAT_VECTORSCombine(SDNode *N, SelectionDAG &DAG,
20717 const RISCVSubtarget &Subtarget,
20718 const RISCVTargetLowering &TLI) {
20719 SDLoc DL(N);
20720 EVT VT = N->getValueType(ResNo: 0);
20721
20722 // Only perform this combine on legal MVTs.
20723 if (!TLI.isTypeLegal(VT))
20724 return SDValue();
20725
20726 // TODO: Potentially extend this to scalable vectors
20727 if (VT.isScalableVector())
20728 return SDValue();
20729
20730 auto *BaseLd = dyn_cast<LoadSDNode>(Val: N->getOperand(Num: 0));
20731 if (!BaseLd || !BaseLd->isSimple() || !ISD::isNormalLoad(N: BaseLd) ||
20732 !SDValue(BaseLd, 0).hasOneUse())
20733 return SDValue();
20734
20735 EVT BaseLdVT = BaseLd->getValueType(ResNo: 0);
20736
20737 // Go through the loads and check that they're strided
20738 SmallVector<LoadSDNode *> Lds;
20739 Lds.push_back(Elt: BaseLd);
20740 Align Align = BaseLd->getAlign();
20741 for (SDValue Op : N->ops().drop_front()) {
20742 auto *Ld = dyn_cast<LoadSDNode>(Val&: Op);
20743 if (!Ld || !Ld->isSimple() || !Op.hasOneUse() ||
20744 Ld->getChain() != BaseLd->getChain() || !ISD::isNormalLoad(N: Ld) ||
20745 Ld->getValueType(ResNo: 0) != BaseLdVT)
20746 return SDValue();
20747
20748 Lds.push_back(Elt: Ld);
20749
20750 // The common alignment is the most restrictive (smallest) of all the loads
20751 Align = std::min(a: Align, b: Ld->getAlign());
20752 }
20753
20754 using PtrDiff = std::pair<std::variant<int64_t, SDValue>, bool>;
20755 auto GetPtrDiff = [&DAG](LoadSDNode *Ld1,
20756 LoadSDNode *Ld2) -> std::optional<PtrDiff> {
20757 // If the load ptrs can be decomposed into a common (Base + Index) with a
20758 // common constant stride, then return the constant stride.
20759 BaseIndexOffset BIO1 = BaseIndexOffset::match(N: Ld1, DAG);
20760 BaseIndexOffset BIO2 = BaseIndexOffset::match(N: Ld2, DAG);
20761 if (BIO1.equalBaseIndex(Other: BIO2, DAG))
20762 return {{BIO2.getOffset() - BIO1.getOffset(), false}};
20763
20764 // Otherwise try to match (add LastPtr, Stride) or (add NextPtr, Stride)
20765 SDValue P1 = Ld1->getBasePtr();
20766 SDValue P2 = Ld2->getBasePtr();
20767 if (P2.getOpcode() == ISD::ADD && P2.getOperand(i: 0) == P1)
20768 return {{P2.getOperand(i: 1), false}};
20769 if (P1.getOpcode() == ISD::ADD && P1.getOperand(i: 0) == P2)
20770 return {{P1.getOperand(i: 1), true}};
20771
20772 return std::nullopt;
20773 };
20774
20775 // Get the distance between the first and second loads
20776 auto BaseDiff = GetPtrDiff(Lds[0], Lds[1]);
20777 if (!BaseDiff)
20778 return SDValue();
20779
20780 // Check all the loads are the same distance apart
20781 for (auto *It = Lds.begin() + 1; It != Lds.end() - 1; It++)
20782 if (GetPtrDiff(*It, *std::next(x: It)) != BaseDiff)
20783 return SDValue();
20784
20785 // TODO: At this point, we've successfully matched a generalized gather
20786 // load. Maybe we should emit that, and then move the specialized
20787 // matchers above and below into a DAG combine?
20788
20789 // Get the widened scalar type, e.g. v4i8 -> i64
20790 unsigned WideScalarBitWidth =
20791 BaseLdVT.getScalarSizeInBits() * BaseLdVT.getVectorNumElements();
20792 MVT WideScalarVT = MVT::getIntegerVT(BitWidth: WideScalarBitWidth);
20793
20794 // Get the vector type for the strided load, e.g. 4 x v4i8 -> v4i64
20795 MVT WideVecVT = MVT::getVectorVT(VT: WideScalarVT, NumElements: N->getNumOperands());
20796 if (!TLI.isTypeLegal(VT: WideVecVT))
20797 return SDValue();
20798
20799 // Check that the operation is legal
20800 if (!TLI.isLegalStridedLoadStore(DataType: WideVecVT, Alignment: Align))
20801 return SDValue();
20802
20803 auto [StrideVariant, MustNegateStride] = *BaseDiff;
20804 SDValue Stride =
20805 std::holds_alternative<SDValue>(v: StrideVariant)
20806 ? std::get<SDValue>(v&: StrideVariant)
20807 : DAG.getSignedConstant(Val: std::get<int64_t>(v&: StrideVariant), DL,
20808 VT: Lds[0]->getOffset().getValueType());
20809 if (MustNegateStride)
20810 Stride = DAG.getNegative(Val: Stride, DL, VT: Stride.getValueType());
20811
20812 SDValue AllOneMask =
20813 DAG.getSplat(VT: WideVecVT.changeVectorElementType(EltVT: MVT::i1), DL,
20814 Op: DAG.getConstant(Val: 1, DL, VT: MVT::i1));
20815
20816 uint64_t MemSize;
20817 if (auto *ConstStride = dyn_cast<ConstantSDNode>(Val&: Stride);
20818 ConstStride && ConstStride->getSExtValue() >= 0)
20819 // total size = (elsize * n) + (stride - elsize) * (n-1)
20820 // = elsize + stride * (n-1)
20821 MemSize = WideScalarVT.getSizeInBits() +
20822 ConstStride->getSExtValue() * (N->getNumOperands() - 1);
20823 else
20824 // If Stride isn't constant, then we can't know how much it will load
20825 MemSize = MemoryLocation::UnknownSize;
20826
20827 MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand(
20828 PtrInfo: BaseLd->getPointerInfo(), F: BaseLd->getMemOperand()->getFlags(), Size: MemSize,
20829 BaseAlignment: Align);
20830
20831 SDValue StridedLoad = DAG.getStridedLoadVP(
20832 VT: WideVecVT, DL, Chain: BaseLd->getChain(), Ptr: BaseLd->getBasePtr(), Stride,
20833 Mask: AllOneMask,
20834 EVL: DAG.getConstant(Val: N->getNumOperands(), DL, VT: Subtarget.getXLenVT()), MMO);
20835
20836 for (SDValue Ld : N->ops())
20837 DAG.makeEquivalentMemoryOrdering(OldLoad: cast<LoadSDNode>(Val&: Ld), NewMemOp: StridedLoad);
20838
20839 return DAG.getBitcast(VT: VT.getSimpleVT(), V: StridedLoad);
20840}
20841
20842static SDValue performVECTOR_SHUFFLECombine(SDNode *N, SelectionDAG &DAG,
20843 const RISCVSubtarget &Subtarget,
20844 const RISCVTargetLowering &TLI) {
20845 SDLoc DL(N);
20846 EVT VT = N->getValueType(ResNo: 0);
20847 const unsigned ElementSize = VT.getScalarSizeInBits();
20848 const unsigned NumElts = VT.getVectorNumElements();
20849 SDValue V1 = N->getOperand(Num: 0);
20850 SDValue V2 = N->getOperand(Num: 1);
20851 ShuffleVectorSDNode *SVN = cast<ShuffleVectorSDNode>(Val: N);
20852 ArrayRef<int> Mask = SVN->getMask();
20853 MVT XLenVT = Subtarget.getXLenVT();
20854
20855 // Recognized a disguised select of add/sub.
20856 bool SwapCC;
20857 if (ShuffleVectorInst::isSelectMask(Mask, NumSrcElts: NumElts) &&
20858 matchSelectAddSub(TrueVal: V1, FalseVal: V2, SwapCC)) {
20859 SDValue Sub = SwapCC ? V1 : V2;
20860 SDValue A = Sub.getOperand(i: 0);
20861 SDValue B = Sub.getOperand(i: 1);
20862
20863 SmallVector<SDValue> MaskVals;
20864 for (int MaskIndex : Mask) {
20865 bool SelectMaskVal = (MaskIndex < (int)NumElts);
20866 MaskVals.push_back(Elt: DAG.getConstant(Val: SelectMaskVal, DL, VT: XLenVT));
20867 }
20868 assert(MaskVals.size() == NumElts && "Unexpected select-like shuffle");
20869 EVT MaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1, NumElements: NumElts);
20870 SDValue CC = DAG.getBuildVector(VT: MaskVT, DL, Ops: MaskVals);
20871
20872 // Arrange the select such that we can match a masked
20873 // vrsub.vi to perform the conditional negate
20874 SDValue NegB = DAG.getNegative(Val: B, DL, VT);
20875 if (!SwapCC)
20876 CC = DAG.getLogicalNOT(DL, Val: CC, VT: CC->getValueType(ResNo: 0));
20877 SDValue NewB = DAG.getNode(Opcode: ISD::VSELECT, DL, VT, N1: CC, N2: NegB, N3: B);
20878 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: A, N2: NewB);
20879 }
20880
20881 if (SDValue V = compressShuffleOfShuffles(SVN, Subtarget, DAG))
20882 return V;
20883
20884 // Custom legalize <N x i128> or <N x i256> to <M x ELEN>. This runs
20885 // during the combine phase before type legalization, and relies on
20886 // DAGCombine not undoing the transform if isShuffleMaskLegal returns false
20887 // for the source mask.
20888 if (TLI.isTypeLegal(VT) || ElementSize <= Subtarget.getELen() ||
20889 !isPowerOf2_64(Value: ElementSize) || VT.getVectorNumElements() % 2 != 0 ||
20890 VT.isFloatingPoint() || TLI.isShuffleMaskLegal(M: Mask, VT))
20891 return SDValue();
20892
20893 SmallVector<int, 8> NewMask;
20894 narrowShuffleMaskElts(Scale: 2, Mask, ScaledMask&: NewMask);
20895
20896 LLVMContext &C = *DAG.getContext();
20897 EVT NewEltVT = EVT::getIntegerVT(Context&: C, BitWidth: ElementSize / 2);
20898 EVT NewVT = EVT::getVectorVT(Context&: C, VT: NewEltVT, NumElements: VT.getVectorNumElements() * 2);
20899 SDValue Res = DAG.getVectorShuffle(VT: NewVT, dl: DL, N1: DAG.getBitcast(VT: NewVT, V: V1),
20900 N2: DAG.getBitcast(VT: NewVT, V: V2), Mask: NewMask);
20901 return DAG.getBitcast(VT, V: Res);
20902}
20903
20904static SDValue combineToVWMACC(SDNode *N, SelectionDAG &DAG,
20905 const RISCVSubtarget &Subtarget) {
20906 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
20907
20908 if (N->getValueType(ResNo: 0).isFixedLengthVector())
20909 return SDValue();
20910
20911 SDValue Addend = N->getOperand(Num: 0);
20912 SDValue MulOp = N->getOperand(Num: 1);
20913
20914 if (N->getOpcode() == RISCVISD::ADD_VL) {
20915 SDValue AddPassthruOp = N->getOperand(Num: 2);
20916 if (!AddPassthruOp.isUndef())
20917 return SDValue();
20918 }
20919
20920 auto IsVWMulOpc = [](unsigned Opc) {
20921 switch (Opc) {
20922 case RISCVISD::VWMUL_VL:
20923 case RISCVISD::VWMULU_VL:
20924 case RISCVISD::VWMULSU_VL:
20925 return true;
20926 default:
20927 return false;
20928 }
20929 };
20930
20931 if (!IsVWMulOpc(MulOp.getOpcode()))
20932 std::swap(a&: Addend, b&: MulOp);
20933
20934 if (!IsVWMulOpc(MulOp.getOpcode()))
20935 return SDValue();
20936
20937 SDValue MulPassthruOp = MulOp.getOperand(i: 2);
20938
20939 if (!MulPassthruOp.isUndef())
20940 return SDValue();
20941
20942 auto [AddMask, AddVL] = [](SDNode *N, SelectionDAG &DAG,
20943 const RISCVSubtarget &Subtarget) {
20944 if (N->getOpcode() == ISD::ADD) {
20945 SDLoc DL(N);
20946 return getDefaultScalableVLOps(VecVT: N->getSimpleValueType(ResNo: 0), DL, DAG,
20947 Subtarget);
20948 }
20949 return std::make_pair(x: N->getOperand(Num: 3), y: N->getOperand(Num: 4));
20950 }(N, DAG, Subtarget);
20951
20952 SDValue MulMask = MulOp.getOperand(i: 3);
20953 SDValue MulVL = MulOp.getOperand(i: 4);
20954
20955 if (AddMask != MulMask || AddVL != MulVL)
20956 return SDValue();
20957
20958 const auto &TSInfo =
20959 static_cast<const RISCVSelectionDAGInfo &>(DAG.getSelectionDAGInfo());
20960 unsigned Opc = TSInfo.getMAccOpcode(MulOpcode: MulOp.getOpcode());
20961
20962 SDLoc DL(N);
20963 EVT VT = N->getValueType(ResNo: 0);
20964 SDValue Ops[] = {MulOp.getOperand(i: 0), MulOp.getOperand(i: 1), Addend, AddMask,
20965 AddVL};
20966 return DAG.getNode(Opcode: Opc, DL, VT, Ops);
20967}
20968
20969static SDValue combineVdota4Accum(SDNode *N, SelectionDAG &DAG,
20970 const RISCVSubtarget &Subtarget) {
20971
20972 assert(N->getOpcode() == RISCVISD::ADD_VL || N->getOpcode() == ISD::ADD);
20973
20974 if (!N->getValueType(ResNo: 0).isVector())
20975 return SDValue();
20976
20977 SDValue Addend = N->getOperand(Num: 0);
20978 SDValue DotOp = N->getOperand(Num: 1);
20979
20980 if (N->getOpcode() == RISCVISD::ADD_VL) {
20981 SDValue AddPassthruOp = N->getOperand(Num: 2);
20982 if (!AddPassthruOp.isUndef())
20983 return SDValue();
20984 }
20985
20986 auto IsVdota4Opc = [](unsigned Opc) {
20987 switch (Opc) {
20988 case RISCVISD::VDOTA4_VL:
20989 case RISCVISD::VDOTA4U_VL:
20990 case RISCVISD::VDOTA4SU_VL:
20991 return true;
20992 default:
20993 return false;
20994 }
20995 };
20996
20997 if (!IsVdota4Opc(DotOp.getOpcode()))
20998 std::swap(a&: Addend, b&: DotOp);
20999
21000 if (!IsVdota4Opc(DotOp.getOpcode()))
21001 return SDValue();
21002
21003 auto [AddMask, AddVL] = [](SDNode *N, SelectionDAG &DAG,
21004 const RISCVSubtarget &Subtarget) {
21005 if (N->getOpcode() == ISD::ADD) {
21006 SDLoc DL(N);
21007 return getDefaultScalableVLOps(VecVT: N->getSimpleValueType(ResNo: 0), DL, DAG,
21008 Subtarget);
21009 }
21010 return std::make_pair(x: N->getOperand(Num: 3), y: N->getOperand(Num: 4));
21011 }(N, DAG, Subtarget);
21012
21013 SDValue MulVL = DotOp.getOperand(i: 4);
21014 if (AddVL != MulVL)
21015 return SDValue();
21016
21017 if (AddMask.getOpcode() != RISCVISD::VMSET_VL ||
21018 AddMask.getOperand(i: 0) != MulVL)
21019 return SDValue();
21020
21021 SDValue AccumOp = DotOp.getOperand(i: 2);
21022 SDLoc DL(N);
21023 EVT VT = N->getValueType(ResNo: 0);
21024 Addend = DAG.getNode(Opcode: RISCVISD::ADD_VL, DL, VT, N1: Addend, N2: AccumOp,
21025 N3: DAG.getUNDEF(VT), N4: AddMask, N5: AddVL);
21026
21027 SDValue Ops[] = {DotOp.getOperand(i: 0), DotOp.getOperand(i: 1), Addend,
21028 DotOp.getOperand(i: 3), DotOp->getOperand(Num: 4)};
21029 return DAG.getNode(Opcode: DotOp->getOpcode(), DL, VT, Ops);
21030}
21031
21032static bool
21033legalizeScatterGatherIndexType(SDLoc DL, SDValue &Index,
21034 ISD::MemIndexType &IndexType,
21035 RISCVTargetLowering::DAGCombinerInfo &DCI) {
21036 if (!DCI.isBeforeLegalize())
21037 return false;
21038
21039 SelectionDAG &DAG = DCI.DAG;
21040 const MVT XLenVT =
21041 DAG.getMachineFunction().getSubtarget<RISCVSubtarget>().getXLenVT();
21042
21043 const EVT IndexVT = Index.getValueType();
21044
21045 // RISC-V indexed loads only support the "unsigned unscaled" addressing
21046 // mode, so anything else must be manually legalized.
21047 if (!isIndexTypeSigned(IndexType))
21048 return false;
21049
21050 if (IndexVT.getVectorElementType().bitsLT(VT: XLenVT)) {
21051 // Any index legalization should first promote to XLenVT, so we don't lose
21052 // bits when scaling. This may create an illegal index type so we let
21053 // LLVM's legalization take care of the splitting.
21054 // FIXME: LLVM can't split VP_GATHER or VP_SCATTER yet.
21055 Index = DAG.getNode(Opcode: ISD::SIGN_EXTEND, DL,
21056 VT: EVT::getVectorVT(Context&: *DAG.getContext(), VT: XLenVT,
21057 EC: IndexVT.getVectorElementCount()),
21058 Operand: Index);
21059 }
21060 IndexType = ISD::UNSIGNED_SCALED;
21061 return true;
21062}
21063
21064/// Match the index vector of a scatter or gather node as the shuffle mask
21065/// which performs the rearrangement if possible. Will only match if
21066/// all lanes are touched, and thus replacing the scatter or gather with
21067/// a unit strided access and shuffle is legal.
21068static bool matchIndexAsShuffle(EVT VT, SDValue Index, SDValue Mask,
21069 SmallVector<int> &ShuffleMask) {
21070 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
21071 return false;
21072 if (!ISD::isBuildVectorOfConstantSDNodes(N: Index.getNode()))
21073 return false;
21074
21075 const unsigned ElementSize = VT.getScalarStoreSize();
21076 const unsigned NumElems = VT.getVectorNumElements();
21077
21078 // Create the shuffle mask and check all bits active
21079 assert(ShuffleMask.empty());
21080 BitVector ActiveLanes(NumElems);
21081 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
21082 // TODO: We've found an active bit of UB, and could be
21083 // more aggressive here if desired.
21084 if (Index->getOperand(Num: i)->isUndef())
21085 return false;
21086 uint64_t C = Index->getConstantOperandVal(Num: i);
21087 if (C % ElementSize != 0)
21088 return false;
21089 C = C / ElementSize;
21090 if (C >= NumElems)
21091 return false;
21092 ShuffleMask.push_back(Elt: C);
21093 ActiveLanes.set(C);
21094 }
21095 return ActiveLanes.all();
21096}
21097
21098/// Match the index of a gather or scatter operation as an operation
21099/// with twice the element width and half the number of elements. This is
21100/// generally profitable (if legal) because these operations are linear
21101/// in VL, so even if we cause some extract VTYPE/VL toggles, we still
21102/// come out ahead.
21103static bool matchIndexAsWiderOp(EVT VT, SDValue Index, SDValue Mask,
21104 Align BaseAlign, const RISCVSubtarget &ST) {
21105 if (!ISD::isConstantSplatVectorAllOnes(N: Mask.getNode()))
21106 return false;
21107 if (!ISD::isBuildVectorOfConstantSDNodes(N: Index.getNode()))
21108 return false;
21109
21110 // Attempt a doubling. If we can use a element type 4x or 8x in
21111 // size, this will happen via multiply iterations of the transform.
21112 const unsigned NumElems = VT.getVectorNumElements();
21113 if (NumElems % 2 != 0)
21114 return false;
21115
21116 const unsigned ElementSize = VT.getScalarStoreSize();
21117 const unsigned WiderElementSize = ElementSize * 2;
21118 if (WiderElementSize > ST.getELen()/8)
21119 return false;
21120
21121 if (!ST.enableUnalignedVectorMem() && BaseAlign < WiderElementSize)
21122 return false;
21123
21124 for (unsigned i = 0; i < Index->getNumOperands(); i++) {
21125 // TODO: We've found an active bit of UB, and could be
21126 // more aggressive here if desired.
21127 if (Index->getOperand(Num: i)->isUndef())
21128 return false;
21129 // TODO: This offset check is too strict if we support fully
21130 // misaligned memory operations.
21131 uint64_t C = Index->getConstantOperandVal(Num: i);
21132 if (i % 2 == 0) {
21133 if (C % WiderElementSize != 0)
21134 return false;
21135 continue;
21136 }
21137 uint64_t Last = Index->getConstantOperandVal(Num: i-1);
21138 if (C != Last + ElementSize)
21139 return false;
21140 }
21141 return true;
21142}
21143
21144// trunc (sra sext (X), zext (Y)) -> sra (X, smin (Y, scalarsize(Y) - 1))
21145// This would be benefit for the cases where X and Y are both the same value
21146// type of low precision vectors. Since the truncate would be lowered into
21147// n-levels TRUNCATE_VECTOR_VL to satisfy RVV's SEW*2->SEW truncate
21148// restriction, such pattern would be expanded into a series of "vsetvli"
21149// and "vnsrl" instructions later to reach this point.
21150static SDValue combineTruncOfSraSext(SDNode *N, SelectionDAG &DAG) {
21151 SDValue Mask = N->getOperand(Num: 1);
21152 SDValue VL = N->getOperand(Num: 2);
21153
21154 bool IsVLMAX = isAllOnesConstant(V: VL) ||
21155 (isa<RegisterSDNode>(Val: VL) &&
21156 cast<RegisterSDNode>(Val&: VL)->getReg() == RISCV::X0);
21157 if (!IsVLMAX || Mask.getOpcode() != RISCVISD::VMSET_VL ||
21158 Mask.getOperand(i: 0) != VL)
21159 return SDValue();
21160
21161 auto IsTruncNode = [&](SDValue V) {
21162 return V.getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL &&
21163 V.getOperand(i: 1) == Mask && V.getOperand(i: 2) == VL;
21164 };
21165
21166 SDValue Op = N->getOperand(Num: 0);
21167
21168 // We need to first find the inner level of TRUNCATE_VECTOR_VL node
21169 // to distinguish such pattern.
21170 while (IsTruncNode(Op)) {
21171 if (!Op.hasOneUse())
21172 return SDValue();
21173 Op = Op.getOperand(i: 0);
21174 }
21175
21176 if (Op.getOpcode() != ISD::SRA || !Op.hasOneUse())
21177 return SDValue();
21178
21179 SDValue N0 = Op.getOperand(i: 0);
21180 SDValue N1 = Op.getOperand(i: 1);
21181 if (N0.getOpcode() != ISD::SIGN_EXTEND || !N0.hasOneUse() ||
21182 N1.getOpcode() != ISD::ZERO_EXTEND || !N1.hasOneUse())
21183 return SDValue();
21184
21185 SDValue N00 = N0.getOperand(i: 0);
21186 SDValue N10 = N1.getOperand(i: 0);
21187 if (!N00.getValueType().isVector() ||
21188 N00.getValueType() != N10.getValueType() ||
21189 N->getValueType(ResNo: 0) != N10.getValueType())
21190 return SDValue();
21191
21192 unsigned MaxShAmt = N10.getValueType().getScalarSizeInBits() - 1;
21193 SDValue SMin =
21194 DAG.getNode(Opcode: ISD::SMIN, DL: SDLoc(N1), VT: N->getValueType(ResNo: 0), N1: N10,
21195 N2: DAG.getConstant(Val: MaxShAmt, DL: SDLoc(N1), VT: N->getValueType(ResNo: 0)));
21196 return DAG.getNode(Opcode: ISD::SRA, DL: SDLoc(N), VT: N->getValueType(ResNo: 0), N1: N00, N2: SMin);
21197}
21198
21199// Combine (truncate_vector_vl (umin X, C)) -> (vnclipu_vl X) if C is the
21200// maximum value for the truncated type.
21201// Combine (truncate_vector_vl (smin (smax X, C2), C1)) -> (vnclip_vl X) if C1
21202// is the signed maximum value for the truncated type and C2 is the signed
21203// minimum value.
21204static SDValue combineTruncToVnclip(SDNode *N, SelectionDAG &DAG,
21205 const RISCVSubtarget &Subtarget) {
21206 assert(N->getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL);
21207
21208 MVT VT = N->getSimpleValueType(ResNo: 0);
21209
21210 SDValue Mask = N->getOperand(Num: 1);
21211 SDValue VL = N->getOperand(Num: 2);
21212
21213 auto MatchMinMax = [&VL, &Mask](SDValue V, unsigned Opc, unsigned OpcVL,
21214 APInt &SplatVal) {
21215 if (V.getOpcode() != Opc &&
21216 !(V.getOpcode() == OpcVL && V.getOperand(i: 2).isUndef() &&
21217 V.getOperand(i: 3) == Mask && V.getOperand(i: 4) == VL))
21218 return SDValue();
21219
21220 SDValue Op = V.getOperand(i: 1);
21221
21222 // Peek through conversion between fixed and scalable vectors.
21223 if (Op.getOpcode() == ISD::INSERT_SUBVECTOR && Op.getOperand(i: 0).isUndef() &&
21224 isNullConstant(V: Op.getOperand(i: 2)) &&
21225 Op.getOperand(i: 1).getValueType().isFixedLengthVector() &&
21226 Op.getOperand(i: 1).getOpcode() == ISD::EXTRACT_SUBVECTOR &&
21227 Op.getOperand(i: 1).getOperand(i: 0).getValueType() == Op.getValueType() &&
21228 isNullConstant(V: Op.getOperand(i: 1).getOperand(i: 1)))
21229 Op = Op.getOperand(i: 1).getOperand(i: 0);
21230
21231 if (ISD::isConstantSplatVector(N: Op.getNode(), SplatValue&: SplatVal))
21232 return V.getOperand(i: 0);
21233
21234 if (Op.getOpcode() == RISCVISD::VMV_V_X_VL && Op.getOperand(i: 0).isUndef() &&
21235 Op.getOperand(i: 2) == VL) {
21236 if (auto *Op1 = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1))) {
21237 SplatVal =
21238 Op1->getAPIntValue().sextOrTrunc(width: Op.getScalarValueSizeInBits());
21239 return V.getOperand(i: 0);
21240 }
21241 }
21242
21243 return SDValue();
21244 };
21245
21246 SDLoc DL(N);
21247
21248 auto DetectUSatPattern = [&](SDValue V) {
21249 APInt LoC, HiC;
21250
21251 // Simple case, V is a UMIN.
21252 if (SDValue UMinOp = MatchMinMax(V, ISD::UMIN, RISCVISD::UMIN_VL, HiC))
21253 if (HiC.isMask(numBits: VT.getScalarSizeInBits()))
21254 return UMinOp;
21255
21256 // If we have an SMAX that removes negative numbers first, then we can match
21257 // SMIN instead of UMIN.
21258 if (SDValue SMinOp = MatchMinMax(V, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21259 if (SDValue SMaxOp =
21260 MatchMinMax(SMinOp, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21261 if (LoC.isNonNegative() && HiC.isMask(numBits: VT.getScalarSizeInBits()))
21262 return SMinOp;
21263
21264 // If we have an SMIN before an SMAX and the SMAX constant is less than or
21265 // equal to the SMIN constant, we can use vnclipu if we insert a new SMAX
21266 // first.
21267 if (SDValue SMaxOp = MatchMinMax(V, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21268 if (SDValue SMinOp =
21269 MatchMinMax(SMaxOp, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21270 if (LoC.isNonNegative() && HiC.isMask(numBits: VT.getScalarSizeInBits()) &&
21271 HiC.uge(RHS: LoC))
21272 return DAG.getNode(Opcode: RISCVISD::SMAX_VL, DL, VT: V.getValueType(), N1: SMinOp,
21273 N2: V.getOperand(i: 1), N3: DAG.getUNDEF(VT: V.getValueType()),
21274 N4: Mask, N5: VL);
21275
21276 return SDValue();
21277 };
21278
21279 auto DetectSSatPattern = [&](SDValue V) {
21280 unsigned NumDstBits = VT.getScalarSizeInBits();
21281 unsigned NumSrcBits = V.getScalarValueSizeInBits();
21282 APInt SignedMax = APInt::getSignedMaxValue(numBits: NumDstBits).sext(width: NumSrcBits);
21283 APInt SignedMin = APInt::getSignedMinValue(numBits: NumDstBits).sext(width: NumSrcBits);
21284
21285 APInt HiC, LoC;
21286 if (SDValue SMinOp = MatchMinMax(V, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21287 if (SDValue SMaxOp =
21288 MatchMinMax(SMinOp, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21289 if (HiC == SignedMax && LoC == SignedMin)
21290 return SMaxOp;
21291
21292 if (SDValue SMaxOp = MatchMinMax(V, ISD::SMAX, RISCVISD::SMAX_VL, LoC))
21293 if (SDValue SMinOp =
21294 MatchMinMax(SMaxOp, ISD::SMIN, RISCVISD::SMIN_VL, HiC))
21295 if (HiC == SignedMax && LoC == SignedMin)
21296 return SMinOp;
21297
21298 return SDValue();
21299 };
21300
21301 SDValue Src = N->getOperand(Num: 0);
21302
21303 // Look through multiple layers of truncates.
21304 while (Src.getOpcode() == RISCVISD::TRUNCATE_VECTOR_VL &&
21305 Src.getOperand(i: 1) == Mask && Src.getOperand(i: 2) == VL &&
21306 Src.hasOneUse())
21307 Src = Src.getOperand(i: 0);
21308
21309 SDValue Val;
21310 unsigned ClipOpc;
21311 if ((Val = DetectUSatPattern(Src)))
21312 ClipOpc = RISCVISD::TRUNCATE_VECTOR_VL_USAT;
21313 else if ((Val = DetectSSatPattern(Src)))
21314 ClipOpc = RISCVISD::TRUNCATE_VECTOR_VL_SSAT;
21315 else
21316 return SDValue();
21317
21318 MVT ValVT = Val.getSimpleValueType();
21319
21320 do {
21321 MVT ValEltVT = MVT::getIntegerVT(BitWidth: ValVT.getScalarSizeInBits() / 2);
21322 ValVT = ValVT.changeVectorElementType(EltVT: ValEltVT);
21323 Val = DAG.getNode(Opcode: ClipOpc, DL, VT: ValVT, N1: Val, N2: Mask, N3: VL);
21324 } while (ValVT != VT);
21325
21326 return Val;
21327}
21328
21329// Convert
21330// (iX ctpop (bitcast (vXi1 A)))
21331// ->
21332// (zext (vcpop.m (nxvYi1 (insert_subvec (vXi1 A)))))
21333// and
21334// (iN reduce.add (zext (vXi1 A to vXiN))
21335// ->
21336// (zext (vcpop.m (nxvYi1 (insert_subvec (vXi1 A)))))
21337// FIXME: It's complicated to match all the variations of this after type
21338// legalization so we only handle the pre-type legalization pattern, but that
21339// requires the fixed vector type to be legal.
21340static SDValue combineToVCPOP(SDNode *N, SelectionDAG &DAG,
21341 const RISCVSubtarget &Subtarget) {
21342 unsigned Opc = N->getOpcode();
21343 assert((Opc == ISD::CTPOP || Opc == ISD::VECREDUCE_ADD) &&
21344 "Unexpected opcode");
21345 EVT VT = N->getValueType(ResNo: 0);
21346 if (!VT.isScalarInteger())
21347 return SDValue();
21348
21349 SDValue Src = N->getOperand(Num: 0);
21350
21351 if (Opc == ISD::CTPOP) {
21352 // Peek through zero_extend. It doesn't change the count.
21353 if (Src.getOpcode() == ISD::ZERO_EXTEND)
21354 Src = Src.getOperand(i: 0);
21355
21356 if (Src.getOpcode() != ISD::BITCAST)
21357 return SDValue();
21358 Src = Src.getOperand(i: 0);
21359 } else if (Opc == ISD::VECREDUCE_ADD) {
21360 if (Src.getOpcode() != ISD::ZERO_EXTEND)
21361 return SDValue();
21362 Src = Src.getOperand(i: 0);
21363 }
21364
21365 EVT SrcEVT = Src.getValueType();
21366 if (!SrcEVT.isSimple())
21367 return SDValue();
21368
21369 MVT SrcMVT = SrcEVT.getSimpleVT();
21370 // Make sure the input is an i1 vector.
21371 if (!SrcMVT.isVector() || SrcMVT.getVectorElementType() != MVT::i1)
21372 return SDValue();
21373
21374 const TargetLowering &TLI = DAG.getTargetLoweringInfo();
21375 if (!TLI.isTypeLegal(VT: SrcMVT))
21376 return SDValue();
21377
21378 // Check that destination type is large enough to hold result without
21379 // overflow.
21380 if (Opc == ISD::VECREDUCE_ADD) {
21381 unsigned EltSize = SrcMVT.getScalarSizeInBits();
21382 unsigned MinSize = SrcMVT.getSizeInBits().getKnownMinValue();
21383 unsigned VectorBitsMax = Subtarget.getRealMaxVLen();
21384 unsigned MaxVLMAX = SrcMVT.isFixedLengthVector()
21385 ? SrcMVT.getVectorNumElements()
21386 : RISCVTargetLowering::computeVLMAX(
21387 VectorBits: VectorBitsMax, EltSize, MinSize);
21388 if (VT.getFixedSizeInBits() < Log2_32(Value: MaxVLMAX) + 1)
21389 return SDValue();
21390 }
21391
21392 MVT ContainerVT = SrcMVT;
21393 if (SrcMVT.isFixedLengthVector()) {
21394 ContainerVT = getContainerForFixedLengthVector(DAG, VT: SrcMVT, Subtarget);
21395 Src = convertToScalableVector(VT: ContainerVT, V: Src, DAG, Subtarget);
21396 }
21397
21398 SDLoc DL(N);
21399 auto [Mask, VL] = getDefaultVLOps(VecVT: SrcMVT, ContainerVT, DL, DAG, Subtarget);
21400
21401 MVT XLenVT = Subtarget.getXLenVT();
21402 SDValue Pop = DAG.getNode(Opcode: RISCVISD::VCPOP_VL, DL, VT: XLenVT, N1: Src, N2: Mask, N3: VL);
21403 return DAG.getZExtOrTrunc(Op: Pop, DL, VT);
21404}
21405
21406static SDValue performSHLCombine(SDNode *N,
21407 TargetLowering::DAGCombinerInfo &DCI,
21408 const RISCVSubtarget &Subtarget) {
21409 // (shl (zext x), y) -> (vwsll x, y)
21410 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
21411 return V;
21412
21413 // (shl (sext x), C) -> (vwmulsu x, 1u << C)
21414 // (shl (zext x), C) -> (vwmulu x, 1u << C)
21415
21416 if (!DCI.isAfterLegalizeDAG())
21417 return SDValue();
21418
21419 SDValue LHS = N->getOperand(Num: 0);
21420 if (!LHS.hasOneUse())
21421 return SDValue();
21422 unsigned Opcode;
21423 switch (LHS.getOpcode()) {
21424 case ISD::SIGN_EXTEND:
21425 case RISCVISD::VSEXT_VL:
21426 Opcode = RISCVISD::VWMULSU_VL;
21427 break;
21428 case ISD::ZERO_EXTEND:
21429 case RISCVISD::VZEXT_VL:
21430 Opcode = RISCVISD::VWMULU_VL;
21431 break;
21432 default:
21433 return SDValue();
21434 }
21435
21436 SDValue RHS = N->getOperand(Num: 1);
21437 APInt ShAmt;
21438 uint64_t ShAmtInt;
21439 if (ISD::isConstantSplatVector(N: RHS.getNode(), SplatValue&: ShAmt))
21440 ShAmtInt = ShAmt.getZExtValue();
21441 else if (RHS.getOpcode() == RISCVISD::VMV_V_X_VL &&
21442 RHS.getOperand(i: 1).getOpcode() == ISD::Constant)
21443 ShAmtInt = RHS.getConstantOperandVal(i: 1);
21444 else
21445 return SDValue();
21446
21447 // Better foldings:
21448 // (shl (sext x), 1) -> (vwadd x, x)
21449 // (shl (zext x), 1) -> (vwaddu x, x)
21450 if (ShAmtInt <= 1)
21451 return SDValue();
21452
21453 SDValue NarrowOp = LHS.getOperand(i: 0);
21454 MVT NarrowVT = NarrowOp.getSimpleValueType();
21455 uint64_t NarrowBits = NarrowVT.getScalarSizeInBits();
21456 if (ShAmtInt >= NarrowBits)
21457 return SDValue();
21458 MVT VT = N->getSimpleValueType(ResNo: 0);
21459 if (NarrowBits * 2 != VT.getScalarSizeInBits())
21460 return SDValue();
21461
21462 SelectionDAG &DAG = DCI.DAG;
21463 SDLoc DL(N);
21464 SDValue Passthru, Mask, VL;
21465 switch (N->getOpcode()) {
21466 case ISD::SHL:
21467 Passthru = DAG.getUNDEF(VT);
21468 std::tie(args&: Mask, args&: VL) = getDefaultScalableVLOps(VecVT: VT, DL, DAG, Subtarget);
21469 break;
21470 case RISCVISD::SHL_VL:
21471 Passthru = N->getOperand(Num: 2);
21472 Mask = N->getOperand(Num: 3);
21473 VL = N->getOperand(Num: 4);
21474 break;
21475 default:
21476 llvm_unreachable("Expected SHL");
21477 }
21478 return DAG.getNode(Opcode, DL, VT, N1: NarrowOp,
21479 N2: DAG.getConstant(Val: 1ULL << ShAmtInt, DL: SDLoc(RHS), VT: NarrowVT),
21480 N3: Passthru, N4: Mask, N5: VL);
21481}
21482
21483SDValue RISCVTargetLowering::PerformDAGCombine(SDNode *N,
21484 DAGCombinerInfo &DCI) const {
21485 SelectionDAG &DAG = DCI.DAG;
21486 const MVT XLenVT = Subtarget.getXLenVT();
21487 SDLoc DL(N);
21488
21489 // Helper to call SimplifyDemandedBits on an operand of N where only some low
21490 // bits are demanded. N will be added to the Worklist if it was not deleted.
21491 // Caller should return SDValue(N, 0) if this returns true.
21492 auto SimplifyDemandedLowBitsHelper = [&](unsigned OpNo, unsigned LowBits) {
21493 SDValue Op = N->getOperand(Num: OpNo);
21494 APInt Mask = APInt::getLowBitsSet(numBits: Op.getValueSizeInBits(), loBitsSet: LowBits);
21495 if (!SimplifyDemandedBits(Op, DemandedBits: Mask, DCI))
21496 return false;
21497
21498 if (N->getOpcode() != ISD::DELETED_NODE)
21499 DCI.AddToWorklist(N);
21500 return true;
21501 };
21502
21503 switch (N->getOpcode()) {
21504 default:
21505 break;
21506 case RISCVISD::SplitF64: {
21507 SDValue Op0 = N->getOperand(Num: 0);
21508 // If the input to SplitF64 is just BuildPairF64 then the operation is
21509 // redundant. Instead, use BuildPairF64's operands directly.
21510 if (Op0->getOpcode() == RISCVISD::BuildPairF64)
21511 return DCI.CombineTo(N, Res0: Op0.getOperand(i: 0), Res1: Op0.getOperand(i: 1));
21512
21513 if (Op0->isUndef()) {
21514 SDValue Lo = DAG.getUNDEF(VT: MVT::i32);
21515 SDValue Hi = DAG.getUNDEF(VT: MVT::i32);
21516 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
21517 }
21518
21519 // It's cheaper to materialise two 32-bit integers than to load a double
21520 // from the constant pool and transfer it to integer registers through the
21521 // stack.
21522 if (ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
21523 APInt V = C->getValueAPF().bitcastToAPInt();
21524 SDValue Lo = DAG.getConstant(Val: V.trunc(width: 32), DL, VT: MVT::i32);
21525 SDValue Hi = DAG.getConstant(Val: V.lshr(shiftAmt: 32).trunc(width: 32), DL, VT: MVT::i32);
21526 return DCI.CombineTo(N, Res0: Lo, Res1: Hi);
21527 }
21528
21529 // This is a target-specific version of a DAGCombine performed in
21530 // DAGCombiner::visitBITCAST. It performs the equivalent of:
21531 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
21532 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
21533 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
21534 !Op0.getNode()->hasOneUse() || Subtarget.hasStdExtZdinx())
21535 break;
21536 SDValue NewSplitF64 =
21537 DAG.getNode(Opcode: RISCVISD::SplitF64, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
21538 N: Op0.getOperand(i: 0));
21539 SDValue Lo = NewSplitF64.getValue(R: 0);
21540 SDValue Hi = NewSplitF64.getValue(R: 1);
21541 APInt SignBit = APInt::getSignMask(BitWidth: 32);
21542 if (Op0.getOpcode() == ISD::FNEG) {
21543 SDValue NewHi = DAG.getNode(Opcode: ISD::XOR, DL, VT: MVT::i32, N1: Hi,
21544 N2: DAG.getConstant(Val: SignBit, DL, VT: MVT::i32));
21545 return DCI.CombineTo(N, Res0: Lo, Res1: NewHi);
21546 }
21547 assert(Op0.getOpcode() == ISD::FABS);
21548 SDValue NewHi = DAG.getNode(Opcode: ISD::AND, DL, VT: MVT::i32, N1: Hi,
21549 N2: DAG.getConstant(Val: ~SignBit, DL, VT: MVT::i32));
21550 return DCI.CombineTo(N, Res0: Lo, Res1: NewHi);
21551 }
21552 case RISCVISD::SLLW:
21553 case RISCVISD::SRAW:
21554 case RISCVISD::SRLW:
21555 case RISCVISD::RORW:
21556 case RISCVISD::ROLW: {
21557 // Only the lower 32 bits of LHS and lower 5 bits of RHS are read.
21558 if (SimplifyDemandedLowBitsHelper(0, 32) ||
21559 SimplifyDemandedLowBitsHelper(1, 5))
21560 return SDValue(N, 0);
21561
21562 break;
21563 }
21564 case RISCVISD::ABSW:
21565 case RISCVISD::CLSW:
21566 case RISCVISD::CLZW:
21567 case RISCVISD::CTZW: {
21568 // Only the lower 32 bits of the first operand are read
21569 if (SimplifyDemandedLowBitsHelper(0, 32))
21570 return SDValue(N, 0);
21571 break;
21572 }
21573 case RISCVISD::WMULSU: {
21574 // Convert to MULHSU if only the upper half is used.
21575 if (!N->hasAnyUseOfValue(Value: 0)) {
21576 SDValue Res = DAG.getNode(Opcode: RISCVISD::MULHSU, DL, VT: N->getValueType(ResNo: 1),
21577 N1: N->getOperand(Num: 0), N2: N->getOperand(Num: 1));
21578 return DCI.CombineTo(N, Res0: Res, Res1: Res);
21579 }
21580 break;
21581 }
21582 case RISCVISD::ADDD: {
21583 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
21584 "ADDD is only for RV32 with P extension");
21585
21586 SDValue Op0Lo = N->getOperand(Num: 0);
21587 SDValue Op0Hi = N->getOperand(Num: 1);
21588 SDValue Op1Lo = N->getOperand(Num: 2);
21589 SDValue Op1Hi = N->getOperand(Num: 3);
21590
21591 // (ADDD lo, hi, x, 0) -> (WADDAU lo, hi, x, 0)
21592 if (isNullConstant(V: Op1Hi)) {
21593 SDValue Result =
21594 DAG.getNode(Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
21595 N1: Op0Lo, N2: Op0Hi, N3: Op1Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
21596 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21597 }
21598 // (ADDD x, 0, lo, hi) -> (WADDAU lo, hi, x, 0)
21599 if (isNullConstant(V: Op0Hi)) {
21600 SDValue Result =
21601 DAG.getNode(Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
21602 N1: Op1Lo, N2: Op1Hi, N3: Op0Lo, N4: DAG.getConstant(Val: 0, DL, VT: MVT::i32));
21603 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21604 }
21605 break;
21606 }
21607 case RISCVISD::SUBD: {
21608 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
21609 "SUBD is only for RV32 with P extension");
21610
21611 SDValue Op0Lo = N->getOperand(Num: 0);
21612 SDValue Op0Hi = N->getOperand(Num: 1);
21613 SDValue Op1Lo = N->getOperand(Num: 2);
21614 SDValue Op1Hi = N->getOperand(Num: 3);
21615
21616 // (SUBD lo, hi, x, 0) -> (WSUBAU lo, hi, 0, x)
21617 // WSUBAU semantics: rd = rd + zext(rs1) - zext(rs2)
21618 if (isNullConstant(V: Op1Hi)) {
21619 SDValue Result =
21620 DAG.getNode(Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
21621 N1: Op0Lo, N2: Op0Hi, N3: DAG.getConstant(Val: 0, DL, VT: MVT::i32), N4: Op1Lo);
21622 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21623 }
21624 break;
21625 }
21626 case RISCVISD::WADDAU: {
21627 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
21628 "WADDAU is only for RV32 with P extension");
21629 SDValue Op0Lo = N->getOperand(Num: 0);
21630 SDValue Op0Hi = N->getOperand(Num: 1);
21631 SDValue Op1 = N->getOperand(Num: 2);
21632 SDValue Op2 = N->getOperand(Num: 3);
21633
21634 // (WADDAU lo, 0, rs1, 0) -> (WADDU lo, rs1)
21635 if (isNullConstant(V: Op0Hi) && isNullConstant(V: Op2)) {
21636 SDValue Result = DAG.getNode(
21637 Opcode: RISCVISD::WADDU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op1);
21638 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21639 }
21640
21641 // (WADDAU -C, -1, rs1, 0) -> (WSUBU rs1, C) where C > 0
21642 if (isNullConstant(V: Op2) && isAllOnesConstant(V: Op0Hi)) {
21643 if (auto *C0 = dyn_cast<ConstantSDNode>(Val&: Op0Lo)) {
21644 int64_t Val = C0->getSExtValue();
21645 if (Val < 0) {
21646 SDValue PosConst = DAG.getConstant(Val: -Val, DL, VT: MVT::i32);
21647 SDValue Result =
21648 DAG.getNode(Opcode: RISCVISD::WSUBU, DL,
21649 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op1, N2: PosConst);
21650 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21651 }
21652 }
21653 }
21654
21655 // FIXME: Canonicalize zero Op1 to Op2.
21656 if (isNullConstant(V: Op2) && Op0Lo.getNode() == Op0Hi.getNode() &&
21657 Op0Lo.getResNo() == 0 && Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() &&
21658 Op0Hi.hasOneUse()) {
21659 // (WADDAU (WADDAU lo, hi, x, 0), y, 0) -> (WADDAU lo, hi, x, y)
21660 if (Op0Lo.getOpcode() == RISCVISD::WADDAU &&
21661 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
21662 SDValue Result = DAG.getNode(
21663 Opcode: RISCVISD::WADDAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
21664 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op1);
21665 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21666 }
21667 // (WADDAU (WSUBAU lo, hi, 0, a), b, 0) -> (WSUBAU lo, hi, b, a)
21668 if (Op0Lo.getOpcode() == RISCVISD::WSUBAU &&
21669 isNullConstant(V: Op0Lo.getOperand(i: 2))) {
21670 SDValue Result = DAG.getNode(
21671 Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
21672 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op1, N4: Op0Lo.getOperand(i: 3));
21673 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21674 }
21675 }
21676 break;
21677 }
21678 case RISCVISD::WSUBAU: {
21679 assert(!Subtarget.is64Bit() && Subtarget.hasStdExtP() &&
21680 "WSUBAU is only for RV32 with P extension");
21681 SDValue Op0Lo = N->getOperand(Num: 0);
21682 SDValue Op0Hi = N->getOperand(Num: 1);
21683 SDValue Op1 = N->getOperand(Num: 2);
21684 SDValue Op2 = N->getOperand(Num: 3);
21685
21686 // (WSUBAU lo, 0, 0, rs2) -> (WSUBU lo, rs2)
21687 if (isNullConstant(V: Op0Hi) && isNullConstant(V: Op1)) {
21688 SDValue Result = DAG.getNode(
21689 Opcode: RISCVISD::WSUBU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N1: Op0Lo, N2: Op2);
21690 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21691 }
21692
21693 // (WSUBAU (WADDAU lo, hi, a, 0), 0, b) -> (WSUBAU lo, hi, a, b)
21694 if (isNullConstant(V: Op1) && Op0Lo.getOpcode() == RISCVISD::WADDAU &&
21695 Op0Lo.getNode() == Op0Hi.getNode() && Op0Lo.getResNo() == 0 &&
21696 Op0Hi.getResNo() == 1 && Op0Lo.hasOneUse() && Op0Hi.hasOneUse() &&
21697 isNullConstant(V: Op0Lo.getOperand(i: 3))) {
21698 SDValue Result = DAG.getNode(
21699 Opcode: RISCVISD::WSUBAU, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32),
21700 N1: Op0Lo.getOperand(i: 0), N2: Op0Lo.getOperand(i: 1), N3: Op0Lo.getOperand(i: 2), N4: Op2);
21701 return DCI.CombineTo(N, Res0: Result.getValue(R: 0), Res1: Result.getValue(R: 1));
21702 }
21703 break;
21704 }
21705 case RISCVISD::FMV_W_X_RV64: {
21706 // If the input to FMV_W_X_RV64 is just FMV_X_ANYEXTW_RV64 the the
21707 // conversion is unnecessary and can be replaced with the
21708 // FMV_X_ANYEXTW_RV64 operand.
21709 SDValue Op0 = N->getOperand(Num: 0);
21710 if (Op0.getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64)
21711 return Op0.getOperand(i: 0);
21712 break;
21713 }
21714 case RISCVISD::FMV_X_ANYEXTH:
21715 case RISCVISD::FMV_X_ANYEXTW_RV64: {
21716 SDLoc DL(N);
21717 SDValue Op0 = N->getOperand(Num: 0);
21718 MVT VT = N->getSimpleValueType(ResNo: 0);
21719
21720 // Constant fold.
21721 if (auto *CFP = dyn_cast<ConstantFPSDNode>(Val&: Op0)) {
21722 APInt Val = CFP->getValueAPF().bitcastToAPInt().sext(width: VT.getSizeInBits());
21723 return DAG.getConstant(Val, DL, VT);
21724 }
21725
21726 // If the input to FMV_X_ANYEXTW_RV64 is just FMV_W_X_RV64 then the
21727 // conversion is unnecessary and can be replaced with the FMV_W_X_RV64
21728 // operand. Similar for FMV_X_ANYEXTH and FMV_H_X.
21729 if ((N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 &&
21730 Op0->getOpcode() == RISCVISD::FMV_W_X_RV64) ||
21731 (N->getOpcode() == RISCVISD::FMV_X_ANYEXTH &&
21732 Op0->getOpcode() == RISCVISD::FMV_H_X)) {
21733 assert(Op0.getOperand(0).getValueType() == VT &&
21734 "Unexpected value type!");
21735 return Op0.getOperand(i: 0);
21736 }
21737
21738 if (ISD::isNormalLoad(N: Op0.getNode()) && Op0.hasOneUse() &&
21739 cast<LoadSDNode>(Val&: Op0)->isSimple()) {
21740 MVT IVT = MVT::getIntegerVT(BitWidth: Op0.getValueSizeInBits());
21741 auto *LN0 = cast<LoadSDNode>(Val&: Op0);
21742 SDValue Load =
21743 DAG.getExtLoad(ExtType: ISD::EXTLOAD, dl: SDLoc(N), VT, Chain: LN0->getChain(),
21744 Ptr: LN0->getBasePtr(), MemVT: IVT, MMO: LN0->getMemOperand());
21745 DAG.ReplaceAllUsesOfValueWith(From: Op0.getValue(R: 1), To: Load.getValue(R: 1));
21746 return Load;
21747 }
21748
21749 // This is a target-specific version of a DAGCombine performed in
21750 // DAGCombiner::visitBITCAST. It performs the equivalent of:
21751 // fold (bitconvert (fneg x)) -> (xor (bitconvert x), signbit)
21752 // fold (bitconvert (fabs x)) -> (and (bitconvert x), (not signbit))
21753 if (!(Op0.getOpcode() == ISD::FNEG || Op0.getOpcode() == ISD::FABS) ||
21754 !Op0.getNode()->hasOneUse())
21755 break;
21756 SDValue NewFMV = DAG.getNode(Opcode: N->getOpcode(), DL, VT, Operand: Op0.getOperand(i: 0));
21757 unsigned FPBits = N->getOpcode() == RISCVISD::FMV_X_ANYEXTW_RV64 ? 32 : 16;
21758 APInt SignBit = APInt::getSignMask(BitWidth: FPBits).sext(width: VT.getSizeInBits());
21759 if (Op0.getOpcode() == ISD::FNEG)
21760 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewFMV,
21761 N2: DAG.getConstant(Val: SignBit, DL, VT));
21762
21763 assert(Op0.getOpcode() == ISD::FABS);
21764 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: NewFMV,
21765 N2: DAG.getConstant(Val: ~SignBit, DL, VT));
21766 }
21767 case ISD::ABS: {
21768 EVT VT = N->getValueType(ResNo: 0);
21769 SDValue N0 = N->getOperand(Num: 0);
21770 // abs (sext) -> zext (abs)
21771 // abs (zext) -> zext (handled elsewhere)
21772 if (VT.isVector() && N0.hasOneUse() && N0.getOpcode() == ISD::SIGN_EXTEND) {
21773 SDValue Src = N0.getOperand(i: 0);
21774 SDLoc DL(N);
21775 return DAG.getNode(Opcode: ISD::ZERO_EXTEND, DL, VT,
21776 Operand: DAG.getNode(Opcode: ISD::ABS, DL, VT: Src.getValueType(), Operand: Src));
21777 }
21778 break;
21779 }
21780 case ISD::ADD: {
21781 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
21782 return V;
21783 if (SDValue V = combineToVWMACC(N, DAG, Subtarget))
21784 return V;
21785 if (SDValue V = combineVdota4Accum(N, DAG, Subtarget))
21786 return V;
21787 return performADDCombine(N, DCI, Subtarget);
21788 }
21789 case ISD::SUB: {
21790 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
21791 return V;
21792 return performSUBCombine(N, DAG, Subtarget);
21793 }
21794 case ISD::AND:
21795 return performANDCombine(N, DCI, Subtarget);
21796 case ISD::OR: {
21797 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
21798 return V;
21799 return performORCombine(N, DCI, Subtarget);
21800 }
21801 case ISD::XOR:
21802 return performXORCombine(N, DAG, Subtarget);
21803 case ISD::MUL:
21804 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
21805 return V;
21806 return performMULCombine(N, DAG, DCI, Subtarget);
21807 case ISD::SDIV:
21808 case ISD::UDIV:
21809 case ISD::SREM:
21810 case ISD::UREM:
21811 if (SDValue V = combineBinOpOfZExt(N, DAG))
21812 return V;
21813 break;
21814 case ISD::FMUL: {
21815 using namespace SDPatternMatch;
21816 SDLoc DL(N);
21817 EVT VT = N->getValueType(ResNo: 0);
21818 SDValue X, Y;
21819 // InstCombine canonicalizes fneg (fmul x, y) -> fmul x, (fneg y), see
21820 // hoistFNegAboveFMulFDiv.
21821 // Undo this and sink the fneg so we match more fmsub/fnmadd patterns.
21822 if (sd_match(N, P: m_FMul(L: m_Value(N&: X), R: m_OneUse(P: m_FNeg(Op: m_Value(N&: Y))))))
21823 return DAG.getNode(Opcode: ISD::FNEG, DL, VT,
21824 Operand: DAG.getNode(Opcode: ISD::FMUL, DL, VT, N1: X, N2: Y, Flags: N->getFlags()),
21825 Flags: N->getFlags());
21826
21827 // fmul X, (copysign 1.0, Y) -> fsgnjx X, Y
21828 SDValue N0 = N->getOperand(Num: 0);
21829 SDValue N1 = N->getOperand(Num: 1);
21830 if (N0->getOpcode() != ISD::FCOPYSIGN)
21831 std::swap(a&: N0, b&: N1);
21832 if (N0->getOpcode() != ISD::FCOPYSIGN)
21833 return SDValue();
21834 ConstantFPSDNode *C = dyn_cast<ConstantFPSDNode>(Val: N0->getOperand(Num: 0));
21835 if (!C || !C->getValueAPF().isExactlyValue(V: +1.0))
21836 return SDValue();
21837 if (VT.isVector() || !isOperationLegal(Op: ISD::FCOPYSIGN, VT))
21838 return SDValue();
21839 SDValue Sign = N0->getOperand(Num: 1);
21840 if (Sign.getValueType() != VT)
21841 return SDValue();
21842 return DAG.getNode(Opcode: RISCVISD::FSGNJX, DL, VT, N1, N2: N0->getOperand(Num: 1));
21843 }
21844 case ISD::FADD:
21845 case ISD::UMAX:
21846 case ISD::UMIN:
21847 case ISD::SMAX:
21848 case ISD::SMIN:
21849 case ISD::FMAXNUM:
21850 case ISD::FMINNUM: {
21851 if (SDValue V = combineBinOpToReduce(N, DAG, Subtarget))
21852 return V;
21853 if (SDValue V = combineBinOpOfExtractToReduceTree(N, DAG, Subtarget))
21854 return V;
21855 return SDValue();
21856 }
21857 case ISD::FMA: {
21858 SDValue N0 = N->getOperand(Num: 0);
21859 SDValue N1 = N->getOperand(Num: 1);
21860 if (N0.getOpcode() != ISD::SPLAT_VECTOR)
21861 std::swap(a&: N0, b&: N1);
21862 if (N0.getOpcode() != ISD::SPLAT_VECTOR)
21863 return SDValue();
21864 SDValue SplatN0 = N0.getOperand(i: 0);
21865 if (SplatN0.getOpcode() != ISD::FNEG || !SplatN0.hasOneUse())
21866 return SDValue();
21867 EVT VT = N->getValueType(ResNo: 0);
21868 SDValue Splat =
21869 DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT, Operand: SplatN0.getOperand(i: 0));
21870 SDValue Fneg = DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: Splat);
21871 return DAG.getNode(Opcode: ISD::FMA, DL, VT, N1: Fneg, N2: N1, N3: N->getOperand(Num: 2));
21872 }
21873 case ISD::SETCC:
21874 return performSETCCCombine(N, DCI, Subtarget);
21875 case ISD::SIGN_EXTEND_INREG:
21876 return performSIGN_EXTEND_INREGCombine(N, DCI, Subtarget);
21877 case ISD::ZERO_EXTEND:
21878 // Fold (zero_extend (fp_to_uint X)) to prevent forming fcvt+zexti32 during
21879 // type legalization. This is safe because fp_to_uint produces poison if
21880 // it overflows.
21881 if (N->getValueType(ResNo: 0) == MVT::i64 && Subtarget.is64Bit()) {
21882 SDValue Src = N->getOperand(Num: 0);
21883 if (Src.getOpcode() == ISD::FP_TO_UINT &&
21884 isTypeLegal(VT: Src.getOperand(i: 0).getValueType()))
21885 return DAG.getNode(Opcode: ISD::FP_TO_UINT, DL: SDLoc(N), VT: MVT::i64,
21886 Operand: Src.getOperand(i: 0));
21887 if (Src.getOpcode() == ISD::STRICT_FP_TO_UINT && Src.hasOneUse() &&
21888 isTypeLegal(VT: Src.getOperand(i: 1).getValueType())) {
21889 SDVTList VTs = DAG.getVTList(VT1: MVT::i64, VT2: MVT::Other);
21890 SDValue Res = DAG.getNode(Opcode: ISD::STRICT_FP_TO_UINT, DL: SDLoc(N), VTList: VTs,
21891 N1: Src.getOperand(i: 0), N2: Src.getOperand(i: 1));
21892 DCI.CombineTo(N, Res);
21893 DAG.ReplaceAllUsesOfValueWith(From: Src.getValue(R: 1), To: Res.getValue(R: 1));
21894 DCI.recursivelyDeleteUnusedNodes(N: Src.getNode());
21895 return SDValue(N, 0); // Return N so it doesn't get rechecked.
21896 }
21897 }
21898 return SDValue();
21899 case RISCVISD::TRUNCATE_VECTOR_VL:
21900 if (SDValue V = combineTruncOfSraSext(N, DAG))
21901 return V;
21902 return combineTruncToVnclip(N, DAG, Subtarget);
21903 case ISD::VP_TRUNCATE:
21904 return performVP_TRUNCATECombine(N, DAG, Subtarget);
21905 case ISD::TRUNCATE:
21906 return performTRUNCATECombine(N, DAG, Subtarget);
21907 case ISD::SELECT:
21908 return performSELECTCombine(N, DAG, Subtarget);
21909 case ISD::VSELECT:
21910 return performVSELECTCombine(N, DAG);
21911 case RISCVISD::CZERO_EQZ:
21912 case RISCVISD::CZERO_NEZ: {
21913 SDValue Val = N->getOperand(Num: 0);
21914 SDValue Cond = N->getOperand(Num: 1);
21915 MVT VT = N->getSimpleValueType(ResNo: 0);
21916
21917 unsigned Opc = N->getOpcode();
21918
21919 // czero_eqz x, x -> x
21920 if (Opc == RISCVISD::CZERO_EQZ && Val == Cond)
21921 return Val;
21922
21923 unsigned InvOpc =
21924 Opc == RISCVISD::CZERO_EQZ ? RISCVISD::CZERO_NEZ : RISCVISD::CZERO_EQZ;
21925
21926 // czero_eqz X, (xor Y, 1) -> czero_nez X, Y if Y is 0 or 1.
21927 // czero_nez X, (xor Y, 1) -> czero_eqz X, Y if Y is 0 or 1.
21928 if (Cond.getOpcode() == ISD::XOR && isOneConstant(V: Cond.getOperand(i: 1))) {
21929 SDValue NewCond = Cond.getOperand(i: 0);
21930 APInt Mask = APInt::getBitsSetFrom(numBits: NewCond.getValueSizeInBits(), loBit: 1);
21931 if (DAG.MaskedValueIsZero(Op: NewCond, Mask))
21932 return DAG.getNode(Opcode: InvOpc, DL: SDLoc(N), VT, N1: Val, N2: NewCond);
21933 }
21934 // czero_eqz x, (setcc y, 0, ne) -> czero_eqz x, y
21935 // czero_nez x, (setcc y, 0, ne) -> czero_nez x, y
21936 // czero_eqz x, (setcc y, 0, eq) -> czero_nez x, y
21937 // czero_nez x, (setcc y, 0, eq) -> czero_eqz x, y
21938 if (Cond.getOpcode() == ISD::SETCC && isNullConstant(V: Cond.getOperand(i: 1))) {
21939 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get();
21940 if (ISD::isIntEqualitySetCC(Code: CCVal))
21941 return DAG.getNode(Opcode: CCVal == ISD::SETNE ? Opc : InvOpc, DL: SDLoc(N), VT,
21942 N1: Val, N2: Cond.getOperand(i: 0));
21943 }
21944
21945 // Remove SRL from bittest patterns (srl (and X, (1 << C)), C) if the and
21946 // is an ANDI. Because only 1 bit can be set after the AND, it doesn't
21947 // matter if we shift it.
21948 if (Cond.getOpcode() == ISD::SRL &&
21949 isa<ConstantSDNode>(Val: Cond.getOperand(i: 1)) &&
21950 Cond.getOperand(i: 0).getOpcode() == ISD::AND) {
21951 const APInt &ShAmt = Cond.getConstantOperandAPInt(i: 1);
21952 unsigned BitWidth = VT.getSizeInBits();
21953 SDValue And = Cond.getOperand(i: 0);
21954 if (ShAmt.ult(RHS: BitWidth) && isa<ConstantSDNode>(Val: And.getOperand(i: 1))) {
21955 uint64_t AndConst = And.getConstantOperandVal(i: 1);
21956 if (AndConst == (1ULL << ShAmt.getZExtValue()) && isInt<12>(x: AndConst))
21957 return DAG.getNode(Opcode: Opc, DL, VT, N1: Val, N2: And);
21958 }
21959 }
21960
21961 // czero_nez (setcc X, Y, CC), (setcc X, Y, eq) -> (setcc X, Y, CC)
21962 // if CC is a strict inequality (lt, gt, ult, ugt), because when X == Y
21963 // the setcc result is already 0. The eq operands can be in either order.
21964 if (Opc == RISCVISD::CZERO_NEZ && Val.getOpcode() == ISD::SETCC &&
21965 Cond.getOpcode() == ISD::SETCC &&
21966 cast<CondCodeSDNode>(Val: Cond.getOperand(i: 2))->get() == ISD::SETEQ) {
21967 ISD::CondCode ValCC = cast<CondCodeSDNode>(Val: Val.getOperand(i: 2))->get();
21968 bool SameOperands = (Val.getOperand(i: 0) == Cond.getOperand(i: 0) &&
21969 Val.getOperand(i: 1) == Cond.getOperand(i: 1)) ||
21970 (Val.getOperand(i: 0) == Cond.getOperand(i: 1) &&
21971 Val.getOperand(i: 1) == Cond.getOperand(i: 0));
21972 if (SameOperands && (ValCC == ISD::SETLT || ValCC == ISD::SETGT ||
21973 ValCC == ISD::SETULT || ValCC == ISD::SETUGT))
21974 return Val;
21975 }
21976
21977 return SDValue();
21978 }
21979 case RISCVISD::SELECT_CC: {
21980 // Transform
21981 SDValue LHS = N->getOperand(Num: 0);
21982 SDValue RHS = N->getOperand(Num: 1);
21983 SDValue CC = N->getOperand(Num: 2);
21984 ISD::CondCode CCVal = cast<CondCodeSDNode>(Val&: CC)->get();
21985 SDValue TrueV = N->getOperand(Num: 3);
21986 SDValue FalseV = N->getOperand(Num: 4);
21987 SDLoc DL(N);
21988 EVT VT = N->getValueType(ResNo: 0);
21989
21990 // If the True and False values are the same, we don't need a select_cc.
21991 if (TrueV == FalseV)
21992 return TrueV;
21993
21994 // (select (x < 0), y, z) -> x >> (XLEN - 1) & (y - z) + z
21995 // (select (x >= 0), y, z) -> x >> (XLEN - 1) & (z - y) + y
21996 if (!Subtarget.hasShortForwardBranchIALU() && isa<ConstantSDNode>(Val: TrueV) &&
21997 isa<ConstantSDNode>(Val: FalseV) && isNullConstant(V: RHS) &&
21998 (CCVal == ISD::CondCode::SETLT || CCVal == ISD::CondCode::SETGE)) {
21999 if (CCVal == ISD::CondCode::SETGE)
22000 std::swap(a&: TrueV, b&: FalseV);
22001
22002 int64_t TrueSImm = cast<ConstantSDNode>(Val&: TrueV)->getSExtValue();
22003 int64_t FalseSImm = cast<ConstantSDNode>(Val&: FalseV)->getSExtValue();
22004 // Only handle simm12, if it is not in this range, it can be considered as
22005 // register.
22006 if (isInt<12>(x: TrueSImm) && isInt<12>(x: FalseSImm) &&
22007 isInt<12>(x: TrueSImm - FalseSImm)) {
22008 SDValue SRA =
22009 DAG.getNode(Opcode: ISD::SRA, DL, VT, N1: LHS,
22010 N2: DAG.getConstant(Val: Subtarget.getXLen() - 1, DL, VT));
22011 SDValue AND =
22012 DAG.getNode(Opcode: ISD::AND, DL, VT, N1: SRA,
22013 N2: DAG.getSignedConstant(Val: TrueSImm - FalseSImm, DL, VT));
22014 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: AND, N2: FalseV);
22015 }
22016
22017 if (CCVal == ISD::CondCode::SETGE)
22018 std::swap(a&: TrueV, b&: FalseV);
22019 }
22020
22021 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
22022 return DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT: N->getValueType(ResNo: 0),
22023 Ops: {LHS, RHS, CC, TrueV, FalseV});
22024
22025 if (!Subtarget.hasConditionalMoveFusion()) {
22026 // (select c, -1, y) -> -c | y
22027 if (isAllOnesConstant(V: TrueV)) {
22028 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: CCVal);
22029 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22030 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: FalseV);
22031 }
22032 // (select c, y, -1) -> -!c | y
22033 if (isAllOnesConstant(V: FalseV)) {
22034 SDValue C =
22035 DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::getSetCCInverse(Operation: CCVal, Type: VT));
22036 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22037 return DAG.getNode(Opcode: ISD::OR, DL, VT, N1: Neg, N2: TrueV);
22038 }
22039
22040 // (select c, 0, y) -> -!c & y
22041 if (isNullConstant(V: TrueV)) {
22042 SDValue C =
22043 DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::getSetCCInverse(Operation: CCVal, Type: VT));
22044 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22045 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: FalseV);
22046 }
22047 // (select c, y, 0) -> -c & y
22048 if (isNullConstant(V: FalseV)) {
22049 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: CCVal);
22050 SDValue Neg = DAG.getNegative(Val: C, DL, VT);
22051 return DAG.getNode(Opcode: ISD::AND, DL, VT, N1: Neg, N2: TrueV);
22052 }
22053 // (riscvisd::select_cc x, 0, ne, x, 1) -> (add x, (setcc x, 0, eq))
22054 // (riscvisd::select_cc x, 0, eq, 1, x) -> (add x, (setcc x, 0, eq))
22055 if (((isOneConstant(V: FalseV) && LHS == TrueV &&
22056 CCVal == ISD::CondCode::SETNE) ||
22057 (isOneConstant(V: TrueV) && LHS == FalseV &&
22058 CCVal == ISD::CondCode::SETEQ)) &&
22059 isNullConstant(V: RHS)) {
22060 // freeze it to be safe.
22061 LHS = DAG.getFreeze(V: LHS);
22062 SDValue C = DAG.getSetCC(DL, VT, LHS, RHS, Cond: ISD::CondCode::SETEQ);
22063 return DAG.getNode(Opcode: ISD::ADD, DL, VT, N1: LHS, N2: C);
22064 }
22065 }
22066
22067 // If both true/false are an xor with 1, pull through the select.
22068 // This can occur after op legalization if both operands are setccs that
22069 // require an xor to invert.
22070 // FIXME: Generalize to other binary ops with identical operand?
22071 if (TrueV.getOpcode() == ISD::XOR && FalseV.getOpcode() == ISD::XOR &&
22072 TrueV.getOperand(i: 1) == FalseV.getOperand(i: 1) &&
22073 isOneConstant(V: TrueV.getOperand(i: 1)) &&
22074 TrueV.hasOneUse() && FalseV.hasOneUse()) {
22075 SDValue NewSel = DAG.getNode(Opcode: RISCVISD::SELECT_CC, DL, VT, N1: LHS, N2: RHS, N3: CC,
22076 N4: TrueV.getOperand(i: 0), N5: FalseV.getOperand(i: 0));
22077 return DAG.getNode(Opcode: ISD::XOR, DL, VT, N1: NewSel, N2: TrueV.getOperand(i: 1));
22078 }
22079
22080 return SDValue();
22081 }
22082 case RISCVISD::BR_CC: {
22083 SDValue LHS = N->getOperand(Num: 1);
22084 SDValue RHS = N->getOperand(Num: 2);
22085 SDValue CC = N->getOperand(Num: 3);
22086 SDLoc DL(N);
22087
22088 if (combine_CC(LHS, RHS, CC, DL, DAG, Subtarget))
22089 return DAG.getNode(Opcode: RISCVISD::BR_CC, DL, VT: N->getValueType(ResNo: 0),
22090 N1: N->getOperand(Num: 0), N2: LHS, N3: RHS, N4: CC, N5: N->getOperand(Num: 4));
22091
22092 return SDValue();
22093 }
22094 case ISD::BITREVERSE:
22095 return performBITREVERSECombine(N, DAG, Subtarget);
22096 case ISD::FP_TO_SINT:
22097 case ISD::FP_TO_UINT:
22098 return performFP_TO_INTCombine(N, DCI, Subtarget);
22099 case ISD::FP_TO_SINT_SAT:
22100 case ISD::FP_TO_UINT_SAT:
22101 return performFP_TO_INT_SATCombine(N, DCI, Subtarget);
22102 case ISD::FCOPYSIGN: {
22103 EVT VT = N->getValueType(ResNo: 0);
22104 if (!VT.isVector())
22105 break;
22106 // There is a form of VFSGNJ which injects the negated sign of its second
22107 // operand. Try and bubble any FNEG up after the extend/round to produce
22108 // this optimized pattern. Avoid modifying cases where FP_ROUND and
22109 // TRUNC=1.
22110 SDValue In2 = N->getOperand(Num: 1);
22111 // Avoid cases where the extend/round has multiple uses, as duplicating
22112 // those is typically more expensive than removing a fneg.
22113 if (!In2.hasOneUse())
22114 break;
22115 if (In2.getOpcode() != ISD::FP_EXTEND &&
22116 (In2.getOpcode() != ISD::FP_ROUND || In2.getConstantOperandVal(i: 1) != 0))
22117 break;
22118 In2 = In2.getOperand(i: 0);
22119 if (In2.getOpcode() != ISD::FNEG)
22120 break;
22121 SDLoc DL(N);
22122 SDValue NewFPExtRound = DAG.getFPExtendOrRound(Op: In2.getOperand(i: 0), DL, VT);
22123 return DAG.getNode(Opcode: ISD::FCOPYSIGN, DL, VT, N1: N->getOperand(Num: 0),
22124 N2: DAG.getNode(Opcode: ISD::FNEG, DL, VT, Operand: NewFPExtRound));
22125 }
22126 case ISD::MGATHER: {
22127 const auto *MGN = cast<MaskedGatherSDNode>(Val: N);
22128 const EVT VT = N->getValueType(ResNo: 0);
22129 SDValue Index = MGN->getIndex();
22130 SDValue ScaleOp = MGN->getScale();
22131 ISD::MemIndexType IndexType = MGN->getIndexType();
22132 assert(!MGN->isIndexScaled() &&
22133 "Scaled gather/scatter should not be formed");
22134
22135 SDLoc DL(N);
22136 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
22137 return DAG.getMaskedGather(
22138 VTs: N->getVTList(), MemVT: MGN->getMemoryVT(), dl: DL,
22139 Ops: {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
22140 MGN->getBasePtr(), Index, ScaleOp},
22141 MMO: MGN->getMemOperand(), IndexType, ExtTy: MGN->getExtensionType());
22142
22143 if (narrowIndex(N&: Index, IndexType, DAG))
22144 return DAG.getMaskedGather(
22145 VTs: N->getVTList(), MemVT: MGN->getMemoryVT(), dl: DL,
22146 Ops: {MGN->getChain(), MGN->getPassThru(), MGN->getMask(),
22147 MGN->getBasePtr(), Index, ScaleOp},
22148 MMO: MGN->getMemOperand(), IndexType, ExtTy: MGN->getExtensionType());
22149
22150 if (Index.getOpcode() == ISD::BUILD_VECTOR &&
22151 MGN->getExtensionType() == ISD::NON_EXTLOAD && isTypeLegal(VT)) {
22152 // The sequence will be XLenVT, not the type of Index. Tell
22153 // isSimpleVIDSequence this so we avoid overflow.
22154 if (std::optional<VIDSequence> SimpleVID =
22155 isSimpleVIDSequence(Op: Index, EltSizeInBits: Subtarget.getXLen());
22156 SimpleVID && SimpleVID->StepDenominator == 1) {
22157 const int64_t StepNumerator = SimpleVID->StepNumerator;
22158 const int64_t Addend = SimpleVID->Addend;
22159
22160 // Note: We don't need to check alignment here since (by assumption
22161 // from the existence of the gather), our offsets must be sufficiently
22162 // aligned.
22163
22164 const EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
22165 assert(MGN->getBasePtr()->getValueType(0) == PtrVT);
22166 assert(IndexType == ISD::UNSIGNED_SCALED);
22167 SDValue BasePtr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: MGN->getBasePtr(),
22168 N2: DAG.getSignedConstant(Val: Addend, DL, VT: PtrVT));
22169
22170 SDValue EVL = DAG.getElementCount(DL, VT: Subtarget.getXLenVT(),
22171 EC: VT.getVectorElementCount());
22172 SDValue StridedLoad = DAG.getStridedLoadVP(
22173 VT, DL, Chain: MGN->getChain(), Ptr: BasePtr,
22174 Stride: DAG.getSignedConstant(Val: StepNumerator, DL, VT: XLenVT), Mask: MGN->getMask(),
22175 EVL, MMO: MGN->getMemOperand());
22176 SDValue Select = DAG.getSelect(DL, VT, Cond: MGN->getMask(), LHS: StridedLoad,
22177 RHS: MGN->getPassThru());
22178 return DAG.getMergeValues(Ops: {Select, SDValue(StridedLoad.getNode(), 1)},
22179 dl: DL);
22180 }
22181 }
22182
22183 SmallVector<int> ShuffleMask;
22184 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
22185 matchIndexAsShuffle(VT, Index, Mask: MGN->getMask(), ShuffleMask)) {
22186 SDValue Load = DAG.getMaskedLoad(VT, dl: DL, Chain: MGN->getChain(),
22187 Base: MGN->getBasePtr(), Offset: DAG.getUNDEF(VT: XLenVT),
22188 Mask: MGN->getMask(), Src0: DAG.getUNDEF(VT),
22189 MemVT: MGN->getMemoryVT(), MMO: MGN->getMemOperand(),
22190 AM: ISD::UNINDEXED, ISD::NON_EXTLOAD);
22191 SDValue Shuffle =
22192 DAG.getVectorShuffle(VT, dl: DL, N1: Load, N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
22193 return DAG.getMergeValues(Ops: {Shuffle, Load.getValue(R: 1)}, dl: DL);
22194 }
22195
22196 if (MGN->getExtensionType() == ISD::NON_EXTLOAD &&
22197 matchIndexAsWiderOp(VT, Index, Mask: MGN->getMask(),
22198 BaseAlign: MGN->getMemOperand()->getBaseAlign(), ST: Subtarget)) {
22199 SmallVector<SDValue> NewIndices;
22200 for (unsigned i = 0; i < Index->getNumOperands(); i += 2)
22201 NewIndices.push_back(Elt: Index.getOperand(i));
22202 EVT IndexVT = Index.getValueType()
22203 .getHalfNumVectorElementsVT(Context&: *DAG.getContext());
22204 Index = DAG.getBuildVector(VT: IndexVT, DL, Ops: NewIndices);
22205
22206 unsigned ElementSize = VT.getScalarStoreSize();
22207 EVT WideScalarVT = MVT::getIntegerVT(BitWidth: ElementSize * 8 * 2);
22208 auto EltCnt = VT.getVectorElementCount();
22209 assert(EltCnt.isKnownEven() && "Splitting vector, but not in half!");
22210 EVT WideVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: WideScalarVT,
22211 EC: EltCnt.divideCoefficientBy(RHS: 2));
22212 SDValue Passthru = DAG.getBitcast(VT: WideVT, V: MGN->getPassThru());
22213 EVT MaskVT = EVT::getVectorVT(Context&: *DAG.getContext(), VT: MVT::i1,
22214 EC: EltCnt.divideCoefficientBy(RHS: 2));
22215 SDValue Mask = DAG.getSplat(VT: MaskVT, DL, Op: DAG.getConstant(Val: 1, DL, VT: MVT::i1));
22216
22217 SDValue Gather =
22218 DAG.getMaskedGather(VTs: DAG.getVTList(VT1: WideVT, VT2: MVT::Other), MemVT: WideVT, dl: DL,
22219 Ops: {MGN->getChain(), Passthru, Mask, MGN->getBasePtr(),
22220 Index, ScaleOp},
22221 MMO: MGN->getMemOperand(), IndexType, ExtTy: ISD::NON_EXTLOAD);
22222 SDValue Result = DAG.getBitcast(VT, V: Gather.getValue(R: 0));
22223 return DAG.getMergeValues(Ops: {Result, Gather.getValue(R: 1)}, dl: DL);
22224 }
22225 break;
22226 }
22227 case ISD::MSCATTER:{
22228 const auto *MSN = cast<MaskedScatterSDNode>(Val: N);
22229 SDValue Index = MSN->getIndex();
22230 SDValue ScaleOp = MSN->getScale();
22231 ISD::MemIndexType IndexType = MSN->getIndexType();
22232 assert(!MSN->isIndexScaled() &&
22233 "Scaled gather/scatter should not be formed");
22234
22235 SDLoc DL(N);
22236 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
22237 return DAG.getMaskedScatter(
22238 VTs: N->getVTList(), MemVT: MSN->getMemoryVT(), dl: DL,
22239 Ops: {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
22240 Index, ScaleOp},
22241 MMO: MSN->getMemOperand(), IndexType, IsTruncating: MSN->isTruncatingStore());
22242
22243 if (narrowIndex(N&: Index, IndexType, DAG))
22244 return DAG.getMaskedScatter(
22245 VTs: N->getVTList(), MemVT: MSN->getMemoryVT(), dl: DL,
22246 Ops: {MSN->getChain(), MSN->getValue(), MSN->getMask(), MSN->getBasePtr(),
22247 Index, ScaleOp},
22248 MMO: MSN->getMemOperand(), IndexType, IsTruncating: MSN->isTruncatingStore());
22249
22250 EVT VT = MSN->getValue()->getValueType(ResNo: 0);
22251 SmallVector<int> ShuffleMask;
22252 if (!MSN->isTruncatingStore() &&
22253 matchIndexAsShuffle(VT, Index, Mask: MSN->getMask(), ShuffleMask)) {
22254 SDValue Shuffle = DAG.getVectorShuffle(VT, dl: DL, N1: MSN->getValue(),
22255 N2: DAG.getUNDEF(VT), Mask: ShuffleMask);
22256 return DAG.getMaskedStore(Chain: MSN->getChain(), dl: DL, Val: Shuffle, Base: MSN->getBasePtr(),
22257 Offset: DAG.getUNDEF(VT: XLenVT), Mask: MSN->getMask(),
22258 MemVT: MSN->getMemoryVT(), MMO: MSN->getMemOperand(),
22259 AM: ISD::UNINDEXED, IsTruncating: false);
22260 }
22261 break;
22262 }
22263 case ISD::VP_GATHER: {
22264 const auto *VPGN = cast<VPGatherSDNode>(Val: N);
22265 SDValue Index = VPGN->getIndex();
22266 SDValue ScaleOp = VPGN->getScale();
22267 ISD::MemIndexType IndexType = VPGN->getIndexType();
22268 assert(!VPGN->isIndexScaled() &&
22269 "Scaled gather/scatter should not be formed");
22270
22271 SDLoc DL(N);
22272 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
22273 return DAG.getGatherVP(VTs: N->getVTList(), VT: VPGN->getMemoryVT(), dl: DL,
22274 Ops: {VPGN->getChain(), VPGN->getBasePtr(), Index,
22275 ScaleOp, VPGN->getMask(),
22276 VPGN->getVectorLength()},
22277 MMO: VPGN->getMemOperand(), IndexType);
22278
22279 if (narrowIndex(N&: Index, IndexType, DAG))
22280 return DAG.getGatherVP(VTs: N->getVTList(), VT: VPGN->getMemoryVT(), dl: DL,
22281 Ops: {VPGN->getChain(), VPGN->getBasePtr(), Index,
22282 ScaleOp, VPGN->getMask(),
22283 VPGN->getVectorLength()},
22284 MMO: VPGN->getMemOperand(), IndexType);
22285
22286 break;
22287 }
22288 case ISD::VP_SCATTER: {
22289 const auto *VPSN = cast<VPScatterSDNode>(Val: N);
22290 SDValue Index = VPSN->getIndex();
22291 SDValue ScaleOp = VPSN->getScale();
22292 ISD::MemIndexType IndexType = VPSN->getIndexType();
22293 assert(!VPSN->isIndexScaled() &&
22294 "Scaled gather/scatter should not be formed");
22295
22296 SDLoc DL(N);
22297 if (legalizeScatterGatherIndexType(DL, Index, IndexType, DCI))
22298 return DAG.getScatterVP(VTs: N->getVTList(), VT: VPSN->getMemoryVT(), dl: DL,
22299 Ops: {VPSN->getChain(), VPSN->getValue(),
22300 VPSN->getBasePtr(), Index, ScaleOp,
22301 VPSN->getMask(), VPSN->getVectorLength()},
22302 MMO: VPSN->getMemOperand(), IndexType);
22303
22304 if (narrowIndex(N&: Index, IndexType, DAG))
22305 return DAG.getScatterVP(VTs: N->getVTList(), VT: VPSN->getMemoryVT(), dl: DL,
22306 Ops: {VPSN->getChain(), VPSN->getValue(),
22307 VPSN->getBasePtr(), Index, ScaleOp,
22308 VPSN->getMask(), VPSN->getVectorLength()},
22309 MMO: VPSN->getMemOperand(), IndexType);
22310 break;
22311 }
22312 case RISCVISD::SHL_VL:
22313 if (SDValue V = performSHLCombine(N, DCI, Subtarget))
22314 return V;
22315 [[fallthrough]];
22316 case RISCVISD::SRA_VL:
22317 case RISCVISD::SRL_VL: {
22318 SDValue ShAmt = N->getOperand(Num: 1);
22319 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
22320 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
22321 SDLoc DL(N);
22322 SDValue VL = N->getOperand(Num: 4);
22323 EVT VT = N->getValueType(ResNo: 0);
22324 ShAmt = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
22325 N2: ShAmt.getOperand(i: 1), N3: VL);
22326 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N->getOperand(Num: 0), N2: ShAmt,
22327 N3: N->getOperand(Num: 2), N4: N->getOperand(Num: 3), N5: N->getOperand(Num: 4));
22328 }
22329 break;
22330 }
22331 case ISD::SRA:
22332 if (SDValue V = performSRACombine(N, DAG, Subtarget))
22333 return V;
22334 [[fallthrough]];
22335 case ISD::SRL:
22336 case ISD::SHL: {
22337 if (N->getOpcode() == ISD::SHL) {
22338 if (SDValue V = performSHLCombine(N, DCI, Subtarget))
22339 return V;
22340 }
22341 SDValue ShAmt = N->getOperand(Num: 1);
22342 if (ShAmt.getOpcode() == RISCVISD::SPLAT_VECTOR_SPLIT_I64_VL) {
22343 // We don't need the upper 32 bits of a 64-bit element for a shift amount.
22344 SDLoc DL(N);
22345 EVT VT = N->getValueType(ResNo: 0);
22346 ShAmt = DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: DAG.getUNDEF(VT),
22347 N2: ShAmt.getOperand(i: 1),
22348 N3: DAG.getRegister(Reg: RISCV::X0, VT: Subtarget.getXLenVT()));
22349 return DAG.getNode(Opcode: N->getOpcode(), DL, VT, N1: N->getOperand(Num: 0), N2: ShAmt);
22350 }
22351 break;
22352 }
22353 case RISCVISD::ADD_VL:
22354 if (SDValue V = simplifyOp_VL(N))
22355 return V;
22356 if (SDValue V = combineOp_VLToVWOp_VL(N, DCI, Subtarget))
22357 return V;
22358 if (SDValue V = combineVdota4Accum(N, DAG, Subtarget))
22359 return V;
22360 return combineToVWMACC(N, DAG, Subtarget);
22361 case RISCVISD::VWADDU_VL:
22362 return performVWABDACombine(N, DAG, Subtarget);
22363 case RISCVISD::VWADDU_W_VL:
22364 if (SDValue V = performVWABDACombineWV(N, DAG, Subtarget))
22365 return V;
22366 [[fallthrough]];
22367 case RISCVISD::VWADD_W_VL:
22368 case RISCVISD::VWSUB_W_VL:
22369 case RISCVISD::VWSUBU_W_VL:
22370 return performVWADDSUBW_VLCombine(N, DCI, Subtarget);
22371 case RISCVISD::OR_VL:
22372 case RISCVISD::SUB_VL:
22373 case RISCVISD::MUL_VL:
22374 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
22375 case RISCVISD::VFMADD_VL:
22376 case RISCVISD::VFNMADD_VL:
22377 case RISCVISD::VFMSUB_VL:
22378 case RISCVISD::VFNMSUB_VL:
22379 case RISCVISD::STRICT_VFMADD_VL:
22380 case RISCVISD::STRICT_VFNMADD_VL:
22381 case RISCVISD::STRICT_VFMSUB_VL:
22382 case RISCVISD::STRICT_VFNMSUB_VL:
22383 return performVFMADD_VLCombine(N, DCI, Subtarget);
22384 case RISCVISD::FADD_VL:
22385 case RISCVISD::FSUB_VL:
22386 case RISCVISD::FMUL_VL:
22387 case RISCVISD::VFWADD_W_VL:
22388 case RISCVISD::VFWSUB_W_VL:
22389 return combineOp_VLToVWOp_VL(N, DCI, Subtarget);
22390 case ISD::LOAD:
22391 case ISD::STORE: {
22392 if (DCI.isAfterLegalizeDAG())
22393 if (SDValue V = performMemPairCombine(N, DCI))
22394 return V;
22395
22396 if (N->getOpcode() != ISD::STORE)
22397 break;
22398
22399 auto *Store = cast<StoreSDNode>(Val: N);
22400 SDValue Chain = Store->getChain();
22401 EVT MemVT = Store->getMemoryVT();
22402 SDValue Val = Store->getValue();
22403 SDLoc DL(N);
22404
22405 bool IsScalarizable =
22406 MemVT.isFixedLengthVector() && ISD::isNormalStore(N: Store) &&
22407 Store->isSimple() &&
22408 MemVT.getVectorElementType().bitsLE(VT: Subtarget.getXLenVT()) &&
22409 isPowerOf2_64(Value: MemVT.getSizeInBits()) &&
22410 MemVT.getSizeInBits() <= Subtarget.getXLen();
22411
22412 // If sufficiently aligned we can scalarize stores of constant vectors of
22413 // any power-of-two size up to XLen bits, provided that they aren't too
22414 // expensive to materialize.
22415 // vsetivli zero, 2, e8, m1, ta, ma
22416 // vmv.v.i v8, 4
22417 // vse64.v v8, (a0)
22418 // ->
22419 // li a1, 1028
22420 // sh a1, 0(a0)
22421 if (DCI.isBeforeLegalize() && IsScalarizable &&
22422 ISD::isBuildVectorOfConstantSDNodes(N: Val.getNode())) {
22423 // Get the constant vector bits
22424 APInt NewC(Val.getValueSizeInBits(), 0);
22425 uint64_t EltSize = Val.getScalarValueSizeInBits();
22426 for (unsigned i = 0; i < Val.getNumOperands(); i++) {
22427 if (Val.getOperand(i).isUndef())
22428 continue;
22429 NewC.insertBits(SubBits: Val.getConstantOperandAPInt(i).trunc(width: EltSize),
22430 bitPosition: i * EltSize);
22431 }
22432 MVT NewVT = MVT::getIntegerVT(BitWidth: MemVT.getSizeInBits());
22433
22434 if (RISCVMatInt::getIntMatCost(Val: NewC, Size: Subtarget.getXLen(), STI: Subtarget,
22435 CompressionCost: true) <= 2 &&
22436 allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
22437 VT: NewVT, MMO: *Store->getMemOperand())) {
22438 SDValue NewV = DAG.getConstant(Val: NewC, DL, VT: NewVT);
22439 return DAG.getStore(Chain, dl: DL, Val: NewV, Ptr: Store->getBasePtr(),
22440 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
22441 MMOFlags: Store->getMemOperand()->getFlags());
22442 }
22443 }
22444
22445 // Similarly, if sufficiently aligned we can scalarize vector copies, e.g.
22446 // vsetivli zero, 2, e16, m1, ta, ma
22447 // vle16.v v8, (a0)
22448 // vse16.v v8, (a1)
22449 if (auto *L = dyn_cast<LoadSDNode>(Val);
22450 L && DCI.isBeforeLegalize() && IsScalarizable && L->isSimple() &&
22451 L->hasNUsesOfValue(NUses: 1, Value: 0) && L->hasNUsesOfValue(NUses: 1, Value: 1) &&
22452 Store->getChain() == SDValue(L, 1) && ISD::isNormalLoad(N: L) &&
22453 L->getMemoryVT() == MemVT) {
22454 MVT NewVT = MVT::getIntegerVT(BitWidth: MemVT.getSizeInBits());
22455 if (allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
22456 VT: NewVT, MMO: *Store->getMemOperand()) &&
22457 allowsMemoryAccessForAlignment(Context&: *DAG.getContext(), DL: DAG.getDataLayout(),
22458 VT: NewVT, MMO: *L->getMemOperand())) {
22459 SDValue NewL = DAG.getLoad(VT: NewVT, dl: DL, Chain: L->getChain(), Ptr: L->getBasePtr(),
22460 PtrInfo: L->getPointerInfo(), Alignment: L->getBaseAlign(),
22461 MMOFlags: L->getMemOperand()->getFlags());
22462 return DAG.getStore(Chain, dl: DL, Val: NewL, Ptr: Store->getBasePtr(),
22463 PtrInfo: Store->getPointerInfo(), Alignment: Store->getBaseAlign(),
22464 MMOFlags: Store->getMemOperand()->getFlags());
22465 }
22466 }
22467
22468 // Combine store of vmv.x.s/vfmv.f.s to vse with VL of 1.
22469 // vfmv.f.s is represented as extract element from 0. Match it late to avoid
22470 // any illegal types.
22471 if ((Val.getOpcode() == RISCVISD::VMV_X_S ||
22472 (DCI.isAfterLegalizeDAG() &&
22473 Val.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
22474 isNullConstant(V: Val.getOperand(i: 1)))) &&
22475 Val.hasOneUse()) {
22476 SDValue Src = Val.getOperand(i: 0);
22477 MVT VecVT = Src.getSimpleValueType();
22478 // VecVT should be scalable and memory VT should match the element type.
22479 if (!Store->isIndexed() && VecVT.isScalableVector() &&
22480 MemVT == VecVT.getVectorElementType()) {
22481 SDLoc DL(N);
22482 MVT MaskVT = getMaskTypeFor(VecVT);
22483 return DAG.getStoreVP(
22484 Chain: Store->getChain(), dl: DL, Val: Src, Ptr: Store->getBasePtr(), Offset: Store->getOffset(),
22485 Mask: DAG.getConstant(Val: 1, DL, VT: MaskVT),
22486 EVL: DAG.getConstant(Val: 1, DL, VT: Subtarget.getXLenVT()), MemVT,
22487 MMO: Store->getMemOperand(), AM: Store->getAddressingMode(),
22488 IsTruncating: Store->isTruncatingStore(), /*IsCompress*/ IsCompressing: false);
22489 }
22490 }
22491
22492 break;
22493 }
22494 case ISD::SPLAT_VECTOR: {
22495 EVT VT = N->getValueType(ResNo: 0);
22496 // Only perform this combine on legal MVT types.
22497 if (!isTypeLegal(VT))
22498 break;
22499 if (auto Gather = matchSplatAsGather(SplatVal: N->getOperand(Num: 0), VT: VT.getSimpleVT(), DL: N,
22500 DAG, Subtarget))
22501 return Gather;
22502 break;
22503 }
22504 case ISD::BUILD_VECTOR:
22505 if (SDValue V = performBUILD_VECTORCombine(N, DAG, Subtarget, TLI: *this))
22506 return V;
22507 break;
22508 case ISD::CONCAT_VECTORS:
22509 if (SDValue V = performCONCAT_VECTORSCombine(N, DAG, Subtarget, TLI: *this))
22510 return V;
22511 break;
22512 case ISD::VECTOR_SHUFFLE:
22513 if (SDValue V = performVECTOR_SHUFFLECombine(N, DAG, Subtarget, TLI: *this))
22514 return V;
22515 break;
22516 case ISD::INSERT_VECTOR_ELT:
22517 if (SDValue V = performINSERT_VECTOR_ELTCombine(N, DAG, Subtarget, TLI: *this))
22518 return V;
22519 break;
22520 case RISCVISD::VFMV_V_F_VL: {
22521 const MVT VT = N->getSimpleValueType(ResNo: 0);
22522 SDValue Passthru = N->getOperand(Num: 0);
22523 SDValue Scalar = N->getOperand(Num: 1);
22524 SDValue VL = N->getOperand(Num: 2);
22525
22526 // If VL is 1, we can use vfmv.s.f.
22527 if (isOneConstant(V: VL))
22528 return DAG.getNode(Opcode: RISCVISD::VFMV_S_F_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
22529 break;
22530 }
22531 case RISCVISD::VMV_V_X_VL: {
22532 const MVT VT = N->getSimpleValueType(ResNo: 0);
22533 SDValue Passthru = N->getOperand(Num: 0);
22534 SDValue Scalar = N->getOperand(Num: 1);
22535 SDValue VL = N->getOperand(Num: 2);
22536
22537 // Tail agnostic VMV.V.X only demands the vector element bitwidth from the
22538 // scalar input.
22539 unsigned ScalarSize = Scalar.getValueSizeInBits();
22540 unsigned EltWidth = VT.getScalarSizeInBits();
22541 if (ScalarSize > EltWidth && Passthru.isUndef())
22542 if (SimplifyDemandedLowBitsHelper(1, EltWidth))
22543 return SDValue(N, 0);
22544
22545 // If VL is 1 and the scalar value won't benefit from immediate, we can
22546 // use vmv.s.x.
22547 ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: Scalar);
22548 if (isOneConstant(V: VL) &&
22549 (!Const || Const->isZero() ||
22550 !Const->getAPIntValue().sextOrTrunc(width: EltWidth).isSignedIntN(N: 5)))
22551 return DAG.getNode(Opcode: RISCVISD::VMV_S_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
22552
22553 break;
22554 }
22555 case RISCVISD::VFMV_S_F_VL: {
22556 SDValue Src = N->getOperand(Num: 1);
22557 // Try to remove vector->scalar->vector if the scalar->vector is inserting
22558 // into an undef vector.
22559 // TODO: Could use a vslide or vmv.v.v for non-undef.
22560 if (N->getOperand(Num: 0).isUndef() &&
22561 Src.getOpcode() == ISD::EXTRACT_VECTOR_ELT &&
22562 isNullConstant(V: Src.getOperand(i: 1)) &&
22563 Src.getOperand(i: 0).getValueType().isScalableVector()) {
22564 EVT VT = N->getValueType(ResNo: 0);
22565 SDValue EVSrc = Src.getOperand(i: 0);
22566 EVT EVSrcVT = EVSrc.getValueType();
22567 assert(EVSrcVT.getVectorElementType() == VT.getVectorElementType());
22568 // Widths match, just return the original vector.
22569 if (EVSrcVT == VT)
22570 return EVSrc;
22571 SDLoc DL(N);
22572 // Width is narrower, using insert_subvector.
22573 if (EVSrcVT.getVectorMinNumElements() < VT.getVectorMinNumElements()) {
22574 return DAG.getNode(Opcode: ISD::INSERT_SUBVECTOR, DL, VT, N1: DAG.getUNDEF(VT),
22575 N2: EVSrc,
22576 N3: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()));
22577 }
22578 // Width is wider, using extract_subvector.
22579 return DAG.getNode(Opcode: ISD::EXTRACT_SUBVECTOR, DL, VT, N1: EVSrc,
22580 N2: DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT()));
22581 }
22582 [[fallthrough]];
22583 }
22584 case RISCVISD::VMV_S_X_VL: {
22585 const MVT VT = N->getSimpleValueType(ResNo: 0);
22586 SDValue Passthru = N->getOperand(Num: 0);
22587 SDValue Scalar = N->getOperand(Num: 1);
22588 SDValue VL = N->getOperand(Num: 2);
22589
22590 // The vmv.s.x instruction copies the scalar integer register to element 0
22591 // of the destination vector register. If SEW < XLEN, the least-significant
22592 // bits are copied and the upper XLEN-SEW bits are ignored.
22593 unsigned ScalarSize = Scalar.getValueSizeInBits();
22594 unsigned EltWidth = VT.getScalarSizeInBits();
22595 if (ScalarSize > EltWidth && SimplifyDemandedLowBitsHelper(1, EltWidth))
22596 return SDValue(N, 0);
22597
22598 if (Scalar.getOpcode() == RISCVISD::VMV_X_S && Passthru.isUndef() &&
22599 Scalar.getOperand(i: 0).getValueType() == N->getValueType(ResNo: 0))
22600 return Scalar.getOperand(i: 0);
22601
22602 // Use M1 or smaller to avoid over constraining register allocation
22603 const MVT M1VT = RISCVTargetLowering::getM1VT(VT);
22604 if (M1VT.bitsLT(VT)) {
22605 SDValue M1Passthru = DAG.getExtractSubvector(DL, VT: M1VT, Vec: Passthru, Idx: 0);
22606 SDValue Result =
22607 DAG.getNode(Opcode: N->getOpcode(), DL, VT: M1VT, N1: M1Passthru, N2: Scalar, N3: VL);
22608 Result = DAG.getInsertSubvector(DL, Vec: Passthru, SubVec: Result, Idx: 0);
22609 return Result;
22610 }
22611
22612 // We use a vmv.v.i if possible. We limit this to LMUL1. LMUL2 or
22613 // higher would involve overly constraining the register allocator for
22614 // no purpose.
22615 if (ConstantSDNode *Const = dyn_cast<ConstantSDNode>(Val&: Scalar);
22616 Const && !Const->isZero() && isInt<5>(x: Const->getSExtValue()) &&
22617 VT.bitsLE(VT: RISCVTargetLowering::getM1VT(VT)) && Passthru.isUndef())
22618 return DAG.getNode(Opcode: RISCVISD::VMV_V_X_VL, DL, VT, N1: Passthru, N2: Scalar, N3: VL);
22619
22620 break;
22621 }
22622 case RISCVISD::VMV_X_S: {
22623 SDValue Vec = N->getOperand(Num: 0);
22624 MVT VecVT = N->getOperand(Num: 0).getSimpleValueType();
22625 const MVT M1VT = RISCVTargetLowering::getM1VT(VT: VecVT);
22626 if (M1VT.bitsLT(VT: VecVT)) {
22627 Vec = DAG.getExtractSubvector(DL, VT: M1VT, Vec, Idx: 0);
22628 return DAG.getNode(Opcode: RISCVISD::VMV_X_S, DL, VT: N->getValueType(ResNo: 0), Operand: Vec);
22629 }
22630 break;
22631 }
22632 case ISD::INTRINSIC_VOID:
22633 case ISD::INTRINSIC_W_CHAIN:
22634 case ISD::INTRINSIC_WO_CHAIN: {
22635 unsigned IntOpNo = N->getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
22636 unsigned IntNo = N->getConstantOperandVal(Num: IntOpNo);
22637 switch (IntNo) {
22638 // By default we do not combine any intrinsic.
22639 default:
22640 return SDValue();
22641 case Intrinsic::riscv_vcpop:
22642 case Intrinsic::riscv_vcpop_mask:
22643 case Intrinsic::riscv_vfirst:
22644 case Intrinsic::riscv_vfirst_mask: {
22645 SDValue VL = N->getOperand(Num: 2);
22646 if (IntNo == Intrinsic::riscv_vcpop_mask ||
22647 IntNo == Intrinsic::riscv_vfirst_mask)
22648 VL = N->getOperand(Num: 3);
22649 if (!isNullConstant(V: VL))
22650 return SDValue();
22651 // If VL is 0, vcpop -> li 0, vfirst -> li -1.
22652 SDLoc DL(N);
22653 EVT VT = N->getValueType(ResNo: 0);
22654 if (IntNo == Intrinsic::riscv_vfirst ||
22655 IntNo == Intrinsic::riscv_vfirst_mask)
22656 return DAG.getAllOnesConstant(DL, VT);
22657 return DAG.getConstant(Val: 0, DL, VT);
22658 }
22659 case Intrinsic::riscv_vsseg2_mask:
22660 case Intrinsic::riscv_vsseg3_mask:
22661 case Intrinsic::riscv_vsseg4_mask:
22662 case Intrinsic::riscv_vsseg5_mask:
22663 case Intrinsic::riscv_vsseg6_mask:
22664 case Intrinsic::riscv_vsseg7_mask:
22665 case Intrinsic::riscv_vsseg8_mask: {
22666 SDValue Tuple = N->getOperand(Num: 2);
22667 unsigned NF = Tuple.getValueType().getRISCVVectorTupleNumFields();
22668
22669 if (Subtarget.hasOptimizedSegmentLoadStore(NF) || !Tuple.hasOneUse() ||
22670 Tuple.getOpcode() != RISCVISD::TUPLE_INSERT ||
22671 !Tuple.getOperand(i: 0).isUndef())
22672 return SDValue();
22673
22674 SDValue Val = Tuple.getOperand(i: 1);
22675 unsigned Idx = Tuple.getConstantOperandVal(i: 2);
22676
22677 unsigned SEW = Val.getValueType().getScalarSizeInBits();
22678 assert(Log2_64(SEW) == N->getConstantOperandVal(6) &&
22679 "Type mismatch without bitcast?");
22680 unsigned Stride = SEW / 8 * NF;
22681 unsigned Offset = SEW / 8 * Idx;
22682
22683 SDValue Ops[] = {
22684 /*Chain=*/N->getOperand(Num: 0),
22685 /*IntID=*/
22686 DAG.getTargetConstant(Val: Intrinsic::riscv_vsse_mask, DL, VT: XLenVT),
22687 /*StoredVal=*/Val,
22688 /*Ptr=*/
22689 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: N->getOperand(Num: 3),
22690 N2: DAG.getConstant(Val: Offset, DL, VT: XLenVT)),
22691 /*Stride=*/DAG.getConstant(Val: Stride, DL, VT: XLenVT),
22692 /*Mask=*/N->getOperand(Num: 4),
22693 /*VL=*/N->getOperand(Num: 5)};
22694
22695 auto *OldMemSD = cast<MemIntrinsicSDNode>(Val: N);
22696 // Match getTgtMemIntrinsic for non-unit stride case
22697 EVT MemVT = OldMemSD->getMemoryVT().getScalarType();
22698 MachineFunction &MF = DAG.getMachineFunction();
22699 MachineMemOperand *MMO = MF.getMachineMemOperand(
22700 MMO: OldMemSD->getMemOperand(), Offset, Size: MemoryLocation::UnknownSize);
22701
22702 SDVTList VTs = DAG.getVTList(VT: MVT::Other);
22703 return DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_VOID, dl: DL, VTList: VTs, Ops, MemVT,
22704 MMO);
22705 }
22706 }
22707 }
22708 case ISD::EXPERIMENTAL_VP_REVERSE:
22709 return performVP_REVERSECombine(N, DAG, Subtarget);
22710 case ISD::VP_STORE:
22711 return performVP_STORECombine(N, DAG, Subtarget);
22712 case ISD::BITCAST: {
22713 assert(Subtarget.useRVVForFixedLengthVectors());
22714 SDValue N0 = N->getOperand(Num: 0);
22715 EVT VT = N->getValueType(ResNo: 0);
22716 EVT SrcVT = N0.getValueType();
22717 if (VT.isRISCVVectorTuple() && N0->getOpcode() == ISD::SPLAT_VECTOR) {
22718 unsigned NF = VT.getRISCVVectorTupleNumFields();
22719 unsigned NumScalElts = VT.getSizeInBits().getKnownMinValue() / (NF * 8);
22720 SDValue EltVal = DAG.getConstant(Val: 0, DL, VT: Subtarget.getXLenVT());
22721 MVT ScalTy = MVT::getScalableVectorVT(VT: MVT::getIntegerVT(BitWidth: 8), NumElements: NumScalElts);
22722
22723 SDValue Splat = DAG.getNode(Opcode: ISD::SPLAT_VECTOR, DL, VT: ScalTy, Operand: EltVal);
22724
22725 SDValue Result = DAG.getUNDEF(VT);
22726 for (unsigned i = 0; i < NF; ++i)
22727 Result = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT, N1: Result, N2: Splat,
22728 N3: DAG.getTargetConstant(Val: i, DL, VT: MVT::i32));
22729 return Result;
22730 }
22731 // If this is a bitcast between a MVT::v4i1/v2i1/v1i1 and an illegal integer
22732 // type, widen both sides to avoid a trip through memory.
22733 if ((SrcVT == MVT::v1i1 || SrcVT == MVT::v2i1 || SrcVT == MVT::v4i1) &&
22734 VT.isScalarInteger()) {
22735 unsigned NumConcats = 8 / SrcVT.getVectorNumElements();
22736 SmallVector<SDValue, 4> Ops(NumConcats, DAG.getUNDEF(VT: SrcVT));
22737 Ops[0] = N0;
22738 SDLoc DL(N);
22739 N0 = DAG.getNode(Opcode: ISD::CONCAT_VECTORS, DL, VT: MVT::v8i1, Ops);
22740 N0 = DAG.getBitcast(VT: MVT::i8, V: N0);
22741 return DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT, Operand: N0);
22742 }
22743
22744 return SDValue();
22745 }
22746 case ISD::VECREDUCE_ADD:
22747 if (SDValue V = performVECREDUCECombine(N, DAG, Subtarget, TLI: *this))
22748 return V;
22749 [[fallthrough]];
22750 case ISD::CTPOP:
22751 if (SDValue V = combineToVCPOP(N, DAG, Subtarget))
22752 return V;
22753 break;
22754 case RISCVISD::VRGATHER_VX_VL: {
22755 // Note this assumes that out of bounds indices produce poison
22756 // and can thus be replaced without having to prove them inbounds..
22757 EVT VT = N->getValueType(ResNo: 0);
22758 SDValue Src = N->getOperand(Num: 0);
22759 SDValue Idx = N->getOperand(Num: 1);
22760 SDValue Passthru = N->getOperand(Num: 2);
22761 SDValue VL = N->getOperand(Num: 4);
22762
22763 // Warning: Unlike most cases we strip an insert_subvector, this one
22764 // does not require the first operand to be undef.
22765 if (Src.getOpcode() == ISD::INSERT_SUBVECTOR &&
22766 isNullConstant(V: Src.getOperand(i: 2)))
22767 Src = Src.getOperand(i: 1);
22768
22769 switch (Src.getOpcode()) {
22770 default:
22771 break;
22772 case RISCVISD::VMV_V_X_VL:
22773 case RISCVISD::VFMV_V_F_VL:
22774 // Drop a redundant vrgather_vx.
22775 // TODO: Remove the type restriction if we find a motivating
22776 // test case?
22777 if (Passthru.isUndef() && VL == Src.getOperand(i: 2) &&
22778 Src.getValueType() == VT)
22779 return Src;
22780 break;
22781 case RISCVISD::VMV_S_X_VL:
22782 case RISCVISD::VFMV_S_F_VL:
22783 // If this use only demands lane zero from the source vmv.s.x, and
22784 // doesn't have a passthru, then this vrgather.vi/vx is equivalent to
22785 // a vmv.v.x. Note that there can be other uses of the original
22786 // vmv.s.x and thus we can't eliminate it. (vfmv.s.f is analogous)
22787 if (isNullConstant(V: Idx) && Passthru.isUndef() &&
22788 VL == Src.getOperand(i: 2)) {
22789 unsigned Opc =
22790 VT.isFloatingPoint() ? RISCVISD::VFMV_V_F_VL : RISCVISD::VMV_V_X_VL;
22791 return DAG.getNode(Opcode: Opc, DL, VT, N1: DAG.getUNDEF(VT), N2: Src.getOperand(i: 1),
22792 N3: VL);
22793 }
22794 break;
22795 }
22796 break;
22797 }
22798 case RISCVISD::TUPLE_EXTRACT: {
22799 EVT VT = N->getValueType(ResNo: 0);
22800 SDValue Tuple = N->getOperand(Num: 0);
22801 unsigned Idx = N->getConstantOperandVal(Num: 1);
22802 if (!Tuple.hasOneUse() || Tuple.getOpcode() != ISD::INTRINSIC_W_CHAIN)
22803 break;
22804
22805 unsigned NF = 0;
22806 switch (Tuple.getConstantOperandVal(i: 1)) {
22807 default:
22808 break;
22809 case Intrinsic::riscv_vlseg2_mask:
22810 case Intrinsic::riscv_vlseg3_mask:
22811 case Intrinsic::riscv_vlseg4_mask:
22812 case Intrinsic::riscv_vlseg5_mask:
22813 case Intrinsic::riscv_vlseg6_mask:
22814 case Intrinsic::riscv_vlseg7_mask:
22815 case Intrinsic::riscv_vlseg8_mask:
22816 NF = Tuple.getValueType().getRISCVVectorTupleNumFields();
22817 break;
22818 }
22819
22820 if (!NF || Subtarget.hasOptimizedSegmentLoadStore(NF))
22821 break;
22822
22823 unsigned SEW = VT.getScalarSizeInBits();
22824 assert(Log2_64(SEW) == Tuple.getConstantOperandVal(7) &&
22825 "Type mismatch without bitcast?");
22826 unsigned Stride = SEW / 8 * NF;
22827 unsigned Offset = SEW / 8 * Idx;
22828
22829 SDValue Passthru = Tuple.getOperand(i: 2);
22830 if (Passthru.isUndef())
22831 Passthru = DAG.getUNDEF(VT);
22832 else
22833 Passthru = DAG.getNode(Opcode: RISCVISD::TUPLE_EXTRACT, DL, VT, N1: Passthru,
22834 N2: N->getOperand(Num: 1));
22835
22836 SDValue Ops[] = {
22837 /*Chain=*/Tuple.getOperand(i: 0),
22838 /*IntID=*/DAG.getTargetConstant(Val: Intrinsic::riscv_vlse_mask, DL, VT: XLenVT),
22839 /*Passthru=*/Passthru,
22840 /*Ptr=*/
22841 DAG.getNode(Opcode: ISD::ADD, DL, VT: XLenVT, N1: Tuple.getOperand(i: 3),
22842 N2: DAG.getConstant(Val: Offset, DL, VT: XLenVT)),
22843 /*Stride=*/DAG.getConstant(Val: Stride, DL, VT: XLenVT),
22844 /*Mask=*/Tuple.getOperand(i: 4),
22845 /*VL=*/Tuple.getOperand(i: 5),
22846 /*Policy=*/Tuple.getOperand(i: 6)};
22847
22848 auto *TupleMemSD = cast<MemIntrinsicSDNode>(Val&: Tuple);
22849 // Match getTgtMemIntrinsic for non-unit stride case
22850 EVT MemVT = TupleMemSD->getMemoryVT().getScalarType();
22851 MachineFunction &MF = DAG.getMachineFunction();
22852 MachineMemOperand *MMO = MF.getMachineMemOperand(
22853 MMO: TupleMemSD->getMemOperand(), Offset, Size: MemoryLocation::UnknownSize);
22854
22855 SDVTList VTs = DAG.getVTList(VTs: {VT, MVT::Other});
22856 SDValue Result = DAG.getMemIntrinsicNode(Opcode: ISD::INTRINSIC_W_CHAIN, dl: DL, VTList: VTs,
22857 Ops, MemVT, MMO);
22858 DAG.ReplaceAllUsesOfValueWith(From: Tuple.getValue(R: 1), To: Result.getValue(R: 1));
22859 return Result.getValue(R: 0);
22860 }
22861 case RISCVISD::TUPLE_INSERT: {
22862 // tuple_insert tuple, undef, idx -> tuple
22863 if (N->getOperand(Num: 1).isUndef())
22864 return N->getOperand(Num: 0);
22865 break;
22866 }
22867 case RISCVISD::VMERGE_VL: {
22868 // vmerge_vl allones, x, y, passthru, vl -> vmv_v_v passthru, x, vl
22869 SDValue Mask = N->getOperand(Num: 0);
22870 SDValue True = N->getOperand(Num: 1);
22871 SDValue Passthru = N->getOperand(Num: 3);
22872 SDValue VL = N->getOperand(Num: 4);
22873
22874 // Fixed vectors are wrapped in scalable containers, unwrap them.
22875 using namespace SDPatternMatch;
22876 SDValue SubVec;
22877 if (sd_match(N: Mask, P: m_InsertSubvector(Base: m_Undef(), Sub: m_Value(N&: SubVec), Idx: m_Zero())))
22878 Mask = SubVec;
22879
22880 if (!isOneOrOneSplat(V: Mask))
22881 break;
22882
22883 return DAG.getNode(Opcode: RISCVISD::VMV_V_V_VL, DL: SDLoc(N), VT: N->getValueType(ResNo: 0),
22884 N1: Passthru, N2: True, N3: VL);
22885 }
22886 case RISCVISD::VMV_V_V_VL: {
22887 // vmv_v_v passthru, splat(x), vl -> vmv_v_x passthru, x, vl
22888 SDValue Passthru = N->getOperand(Num: 0);
22889 SDValue Src = N->getOperand(Num: 1);
22890 SDValue VL = N->getOperand(Num: 2);
22891
22892 // Fixed vectors are wrapped in scalable containers, unwrap them.
22893 using namespace SDPatternMatch;
22894 SDValue SubVec;
22895 if (sd_match(N: Src, P: m_InsertSubvector(Base: m_Undef(), Sub: m_Value(N&: SubVec), Idx: m_Zero())))
22896 Src = SubVec;
22897
22898 SDValue SplatVal = DAG.getSplatValue(V: Src, /*LegalTypes=*/true);
22899 if (!SplatVal)
22900 break;
22901 MVT VT = N->getSimpleValueType(ResNo: 0);
22902 return lowerScalarSplat(Passthru, Scalar: SplatVal, VL, VT, DL: SDLoc(N), DAG,
22903 Subtarget);
22904 }
22905 case RISCVISD::VSLIDEDOWN_VL:
22906 case RISCVISD::VSLIDEUP_VL:
22907 if (N->getOperand(Num: 1)->isUndef())
22908 return N->getOperand(Num: 0);
22909 break;
22910 case RISCVISD::VSLIDE1UP_VL:
22911 case RISCVISD::VFSLIDE1UP_VL: {
22912 using namespace SDPatternMatch;
22913 SDValue SrcVec;
22914 SDLoc DL(N);
22915 MVT VT = N->getSimpleValueType(ResNo: 0);
22916 // If the scalar we're sliding in was extracted from the first element of a
22917 // vector, we can use that vector as the passthru in a normal slideup of 1.
22918 // This saves us an extract_element instruction (i.e. vfmv.f.s, vmv.x.s).
22919 if (!N->getOperand(Num: 0).isUndef() ||
22920 !sd_match(N: N->getOperand(Num: 2),
22921 P: m_AnyOf(preds: m_ExtractElt(Vec: m_Value(N&: SrcVec), Idx: m_Zero()),
22922 preds: m_Node(Opcode: RISCVISD::VMV_X_S, preds: m_Value(N&: SrcVec)))))
22923 break;
22924
22925 MVT SrcVecVT = SrcVec.getSimpleValueType();
22926 if (SrcVecVT.getVectorElementType() != VT.getVectorElementType())
22927 break;
22928 // Adapt the value type of source vector.
22929 if (SrcVecVT.isFixedLengthVector()) {
22930 SrcVecVT = getContainerForFixedLengthVector(VT: SrcVecVT);
22931 SrcVec = convertToScalableVector(VT: SrcVecVT, V: SrcVec, DAG, Subtarget);
22932 }
22933 if (SrcVecVT.getVectorMinNumElements() < VT.getVectorMinNumElements())
22934 SrcVec = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT), SubVec: SrcVec, Idx: 0);
22935 else
22936 SrcVec = DAG.getExtractSubvector(DL, VT, Vec: SrcVec, Idx: 0);
22937
22938 return getVSlideup(DAG, Subtarget, DL, VT, Passthru: SrcVec, Op: N->getOperand(Num: 1),
22939 Offset: DAG.getConstant(Val: 1, DL, VT: XLenVT), Mask: N->getOperand(Num: 3),
22940 VL: N->getOperand(Num: 4));
22941 }
22942 }
22943
22944 return SDValue();
22945}
22946
22947bool RISCVTargetLowering::shouldTransformSignedTruncationCheck(
22948 EVT XVT, unsigned KeptBits) const {
22949 // For vectors, we don't have a preference..
22950 if (XVT.isVector())
22951 return false;
22952
22953 if (XVT != MVT::i32 && XVT != MVT::i64)
22954 return false;
22955
22956 // We can use sext.w for RV64 or an srai 31 on RV32.
22957 if (KeptBits == 32 || KeptBits == 64)
22958 return true;
22959
22960 // With Zbb we can use sext.h/sext.b.
22961 return Subtarget.hasStdExtZbb() &&
22962 ((KeptBits == 8 && XVT == MVT::i64 && !Subtarget.is64Bit()) ||
22963 KeptBits == 16);
22964}
22965
22966bool RISCVTargetLowering::isDesirableToCommuteWithShift(
22967 const SDNode *N, CombineLevel Level) const {
22968 assert((N->getOpcode() == ISD::SHL || N->getOpcode() == ISD::SRA ||
22969 N->getOpcode() == ISD::SRL) &&
22970 "Expected shift op");
22971
22972 // The following folds are only desirable if `(OP _, c1 << c2)` can be
22973 // materialised in fewer instructions than `(OP _, c1)`:
22974 //
22975 // (shl (add x, c1), c2) -> (add (shl x, c2), c1 << c2)
22976 // (shl (or x, c1), c2) -> (or (shl x, c2), c1 << c2)
22977 SDValue N0 = N->getOperand(Num: 0);
22978 EVT Ty = N0.getValueType();
22979
22980 // LD/ST will optimize constant Offset extraction, so when AddNode is used by
22981 // LD/ST, it can still complete the folding optimization operation performed
22982 // above.
22983 auto isUsedByLdSt = [](const SDNode *X, const SDNode *User) {
22984 for (SDNode *Use : X->users()) {
22985 // This use is the one we're on right now. Skip it
22986 if (Use == User || Use->getOpcode() == ISD::SELECT)
22987 continue;
22988 if (!isa<StoreSDNode>(Val: Use) && !isa<LoadSDNode>(Val: Use))
22989 return false;
22990 }
22991 return true;
22992 };
22993
22994 if (Ty.isScalarInteger() &&
22995 (N0.getOpcode() == ISD::ADD || N0.getOpcode() == ISD::OR)) {
22996 if (N0.getOpcode() == ISD::ADD && !N0->hasOneUse())
22997 return isUsedByLdSt(N0.getNode(), N);
22998
22999 auto *C1 = dyn_cast<ConstantSDNode>(Val: N0->getOperand(Num: 1));
23000 auto *C2 = dyn_cast<ConstantSDNode>(Val: N->getOperand(Num: 1));
23001
23002 // Bail if we might break a sh{1,2,3}add/qc.shladd pattern.
23003 if (C2 && Subtarget.hasShlAdd(ShAmt: C2->getZExtValue()) && N->hasOneUse() &&
23004 N->user_begin()->getOpcode() == ISD::ADD &&
23005 !isUsedByLdSt(*N->user_begin(), nullptr) &&
23006 !isa<ConstantSDNode>(Val: N->user_begin()->getOperand(Num: 1)))
23007 return false;
23008
23009 if (C1 && C2) {
23010 const APInt &C1Int = C1->getAPIntValue();
23011 APInt ShiftedC1Int = C1Int << C2->getAPIntValue();
23012
23013 // We can materialise `c1 << c2` into an add immediate, so it's "free",
23014 // and the combine should happen, to potentially allow further combines
23015 // later.
23016 if (ShiftedC1Int.getSignificantBits() <= 64 &&
23017 isLegalAddImmediate(Imm: ShiftedC1Int.getSExtValue()))
23018 return true;
23019
23020 // We can materialise `c1` in an add immediate, so it's "free", and the
23021 // combine should be prevented.
23022 if (C1Int.getSignificantBits() <= 64 &&
23023 isLegalAddImmediate(Imm: C1Int.getSExtValue()))
23024 return false;
23025
23026 // Neither constant will fit into an immediate, so find materialisation
23027 // costs.
23028 int C1Cost =
23029 RISCVMatInt::getIntMatCost(Val: C1Int, Size: Ty.getSizeInBits(), STI: Subtarget,
23030 /*CompressionCost*/ true);
23031 int ShiftedC1Cost = RISCVMatInt::getIntMatCost(
23032 Val: ShiftedC1Int, Size: Ty.getSizeInBits(), STI: Subtarget,
23033 /*CompressionCost*/ true);
23034
23035 // Materialising `c1` is cheaper than materialising `c1 << c2`, so the
23036 // combine should be prevented.
23037 if (C1Cost < ShiftedC1Cost)
23038 return false;
23039 }
23040 }
23041
23042 if (!N0->hasOneUse())
23043 return false;
23044
23045 if (N0->getOpcode() == ISD::SIGN_EXTEND &&
23046 N0->getOperand(Num: 0)->getOpcode() == ISD::ADD &&
23047 !N0->getOperand(Num: 0)->hasOneUse())
23048 return isUsedByLdSt(N0->getOperand(Num: 0).getNode(), N0.getNode());
23049
23050 return true;
23051}
23052
23053bool RISCVTargetLowering::targetShrinkDemandedConstant(
23054 SDValue Op, const APInt &DemandedBits, const APInt &DemandedElts,
23055 TargetLoweringOpt &TLO) const {
23056 // Delay this optimization as late as possible.
23057 if (!TLO.LegalOps)
23058 return false;
23059
23060 EVT VT = Op.getValueType();
23061 if (VT.isVector())
23062 return false;
23063
23064 unsigned Opcode = Op.getOpcode();
23065 if (Opcode != ISD::AND && Opcode != ISD::OR && Opcode != ISD::XOR)
23066 return false;
23067
23068 ConstantSDNode *C = dyn_cast<ConstantSDNode>(Val: Op.getOperand(i: 1));
23069 if (!C)
23070 return false;
23071
23072 const APInt &Mask = C->getAPIntValue();
23073
23074 // Clear all non-demanded bits initially.
23075 APInt ShrunkMask = Mask & DemandedBits;
23076
23077 // Try to make a smaller immediate by setting undemanded bits.
23078
23079 APInt ExpandedMask = Mask | ~DemandedBits;
23080
23081 auto IsLegalMask = [ShrunkMask, ExpandedMask](const APInt &Mask) -> bool {
23082 return ShrunkMask.isSubsetOf(RHS: Mask) && Mask.isSubsetOf(RHS: ExpandedMask);
23083 };
23084 auto UseMask = [Mask, Op, &TLO](const APInt &NewMask) -> bool {
23085 if (NewMask == Mask)
23086 return true;
23087 SDLoc DL(Op);
23088 SDValue NewC = TLO.DAG.getConstant(Val: NewMask, DL, VT: Op.getValueType());
23089 SDValue NewOp = TLO.DAG.getNode(Opcode: Op.getOpcode(), DL, VT: Op.getValueType(),
23090 N1: Op.getOperand(i: 0), N2: NewC);
23091 return TLO.CombineTo(O: Op, N: NewOp);
23092 };
23093
23094 // If the shrunk mask fits in sign extended 12 bits, let the target
23095 // independent code apply it.
23096 if (ShrunkMask.isSignedIntN(N: 12))
23097 return false;
23098
23099 // And has a few special cases for zext.
23100 if (Opcode == ISD::AND) {
23101 // Preserve (and X, 0xffff), if zext.h exists use zext.h,
23102 // otherwise use SLLI + SRLI.
23103 APInt NewMask = APInt(Mask.getBitWidth(), 0xffff);
23104 if (IsLegalMask(NewMask))
23105 return UseMask(NewMask);
23106
23107 // Try to preserve (and X, 0xffffffff), the (zext_inreg X, i32) pattern.
23108 if (VT == MVT::i64) {
23109 APInt NewMask = APInt(64, 0xffffffff);
23110 if (IsLegalMask(NewMask))
23111 return UseMask(NewMask);
23112 }
23113 }
23114
23115 // For the remaining optimizations, we need to be able to make a negative
23116 // number through a combination of mask and undemanded bits.
23117 if (!ExpandedMask.isNegative())
23118 return false;
23119
23120 // What is the fewest number of bits we need to represent the negative number.
23121 unsigned MinSignedBits = ExpandedMask.getSignificantBits();
23122
23123 // Try to make a 12 bit negative immediate. If that fails try to make a 32
23124 // bit negative immediate unless the shrunk immediate already fits in 32 bits.
23125 // If we can't create a simm12, we shouldn't change opaque constants.
23126 APInt NewMask = ShrunkMask;
23127 if (MinSignedBits <= 12)
23128 NewMask.setBitsFrom(11);
23129 else if (!C->isOpaque() && MinSignedBits <= 32 && !ShrunkMask.isSignedIntN(N: 32))
23130 NewMask.setBitsFrom(31);
23131 else
23132 return false;
23133
23134 // Check that our new mask is a subset of the demanded mask.
23135 assert(IsLegalMask(NewMask));
23136 return UseMask(NewMask);
23137}
23138
23139static uint64_t computeGREVOrGORC(uint64_t x, unsigned ShAmt, bool IsGORC) {
23140 static const uint64_t GREVMasks[] = {
23141 0x5555555555555555ULL, 0x3333333333333333ULL, 0x0F0F0F0F0F0F0F0FULL,
23142 0x00FF00FF00FF00FFULL, 0x0000FFFF0000FFFFULL, 0x00000000FFFFFFFFULL};
23143
23144 for (unsigned Stage = 0; Stage != 6; ++Stage) {
23145 unsigned Shift = 1 << Stage;
23146 if (ShAmt & Shift) {
23147 uint64_t Mask = GREVMasks[Stage];
23148 uint64_t Res = ((x & Mask) << Shift) | ((x >> Shift) & Mask);
23149 if (IsGORC)
23150 Res |= x;
23151 x = Res;
23152 }
23153 }
23154
23155 return x;
23156}
23157
23158void RISCVTargetLowering::computeKnownBitsForTargetNode(const SDValue Op,
23159 KnownBits &Known,
23160 const APInt &DemandedElts,
23161 const SelectionDAG &DAG,
23162 unsigned Depth) const {
23163 unsigned BitWidth = Known.getBitWidth();
23164 unsigned Opc = Op.getOpcode();
23165 assert((Opc >= ISD::BUILTIN_OP_END ||
23166 Opc == ISD::INTRINSIC_WO_CHAIN ||
23167 Opc == ISD::INTRINSIC_W_CHAIN ||
23168 Opc == ISD::INTRINSIC_VOID) &&
23169 "Should use MaskedValueIsZero if you don't know whether Op"
23170 " is a target node!");
23171
23172 Known.resetAll();
23173 switch (Opc) {
23174 default: break;
23175 case RISCVISD::SELECT_CC: {
23176 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 4), Depth: Depth + 1);
23177 // If we don't know any bits, early out.
23178 if (Known.isUnknown())
23179 break;
23180 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 3), Depth: Depth + 1);
23181
23182 // Only known if known in both the LHS and RHS.
23183 Known = Known.intersectWith(RHS: Known2);
23184 break;
23185 }
23186 case RISCVISD::VCPOP_VL: {
23187 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), Depth: Depth + 1);
23188 Known.Zero.setBitsFrom(Known2.countMaxActiveBits());
23189 break;
23190 }
23191 case RISCVISD::CZERO_EQZ:
23192 case RISCVISD::CZERO_NEZ:
23193 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
23194 // Result is either all zero or operand 0. We can propagate zeros, but not
23195 // ones.
23196 Known.One.clearAllBits();
23197 break;
23198 case RISCVISD::REMUW: {
23199 KnownBits Known2;
23200 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23201 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
23202 // We only care about the lower 32 bits.
23203 Known = KnownBits::urem(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 32));
23204 // Restore the original width by sign extending.
23205 Known = Known.sext(BitWidth);
23206 break;
23207 }
23208 case RISCVISD::DIVUW: {
23209 KnownBits Known2;
23210 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23211 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
23212 // We only care about the lower 32 bits.
23213 Known = KnownBits::udiv(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 32));
23214 // Restore the original width by sign extending.
23215 Known = Known.sext(BitWidth);
23216 break;
23217 }
23218 case RISCVISD::SLLW: {
23219 KnownBits Known2;
23220 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23221 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
23222 Known = KnownBits::shl(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
23223 // Restore the original width by sign extending.
23224 Known = Known.sext(BitWidth);
23225 break;
23226 }
23227 case RISCVISD::SRLW: {
23228 KnownBits Known2;
23229 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23230 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
23231 Known = KnownBits::lshr(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
23232 // Restore the original width by sign extending.
23233 Known = Known.sext(BitWidth);
23234 break;
23235 }
23236 case RISCVISD::SRAW: {
23237 KnownBits Known2;
23238 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23239 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 1), DemandedElts, Depth: Depth + 1);
23240 Known = KnownBits::ashr(LHS: Known.trunc(BitWidth: 32), RHS: Known2.trunc(BitWidth: 5).zext(BitWidth: 32));
23241 // Restore the original width by sign extending.
23242 Known = Known.sext(BitWidth);
23243 break;
23244 }
23245 case RISCVISD::SHL_ADD: {
23246 KnownBits Known2;
23247 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23248 unsigned ShAmt = Op.getConstantOperandVal(i: 1);
23249 Known <<= ShAmt;
23250 Known.Zero.setLowBits(ShAmt); // the <<= operator left these bits unknown
23251 Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 2), DemandedElts, Depth: Depth + 1);
23252 Known = KnownBits::add(LHS: Known, RHS: Known2);
23253 break;
23254 }
23255 case RISCVISD::CTZW: {
23256 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
23257 unsigned PossibleTZ = Known2.trunc(BitWidth: 32).countMaxTrailingZeros();
23258 unsigned LowBits = llvm::bit_width(Value: PossibleTZ);
23259 Known.Zero.setBitsFrom(LowBits);
23260 break;
23261 }
23262 case RISCVISD::CLZW: {
23263 KnownBits Known2 = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
23264 unsigned PossibleLZ = Known2.trunc(BitWidth: 32).countMaxLeadingZeros();
23265 unsigned LowBits = llvm::bit_width(Value: PossibleLZ);
23266 Known.Zero.setBitsFrom(LowBits);
23267 break;
23268 }
23269 case RISCVISD::CLSW: {
23270 // The upper 32 bits are ignored by the instruction, but ComputeNumSignBits
23271 // doesn't give us a way to ignore them. If there are fewer than 33 sign
23272 // bits in the input consider it as having no redundant sign bits. Otherwise
23273 // the lower bound of the result is NumSignBits-33. The maximum value of the
23274 // the result is 31.
23275 unsigned NumSignBits = DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
23276 unsigned MinRedundantSignBits = NumSignBits < 33 ? 0 : NumSignBits - 33;
23277 // Create a ConstantRange [MinRedundantSignBits, 32) and convert it to
23278 // KnownBits.
23279 ConstantRange Range(APInt(BitWidth, MinRedundantSignBits),
23280 APInt(BitWidth, 32));
23281 Known = Range.toKnownBits();
23282 break;
23283 }
23284 case RISCVISD::BREV8:
23285 case RISCVISD::ORC_B: {
23286 // FIXME: This is based on the non-ratified Zbp GREV and GORC where a
23287 // control value of 7 is equivalent to brev8 and orc.b.
23288 Known = DAG.computeKnownBits(Op: Op.getOperand(i: 0), Depth: Depth + 1);
23289 bool IsGORC = Op.getOpcode() == RISCVISD::ORC_B;
23290 // To compute zeros for ORC_B, we need to invert the value and invert it
23291 // back after. This inverting is harmless for BREV8.
23292 Known.Zero =
23293 ~computeGREVOrGORC(x: ~Known.Zero.getZExtValue(), ShAmt: 7, IsGORC);
23294 Known.One = computeGREVOrGORC(x: Known.One.getZExtValue(), ShAmt: 7, IsGORC);
23295 break;
23296 }
23297 case RISCVISD::READ_VLENB: {
23298 // We can use the minimum and maximum VLEN values to bound VLENB. We
23299 // know VLEN must be a power of two.
23300 const unsigned MinVLenB = Subtarget.getRealMinVLen() / 8;
23301 const unsigned MaxVLenB = Subtarget.getRealMaxVLen() / 8;
23302 assert(MinVLenB > 0 && "READ_VLENB without vector extension enabled?");
23303 Known.Zero.setLowBits(Log2_32(Value: MinVLenB));
23304 Known.Zero.setBitsFrom(Log2_32(Value: MaxVLenB)+1);
23305 if (MaxVLenB == MinVLenB)
23306 Known.One.setBit(Log2_32(Value: MinVLenB));
23307 break;
23308 }
23309 case RISCVISD::FCLASS: {
23310 // fclass will only set one of the low 10 bits.
23311 Known.Zero.setBitsFrom(10);
23312 break;
23313 }
23314 case ISD::INTRINSIC_W_CHAIN:
23315 case ISD::INTRINSIC_WO_CHAIN: {
23316 unsigned IntNo =
23317 Op.getConstantOperandVal(i: Opc == ISD::INTRINSIC_WO_CHAIN ? 0 : 1);
23318 switch (IntNo) {
23319 default:
23320 // We can't do anything for most intrinsics.
23321 break;
23322 case Intrinsic::riscv_vsetvli:
23323 case Intrinsic::riscv_vsetvlimax: {
23324 bool HasAVL = IntNo == Intrinsic::riscv_vsetvli;
23325 unsigned VSEW = Op.getConstantOperandVal(i: HasAVL + 1);
23326 RISCVVType::VLMUL VLMUL =
23327 static_cast<RISCVVType::VLMUL>(Op.getConstantOperandVal(i: HasAVL + 2));
23328 unsigned SEW = RISCVVType::decodeVSEW(VSEW);
23329 auto [LMul, Fractional] = RISCVVType::decodeVLMUL(VLMul: VLMUL);
23330 uint64_t MaxVL = Subtarget.getRealMaxVLen() / SEW;
23331 MaxVL = (Fractional) ? MaxVL / LMul : MaxVL * LMul;
23332
23333 // Result of vsetvli must be not larger than AVL.
23334 if (HasAVL && isa<ConstantSDNode>(Val: Op.getOperand(i: 1)))
23335 MaxVL = std::min(a: MaxVL, b: Op.getConstantOperandVal(i: 1));
23336
23337 unsigned KnownZeroFirstBit = Log2_32(Value: MaxVL) + 1;
23338 if (BitWidth > KnownZeroFirstBit)
23339 Known.Zero.setBitsFrom(KnownZeroFirstBit);
23340 break;
23341 }
23342 }
23343 break;
23344 }
23345 }
23346}
23347
23348unsigned RISCVTargetLowering::ComputeNumSignBitsForTargetNode(
23349 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
23350 unsigned Depth) const {
23351 switch (Op.getOpcode()) {
23352 default:
23353 break;
23354 case RISCVISD::SELECT_CC: {
23355 unsigned Tmp =
23356 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 3), DemandedElts, Depth: Depth + 1);
23357 if (Tmp == 1) return 1; // Early out.
23358 unsigned Tmp2 =
23359 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 4), DemandedElts, Depth: Depth + 1);
23360 return std::min(a: Tmp, b: Tmp2);
23361 }
23362 case RISCVISD::CZERO_EQZ:
23363 case RISCVISD::CZERO_NEZ:
23364 // Output is either all zero or operand 0. We can propagate sign bit count
23365 // from operand 0.
23366 return DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23367 case RISCVISD::NEGW_MAX: {
23368 // We expand this at isel to negw+max. The result will have 33 sign bits
23369 // if the input has at least 33 sign bits.
23370 unsigned Tmp =
23371 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23372 if (Tmp < 33) return 1;
23373 return 33;
23374 }
23375 case RISCVISD::SRAW: {
23376 unsigned Tmp =
23377 DAG.ComputeNumSignBits(Op: Op.getOperand(i: 0), DemandedElts, Depth: Depth + 1);
23378 // sraw produces at least 33 sign bits. If the input already has more than
23379 // 33 sign bits sraw, will preserve them.
23380 // TODO: A more precise answer could be calculated depending on known bits
23381 // in the shift amount.
23382 return std::max(a: Tmp, b: 33U);
23383 }
23384 case RISCVISD::SLLW:
23385 case RISCVISD::SRLW:
23386 case RISCVISD::DIVW:
23387 case RISCVISD::DIVUW:
23388 case RISCVISD::REMUW:
23389 case RISCVISD::ROLW:
23390 case RISCVISD::RORW:
23391 case RISCVISD::ABSW:
23392 case RISCVISD::FCVT_W_RV64:
23393 case RISCVISD::FCVT_WU_RV64:
23394 case RISCVISD::STRICT_FCVT_W_RV64:
23395 case RISCVISD::STRICT_FCVT_WU_RV64:
23396 // TODO: As the result is sign-extended, this is conservatively correct.
23397 return 33;
23398 case RISCVISD::VMV_X_S: {
23399 // The number of sign bits of the scalar result is computed by obtaining the
23400 // element type of the input vector operand, subtracting its width from the
23401 // XLEN, and then adding one (sign bit within the element type). If the
23402 // element type is wider than XLen, the least-significant XLEN bits are
23403 // taken.
23404 unsigned XLen = Subtarget.getXLen();
23405 unsigned EltBits = Op.getOperand(i: 0).getScalarValueSizeInBits();
23406 if (EltBits <= XLen)
23407 return XLen - EltBits + 1;
23408 break;
23409 }
23410 case ISD::INTRINSIC_W_CHAIN: {
23411 unsigned IntNo = Op.getConstantOperandVal(i: 1);
23412 switch (IntNo) {
23413 default:
23414 break;
23415 case Intrinsic::riscv_masked_atomicrmw_xchg:
23416 case Intrinsic::riscv_masked_atomicrmw_add:
23417 case Intrinsic::riscv_masked_atomicrmw_sub:
23418 case Intrinsic::riscv_masked_atomicrmw_nand:
23419 case Intrinsic::riscv_masked_atomicrmw_max:
23420 case Intrinsic::riscv_masked_atomicrmw_min:
23421 case Intrinsic::riscv_masked_atomicrmw_umax:
23422 case Intrinsic::riscv_masked_atomicrmw_umin:
23423 case Intrinsic::riscv_masked_cmpxchg:
23424 // riscv_masked_{atomicrmw_*,cmpxchg} intrinsics represent an emulated
23425 // narrow atomic operation. These are implemented using atomic
23426 // operations at the minimum supported atomicrmw/cmpxchg width whose
23427 // result is then sign extended to XLEN. With +A, the minimum width is
23428 // 32 for both 64 and 32.
23429 assert(getMinCmpXchgSizeInBits() == 32);
23430 assert(Subtarget.hasStdExtZalrsc());
23431 return Op.getValueSizeInBits() - 31;
23432 }
23433 break;
23434 }
23435 }
23436
23437 return 1;
23438}
23439
23440bool RISCVTargetLowering::SimplifyDemandedBitsForTargetNode(
23441 SDValue Op, const APInt &OriginalDemandedBits,
23442 const APInt &OriginalDemandedElts, KnownBits &Known, TargetLoweringOpt &TLO,
23443 unsigned Depth) const {
23444 unsigned BitWidth = OriginalDemandedBits.getBitWidth();
23445
23446 switch (Op.getOpcode()) {
23447 case RISCVISD::BREV8:
23448 case RISCVISD::ORC_B: {
23449 KnownBits Known2;
23450 bool IsGORC = Op.getOpcode() == RISCVISD::ORC_B;
23451 // For BREV8, we need to do BREV8 on the demanded bits.
23452 // For ORC_B, any bit in the output demandeds all bits from the same byte.
23453 // So we need to do ORC_B on the demanded bits.
23454 APInt DemandedBits =
23455 APInt(BitWidth, computeGREVOrGORC(x: OriginalDemandedBits.getZExtValue(),
23456 ShAmt: 7, IsGORC));
23457 if (SimplifyDemandedBits(Op: Op.getOperand(i: 0), DemandedBits,
23458 DemandedElts: OriginalDemandedElts, Known&: Known2, TLO, Depth: Depth + 1))
23459 return true;
23460
23461 // To compute zeros for ORC_B, we need to invert the value and invert it
23462 // back after. This inverting is harmless for BREV8.
23463 Known.Zero = ~computeGREVOrGORC(x: ~Known2.Zero.getZExtValue(), ShAmt: 7, IsGORC);
23464 Known.One = computeGREVOrGORC(x: Known2.One.getZExtValue(), ShAmt: 7, IsGORC);
23465 return false;
23466 }
23467 }
23468
23469 return TargetLowering::SimplifyDemandedBitsForTargetNode(
23470 Op, DemandedBits: OriginalDemandedBits, DemandedElts: OriginalDemandedElts, Known, TLO, Depth);
23471}
23472
23473bool RISCVTargetLowering::canCreateUndefOrPoisonForTargetNode(
23474 SDValue Op, const APInt &DemandedElts, const SelectionDAG &DAG,
23475 bool PoisonOnly, bool ConsiderFlags, unsigned Depth) const {
23476
23477 // TODO: Add more target nodes.
23478 switch (Op.getOpcode()) {
23479 case RISCVISD::SLLW:
23480 case RISCVISD::SRAW:
23481 case RISCVISD::SRLW:
23482 case RISCVISD::RORW:
23483 case RISCVISD::ROLW:
23484 // Only the lower 5 bits of RHS are read, guaranteeing the rotate/shift
23485 // amount is bounds.
23486 return false;
23487 case RISCVISD::SELECT_CC:
23488 // Integer comparisons cannot create poison.
23489 assert(Op.getOperand(0).getValueType().isInteger() &&
23490 "RISCVISD::SELECT_CC only compares integers");
23491 return false;
23492 }
23493 return TargetLowering::canCreateUndefOrPoisonForTargetNode(
23494 Op, DemandedElts, DAG, PoisonOnly, ConsiderFlags, Depth);
23495}
23496
23497const Constant *
23498RISCVTargetLowering::getTargetConstantFromLoad(LoadSDNode *Ld) const {
23499 assert(Ld && "Unexpected null LoadSDNode");
23500 if (!ISD::isNormalLoad(N: Ld))
23501 return nullptr;
23502
23503 SDValue Ptr = Ld->getBasePtr();
23504
23505 // Only constant pools with no offset are supported.
23506 auto GetSupportedConstantPool = [](SDValue Ptr) -> ConstantPoolSDNode * {
23507 auto *CNode = dyn_cast<ConstantPoolSDNode>(Val&: Ptr);
23508 if (!CNode || CNode->isMachineConstantPoolEntry() ||
23509 CNode->getOffset() != 0)
23510 return nullptr;
23511
23512 return CNode;
23513 };
23514
23515 // Simple case, LLA.
23516 if (Ptr.getOpcode() == RISCVISD::LLA) {
23517 auto *CNode = GetSupportedConstantPool(Ptr.getOperand(i: 0));
23518 if (!CNode || CNode->getTargetFlags() != 0)
23519 return nullptr;
23520
23521 return CNode->getConstVal();
23522 }
23523
23524 // Look for a HI and ADD_LO pair.
23525 if (Ptr.getOpcode() != RISCVISD::ADD_LO ||
23526 Ptr.getOperand(i: 0).getOpcode() != RISCVISD::HI)
23527 return nullptr;
23528
23529 auto *CNodeLo = GetSupportedConstantPool(Ptr.getOperand(i: 1));
23530 auto *CNodeHi = GetSupportedConstantPool(Ptr.getOperand(i: 0).getOperand(i: 0));
23531
23532 if (!CNodeLo || CNodeLo->getTargetFlags() != RISCVII::MO_LO ||
23533 !CNodeHi || CNodeHi->getTargetFlags() != RISCVII::MO_HI)
23534 return nullptr;
23535
23536 if (CNodeLo->getConstVal() != CNodeHi->getConstVal())
23537 return nullptr;
23538
23539 return CNodeLo->getConstVal();
23540}
23541
23542static MachineBasicBlock *emitReadCounterWidePseudo(MachineInstr &MI,
23543 MachineBasicBlock *BB) {
23544 assert(MI.getOpcode() == RISCV::ReadCounterWide && "Unexpected instruction");
23545
23546 // To read a 64-bit counter CSR on a 32-bit target, we read the two halves.
23547 // Should the count have wrapped while it was being read, we need to try
23548 // again.
23549 // For example:
23550 // ```
23551 // read:
23552 // csrrs x3, counterh # load high word of counter
23553 // csrrs x2, counter # load low word of counter
23554 // csrrs x4, counterh # load high word of counter
23555 // bne x3, x4, read # check if high word reads match, otherwise try again
23556 // ```
23557
23558 MachineFunction &MF = *BB->getParent();
23559 const BasicBlock *LLVMBB = BB->getBasicBlock();
23560 MachineFunction::iterator It = ++BB->getIterator();
23561
23562 MachineBasicBlock *LoopMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
23563 MF.insert(MBBI: It, MBB: LoopMBB);
23564
23565 MachineBasicBlock *DoneMBB = MF.CreateMachineBasicBlock(BB: LLVMBB);
23566 MF.insert(MBBI: It, MBB: DoneMBB);
23567
23568 // Transfer the remainder of BB and its successor edges to DoneMBB.
23569 DoneMBB->splice(Where: DoneMBB->begin(), Other: BB,
23570 From: std::next(x: MachineBasicBlock::iterator(MI)), To: BB->end());
23571 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: BB);
23572
23573 BB->addSuccessor(Succ: LoopMBB);
23574
23575 MachineRegisterInfo &RegInfo = MF.getRegInfo();
23576 Register ReadAgainReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
23577 Register LoReg = MI.getOperand(i: 0).getReg();
23578 Register HiReg = MI.getOperand(i: 1).getReg();
23579 int64_t LoCounter = MI.getOperand(i: 2).getImm();
23580 int64_t HiCounter = MI.getOperand(i: 3).getImm();
23581 DebugLoc DL = MI.getDebugLoc();
23582
23583 const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
23584 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: HiReg)
23585 .addImm(Val: HiCounter)
23586 .addReg(RegNo: RISCV::X0);
23587 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: LoReg)
23588 .addImm(Val: LoCounter)
23589 .addReg(RegNo: RISCV::X0);
23590 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::CSRRS), DestReg: ReadAgainReg)
23591 .addImm(Val: HiCounter)
23592 .addReg(RegNo: RISCV::X0);
23593
23594 BuildMI(BB: LoopMBB, MIMD: DL, MCID: TII->get(Opcode: RISCV::BNE))
23595 .addReg(RegNo: HiReg)
23596 .addReg(RegNo: ReadAgainReg)
23597 .addMBB(MBB: LoopMBB);
23598
23599 LoopMBB->addSuccessor(Succ: LoopMBB);
23600 LoopMBB->addSuccessor(Succ: DoneMBB);
23601
23602 MI.eraseFromParent();
23603
23604 return DoneMBB;
23605}
23606
23607static MachineBasicBlock *emitSplitF64Pseudo(MachineInstr &MI,
23608 MachineBasicBlock *BB,
23609 const RISCVSubtarget &Subtarget) {
23610 assert(MI.getOpcode() == RISCV::SplitF64Pseudo && "Unexpected instruction");
23611
23612 MachineFunction &MF = *BB->getParent();
23613 DebugLoc DL = MI.getDebugLoc();
23614 const RISCVInstrInfo &TII = *MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
23615 Register LoReg = MI.getOperand(i: 0).getReg();
23616 Register HiReg = MI.getOperand(i: 1).getReg();
23617 Register SrcReg = MI.getOperand(i: 2).getReg();
23618
23619 const TargetRegisterClass *SrcRC = &RISCV::FPR64RegClass;
23620 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
23621
23622 TII.storeRegToStackSlot(MBB&: *BB, MBBI: MI, SrcReg, IsKill: MI.getOperand(i: 2).isKill(), FrameIndex: FI, RC: SrcRC,
23623 VReg: Register());
23624 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
23625 MachineMemOperand *MMOLo =
23626 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(8));
23627 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
23628 PtrInfo: MPI.getWithOffset(O: 4), F: MachineMemOperand::MOLoad, Size: 4, BaseAlignment: Align(8));
23629
23630 // For big-endian, the high part is at offset 0 and the low part at offset 4.
23631 if (!Subtarget.isLittleEndian())
23632 std::swap(a&: LoReg, b&: HiReg);
23633
23634 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::LW), DestReg: LoReg)
23635 .addFrameIndex(Idx: FI)
23636 .addImm(Val: 0)
23637 .addMemOperand(MMO: MMOLo);
23638 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::LW), DestReg: HiReg)
23639 .addFrameIndex(Idx: FI)
23640 .addImm(Val: 4)
23641 .addMemOperand(MMO: MMOHi);
23642 MI.eraseFromParent(); // The pseudo instruction is gone now.
23643 return BB;
23644}
23645
23646static MachineBasicBlock *emitBuildPairF64Pseudo(MachineInstr &MI,
23647 MachineBasicBlock *BB,
23648 const RISCVSubtarget &Subtarget) {
23649 assert(MI.getOpcode() == RISCV::BuildPairF64Pseudo &&
23650 "Unexpected instruction");
23651
23652 MachineFunction &MF = *BB->getParent();
23653 DebugLoc DL = MI.getDebugLoc();
23654 const RISCVInstrInfo &TII = *MF.getSubtarget<RISCVSubtarget>().getInstrInfo();
23655 Register DstReg = MI.getOperand(i: 0).getReg();
23656 Register LoReg = MI.getOperand(i: 1).getReg();
23657 Register HiReg = MI.getOperand(i: 2).getReg();
23658 bool KillLo = MI.getOperand(i: 1).isKill();
23659 bool KillHi = MI.getOperand(i: 2).isKill();
23660
23661 const TargetRegisterClass *DstRC = &RISCV::FPR64RegClass;
23662 int FI = MF.getInfo<RISCVMachineFunctionInfo>()->getMoveF64FrameIndex(MF);
23663
23664 MachinePointerInfo MPI = MachinePointerInfo::getFixedStack(MF, FI);
23665 MachineMemOperand *MMOLo =
23666 MF.getMachineMemOperand(PtrInfo: MPI, F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(8));
23667 MachineMemOperand *MMOHi = MF.getMachineMemOperand(
23668 PtrInfo: MPI.getWithOffset(O: 4), F: MachineMemOperand::MOStore, Size: 4, BaseAlignment: Align(8));
23669
23670 // For big-endian, store the high part at offset 0 and the low part at
23671 // offset 4.
23672 if (!Subtarget.isLittleEndian()) {
23673 std::swap(a&: LoReg, b&: HiReg);
23674 std::swap(a&: KillLo, b&: KillHi);
23675 }
23676
23677 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::SW))
23678 .addReg(RegNo: LoReg, Flags: getKillRegState(B: KillLo))
23679 .addFrameIndex(Idx: FI)
23680 .addImm(Val: 0)
23681 .addMemOperand(MMO: MMOLo);
23682 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::SW))
23683 .addReg(RegNo: HiReg, Flags: getKillRegState(B: KillHi))
23684 .addFrameIndex(Idx: FI)
23685 .addImm(Val: 4)
23686 .addMemOperand(MMO: MMOHi);
23687 TII.loadRegFromStackSlot(MBB&: *BB, MBBI: MI, DstReg, FrameIndex: FI, RC: DstRC, VReg: Register());
23688 MI.eraseFromParent(); // The pseudo instruction is gone now.
23689 return BB;
23690}
23691
23692static MachineBasicBlock *emitQuietFCMP(MachineInstr &MI, MachineBasicBlock *BB,
23693 unsigned RelOpcode, unsigned EqOpcode,
23694 const RISCVSubtarget &Subtarget) {
23695 DebugLoc DL = MI.getDebugLoc();
23696 Register DstReg = MI.getOperand(i: 0).getReg();
23697 Register Src1Reg = MI.getOperand(i: 1).getReg();
23698 Register Src2Reg = MI.getOperand(i: 2).getReg();
23699 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
23700 Register SavedFFlags = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
23701 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
23702
23703 // Save the current FFLAGS.
23704 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::ReadFFLAGS), DestReg: SavedFFlags);
23705
23706 auto MIB = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RelOpcode), DestReg: DstReg)
23707 .addReg(RegNo: Src1Reg)
23708 .addReg(RegNo: Src2Reg);
23709 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
23710 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
23711
23712 // Restore the FFLAGS.
23713 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::WriteFFLAGS))
23714 .addReg(RegNo: SavedFFlags, Flags: RegState::Kill);
23715
23716 // Issue a dummy FEQ opcode to raise exception for signaling NaNs.
23717 auto MIB2 = BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: EqOpcode), DestReg: RISCV::X0)
23718 .addReg(RegNo: Src1Reg, Flags: getKillRegState(B: MI.getOperand(i: 1).isKill()))
23719 .addReg(RegNo: Src2Reg, Flags: getKillRegState(B: MI.getOperand(i: 2).isKill()));
23720 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
23721 MIB2->setFlag(MachineInstr::MIFlag::NoFPExcept);
23722
23723 // Erase the pseudoinstruction.
23724 MI.eraseFromParent();
23725 return BB;
23726}
23727
23728static MachineBasicBlock *
23729EmitLoweredCascadedSelect(MachineInstr &First, MachineInstr &Second,
23730 MachineBasicBlock *ThisMBB,
23731 const RISCVSubtarget &Subtarget) {
23732 // Select_FPRX_ (rs1, rs2, imm, rs4, (Select_FPRX_ rs1, rs2, imm, rs4, rs5)
23733 // Without this, custom-inserter would have generated:
23734 //
23735 // A
23736 // | \
23737 // | B
23738 // | /
23739 // C
23740 // | \
23741 // | D
23742 // | /
23743 // E
23744 //
23745 // A: X = ...; Y = ...
23746 // B: empty
23747 // C: Z = PHI [X, A], [Y, B]
23748 // D: empty
23749 // E: PHI [X, C], [Z, D]
23750 //
23751 // If we lower both Select_FPRX_ in a single step, we can instead generate:
23752 //
23753 // A
23754 // | \
23755 // | C
23756 // | /|
23757 // |/ |
23758 // | |
23759 // | D
23760 // | /
23761 // E
23762 //
23763 // A: X = ...; Y = ...
23764 // D: empty
23765 // E: PHI [X, A], [X, C], [Y, D]
23766
23767 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
23768 const DebugLoc &DL = First.getDebugLoc();
23769 const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
23770 MachineFunction *F = ThisMBB->getParent();
23771 MachineBasicBlock *FirstMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
23772 MachineBasicBlock *SecondMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
23773 MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
23774 MachineFunction::iterator It = ++ThisMBB->getIterator();
23775 F->insert(MBBI: It, MBB: FirstMBB);
23776 F->insert(MBBI: It, MBB: SecondMBB);
23777 F->insert(MBBI: It, MBB: SinkMBB);
23778
23779 // Transfer the remainder of ThisMBB and its successor edges to SinkMBB.
23780 SinkMBB->splice(Where: SinkMBB->begin(), Other: ThisMBB,
23781 From: std::next(x: MachineBasicBlock::iterator(First)),
23782 To: ThisMBB->end());
23783 SinkMBB->transferSuccessorsAndUpdatePHIs(FromMBB: ThisMBB);
23784
23785 // Fallthrough block for ThisMBB.
23786 ThisMBB->addSuccessor(Succ: FirstMBB);
23787 // Fallthrough block for FirstMBB.
23788 FirstMBB->addSuccessor(Succ: SecondMBB);
23789 ThisMBB->addSuccessor(Succ: SinkMBB);
23790 FirstMBB->addSuccessor(Succ: SinkMBB);
23791 // This is fallthrough.
23792 SecondMBB->addSuccessor(Succ: SinkMBB);
23793
23794 auto FirstCC = static_cast<RISCVCC::CondCode>(First.getOperand(i: 3).getImm());
23795 Register FLHS = First.getOperand(i: 1).getReg();
23796 Register FRHS = First.getOperand(i: 2).getReg();
23797 // Insert appropriate branch.
23798 BuildMI(BB: FirstMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC: FirstCC, SelectOpc: First.getOpcode())))
23799 .addReg(RegNo: FLHS)
23800 .addReg(RegNo: FRHS)
23801 .addMBB(MBB: SinkMBB);
23802
23803 Register SLHS = Second.getOperand(i: 1).getReg();
23804 Register SRHS = Second.getOperand(i: 2).getReg();
23805 Register Op1Reg4 = First.getOperand(i: 4).getReg();
23806 Register Op1Reg5 = First.getOperand(i: 5).getReg();
23807
23808 auto SecondCC = static_cast<RISCVCC::CondCode>(Second.getOperand(i: 3).getImm());
23809 // Insert appropriate branch.
23810 BuildMI(BB: ThisMBB, MIMD: DL,
23811 MCID: TII.get(Opcode: RISCVCC::getBrCond(CC: SecondCC, SelectOpc: Second.getOpcode())))
23812 .addReg(RegNo: SLHS)
23813 .addReg(RegNo: SRHS)
23814 .addMBB(MBB: SinkMBB);
23815
23816 Register DestReg = Second.getOperand(i: 0).getReg();
23817 Register Op2Reg4 = Second.getOperand(i: 4).getReg();
23818 BuildMI(BB&: *SinkMBB, I: SinkMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: RISCV::PHI), DestReg)
23819 .addReg(RegNo: Op2Reg4)
23820 .addMBB(MBB: ThisMBB)
23821 .addReg(RegNo: Op1Reg4)
23822 .addMBB(MBB: FirstMBB)
23823 .addReg(RegNo: Op1Reg5)
23824 .addMBB(MBB: SecondMBB);
23825
23826 // Now remove the Select_FPRX_s.
23827 First.eraseFromParent();
23828 Second.eraseFromParent();
23829 return SinkMBB;
23830}
23831
23832static MachineBasicBlock *emitSelectPseudo(MachineInstr &MI,
23833 MachineBasicBlock *BB,
23834 const RISCVSubtarget &Subtarget) {
23835 // To "insert" Select_* instructions, we actually have to insert the triangle
23836 // control-flow pattern. The incoming instructions know the destination vreg
23837 // to set, the condition code register to branch on, the true/false values to
23838 // select between, and the condcode to use to select the appropriate branch.
23839 //
23840 // We produce the following control flow:
23841 // HeadMBB
23842 // | \
23843 // | IfFalseMBB
23844 // | /
23845 // TailMBB
23846 //
23847 // When we find a sequence of selects we attempt to optimize their emission
23848 // by sharing the control flow. Currently we only handle cases where we have
23849 // multiple selects with the exact same condition (same LHS, RHS and CC).
23850 // The selects may be interleaved with other instructions if the other
23851 // instructions meet some requirements we deem safe:
23852 // - They are not pseudo instructions.
23853 // - They are debug instructions. Otherwise,
23854 // - They do not have side-effects, do not access memory and their inputs do
23855 // not depend on the results of the select pseudo-instructions.
23856 // - They don't adjust stack.
23857 // The TrueV/FalseV operands of the selects cannot depend on the result of
23858 // previous selects in the sequence.
23859 // These conditions could be further relaxed. See the X86 target for a
23860 // related approach and more information.
23861 //
23862 // Select_FPRX_ (rs1, rs2, imm, rs4, (Select_FPRX_ rs1, rs2, imm, rs4, rs5))
23863 // is checked here and handled by a separate function -
23864 // EmitLoweredCascadedSelect.
23865
23866 auto Next = next_nodbg(It: MI.getIterator(), End: BB->instr_end());
23867 if (MI.getOpcode() != RISCV::Select_GPR_Using_CC_GPR &&
23868 MI.getOperand(i: 1).isReg() && MI.getOperand(i: 2).isReg() &&
23869 Next != BB->end() && Next->getOpcode() == MI.getOpcode() &&
23870 Next->getOperand(i: 5).getReg() == MI.getOperand(i: 0).getReg() &&
23871 Next->getOperand(i: 5).isKill())
23872 return EmitLoweredCascadedSelect(First&: MI, Second&: *Next, ThisMBB: BB, Subtarget);
23873
23874 Register LHS = MI.getOperand(i: 1).getReg();
23875 Register RHS;
23876 if (MI.getOperand(i: 2).isReg())
23877 RHS = MI.getOperand(i: 2).getReg();
23878 auto CC = static_cast<RISCVCC::CondCode>(MI.getOperand(i: 3).getImm());
23879
23880 SmallVector<MachineInstr *, 4> SelectDebugValues;
23881 SmallSet<Register, 4> SelectDests;
23882 SelectDests.insert(V: MI.getOperand(i: 0).getReg());
23883
23884 MachineInstr *LastSelectPseudo = &MI;
23885 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
23886
23887 for (auto E = BB->end(), SequenceMBBI = MachineBasicBlock::iterator(MI);
23888 SequenceMBBI != E; ++SequenceMBBI) {
23889 if (SequenceMBBI->isDebugInstr())
23890 continue;
23891 if (RISCVInstrInfo::isSelectPseudo(MI: *SequenceMBBI)) {
23892 if (SequenceMBBI->getOperand(i: 1).getReg() != LHS ||
23893 !SequenceMBBI->getOperand(i: 2).isReg() ||
23894 SequenceMBBI->getOperand(i: 2).getReg() != RHS ||
23895 SequenceMBBI->getOperand(i: 3).getImm() != CC ||
23896 SelectDests.count(V: SequenceMBBI->getOperand(i: 4).getReg()) ||
23897 SelectDests.count(V: SequenceMBBI->getOperand(i: 5).getReg()))
23898 break;
23899 LastSelectPseudo = &*SequenceMBBI;
23900 SequenceMBBI->collectDebugValues(DbgValues&: SelectDebugValues);
23901 SelectDests.insert(V: SequenceMBBI->getOperand(i: 0).getReg());
23902 continue;
23903 }
23904 if (SequenceMBBI->hasUnmodeledSideEffects() ||
23905 SequenceMBBI->mayLoadOrStore() ||
23906 SequenceMBBI->usesCustomInsertionHook() ||
23907 TII.isFrameInstr(I: *SequenceMBBI) ||
23908 SequenceMBBI->isStackAligningInlineAsm())
23909 break;
23910 if (llvm::any_of(Range: SequenceMBBI->operands(), P: [&](MachineOperand &MO) {
23911 return MO.isReg() && MO.isUse() && SelectDests.count(V: MO.getReg());
23912 }))
23913 break;
23914 }
23915
23916 const BasicBlock *LLVM_BB = BB->getBasicBlock();
23917 DebugLoc DL = MI.getDebugLoc();
23918 MachineFunction::iterator I = ++BB->getIterator();
23919
23920 MachineBasicBlock *HeadMBB = BB;
23921 MachineFunction *F = BB->getParent();
23922 MachineBasicBlock *TailMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
23923 MachineBasicBlock *IfFalseMBB = F->CreateMachineBasicBlock(BB: LLVM_BB);
23924
23925 F->insert(MBBI: I, MBB: IfFalseMBB);
23926 F->insert(MBBI: I, MBB: TailMBB);
23927
23928 // Set the call frame size on entry to the new basic blocks.
23929 unsigned CallFrameSize = TII.getCallFrameSizeAt(MI&: *LastSelectPseudo);
23930 IfFalseMBB->setCallFrameSize(CallFrameSize);
23931 TailMBB->setCallFrameSize(CallFrameSize);
23932
23933 // Transfer debug instructions associated with the selects to TailMBB.
23934 for (MachineInstr *DebugInstr : SelectDebugValues) {
23935 TailMBB->push_back(MI: DebugInstr->removeFromParent());
23936 }
23937
23938 // Move all instructions after the sequence to TailMBB.
23939 TailMBB->splice(Where: TailMBB->end(), Other: HeadMBB,
23940 From: std::next(x: LastSelectPseudo->getIterator()), To: HeadMBB->end());
23941 // Update machine-CFG edges by transferring all successors of the current
23942 // block to the new block which will contain the Phi nodes for the selects.
23943 TailMBB->transferSuccessorsAndUpdatePHIs(FromMBB: HeadMBB);
23944 // Set the successors for HeadMBB.
23945 HeadMBB->addSuccessor(Succ: IfFalseMBB);
23946 HeadMBB->addSuccessor(Succ: TailMBB);
23947
23948 // Insert appropriate branch.
23949 if (MI.getOperand(i: 2).isImm())
23950 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC, SelectOpc: MI.getOpcode())))
23951 .addReg(RegNo: LHS)
23952 .addImm(Val: MI.getOperand(i: 2).getImm())
23953 .addMBB(MBB: TailMBB);
23954 else
23955 BuildMI(BB: HeadMBB, MIMD: DL, MCID: TII.get(Opcode: RISCVCC::getBrCond(CC, SelectOpc: MI.getOpcode())))
23956 .addReg(RegNo: LHS)
23957 .addReg(RegNo: RHS)
23958 .addMBB(MBB: TailMBB);
23959
23960 // IfFalseMBB just falls through to TailMBB.
23961 IfFalseMBB->addSuccessor(Succ: TailMBB);
23962
23963 // Create PHIs for all of the select pseudo-instructions.
23964 auto SelectMBBI = MI.getIterator();
23965 auto SelectEnd = std::next(x: LastSelectPseudo->getIterator());
23966 auto InsertionPoint = TailMBB->begin();
23967 while (SelectMBBI != SelectEnd) {
23968 auto Next = std::next(x: SelectMBBI);
23969 if (RISCVInstrInfo::isSelectPseudo(MI: *SelectMBBI)) {
23970 // %Result = phi [ %TrueValue, HeadMBB ], [ %FalseValue, IfFalseMBB ]
23971 BuildMI(BB&: *TailMBB, I: InsertionPoint, MIMD: SelectMBBI->getDebugLoc(),
23972 MCID: TII.get(Opcode: RISCV::PHI), DestReg: SelectMBBI->getOperand(i: 0).getReg())
23973 .addReg(RegNo: SelectMBBI->getOperand(i: 4).getReg())
23974 .addMBB(MBB: HeadMBB)
23975 .addReg(RegNo: SelectMBBI->getOperand(i: 5).getReg())
23976 .addMBB(MBB: IfFalseMBB);
23977 SelectMBBI->eraseFromParent();
23978 }
23979 SelectMBBI = Next;
23980 }
23981
23982 F->getProperties().resetNoPHIs();
23983 return TailMBB;
23984}
23985
23986// Helper to find Masked Pseudo instruction from MC instruction, LMUL and SEW.
23987static const RISCV::RISCVMaskedPseudoInfo *
23988lookupMaskedIntrinsic(uint16_t MCOpcode, RISCVVType::VLMUL LMul, unsigned SEW) {
23989 const RISCVVInversePseudosTable::PseudoInfo *Inverse =
23990 RISCVVInversePseudosTable::getBaseInfo(BaseInstr: MCOpcode, VLMul: LMul, SEW);
23991 assert(Inverse && "Unexpected LMUL and SEW pair for instruction");
23992 const RISCV::RISCVMaskedPseudoInfo *Masked =
23993 RISCV::lookupMaskedIntrinsicByUnmasked(UnmaskedPseudo: Inverse->Pseudo);
23994 assert(Masked && "Could not find masked instruction for LMUL and SEW pair");
23995 return Masked;
23996}
23997
23998static MachineBasicBlock *emitVFROUND_NOEXCEPT_MASK(MachineInstr &MI,
23999 MachineBasicBlock *BB,
24000 unsigned CVTXOpc) {
24001 DebugLoc DL = MI.getDebugLoc();
24002
24003 const TargetInstrInfo &TII = *BB->getParent()->getSubtarget().getInstrInfo();
24004
24005 MachineRegisterInfo &MRI = BB->getParent()->getRegInfo();
24006 Register SavedFFLAGS = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24007
24008 // Save the old value of FFLAGS.
24009 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::ReadFFLAGS), DestReg: SavedFFLAGS);
24010
24011 assert(MI.getNumOperands() == 7);
24012
24013 // Emit a VFCVT_X_F
24014 const TargetRegisterInfo *TRI =
24015 BB->getParent()->getSubtarget().getRegisterInfo();
24016 const TargetRegisterClass *RC = MI.getRegClassConstraint(OpIdx: 0, TII: &TII, TRI);
24017 Register Tmp = MRI.createVirtualRegister(RegClass: RC);
24018 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: CVTXOpc), DestReg: Tmp)
24019 .add(MO: MI.getOperand(i: 1))
24020 .add(MO: MI.getOperand(i: 2))
24021 .add(MO: MI.getOperand(i: 3))
24022 .add(MO: MachineOperand::CreateImm(Val: 7)) // frm = DYN
24023 .add(MO: MI.getOperand(i: 4))
24024 .add(MO: MI.getOperand(i: 5))
24025 .add(MO: MI.getOperand(i: 6))
24026 .add(MO: MachineOperand::CreateReg(Reg: RISCV::FRM,
24027 /*IsDef*/ isDef: false,
24028 /*IsImp*/ isImp: true));
24029
24030 // Emit a VFCVT_F_X
24031 RISCVVType::VLMUL LMul = RISCVII::getLMul(TSFlags: MI.getDesc().TSFlags);
24032 unsigned Log2SEW = MI.getOperand(i: RISCVII::getSEWOpNum(Desc: MI.getDesc())).getImm();
24033 // There is no E8 variant for VFCVT_F_X.
24034 assert(Log2SEW >= 4);
24035 unsigned CVTFOpc =
24036 lookupMaskedIntrinsic(MCOpcode: RISCV::VFCVT_F_X_V, LMul, SEW: 1 << Log2SEW)
24037 ->MaskedPseudo;
24038
24039 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: CVTFOpc))
24040 .add(MO: MI.getOperand(i: 0))
24041 .add(MO: MI.getOperand(i: 1))
24042 .addReg(RegNo: Tmp)
24043 .add(MO: MI.getOperand(i: 3))
24044 .add(MO: MachineOperand::CreateImm(Val: 7)) // frm = DYN
24045 .add(MO: MI.getOperand(i: 4))
24046 .add(MO: MI.getOperand(i: 5))
24047 .add(MO: MI.getOperand(i: 6))
24048 .add(MO: MachineOperand::CreateReg(Reg: RISCV::FRM,
24049 /*IsDef*/ isDef: false,
24050 /*IsImp*/ isImp: true));
24051
24052 // Restore FFLAGS.
24053 BuildMI(BB&: *BB, I&: MI, MIMD: DL, MCID: TII.get(Opcode: RISCV::WriteFFLAGS))
24054 .addReg(RegNo: SavedFFLAGS, Flags: RegState::Kill);
24055
24056 // Erase the pseudoinstruction.
24057 MI.eraseFromParent();
24058 return BB;
24059}
24060
24061static MachineBasicBlock *emitFROUND(MachineInstr &MI, MachineBasicBlock *MBB,
24062 const RISCVSubtarget &Subtarget) {
24063 unsigned CmpOpc, F2IOpc, I2FOpc, FSGNJOpc, FSGNJXOpc;
24064 const TargetRegisterClass *RC;
24065 switch (MI.getOpcode()) {
24066 default:
24067 llvm_unreachable("Unexpected opcode");
24068 case RISCV::PseudoFROUND_H:
24069 CmpOpc = RISCV::FLT_H;
24070 F2IOpc = RISCV::FCVT_W_H;
24071 I2FOpc = RISCV::FCVT_H_W;
24072 FSGNJOpc = RISCV::FSGNJ_H;
24073 FSGNJXOpc = RISCV::FSGNJX_H;
24074 RC = &RISCV::FPR16RegClass;
24075 break;
24076 case RISCV::PseudoFROUND_H_INX:
24077 CmpOpc = RISCV::FLT_H_INX;
24078 F2IOpc = RISCV::FCVT_W_H_INX;
24079 I2FOpc = RISCV::FCVT_H_W_INX;
24080 FSGNJOpc = RISCV::FSGNJ_H_INX;
24081 FSGNJXOpc = RISCV::FSGNJX_H_INX;
24082 RC = &RISCV::GPRF16RegClass;
24083 break;
24084 case RISCV::PseudoFROUND_S:
24085 CmpOpc = RISCV::FLT_S;
24086 F2IOpc = RISCV::FCVT_W_S;
24087 I2FOpc = RISCV::FCVT_S_W;
24088 FSGNJOpc = RISCV::FSGNJ_S;
24089 FSGNJXOpc = RISCV::FSGNJX_S;
24090 RC = &RISCV::FPR32RegClass;
24091 break;
24092 case RISCV::PseudoFROUND_S_INX:
24093 CmpOpc = RISCV::FLT_S_INX;
24094 F2IOpc = RISCV::FCVT_W_S_INX;
24095 I2FOpc = RISCV::FCVT_S_W_INX;
24096 FSGNJOpc = RISCV::FSGNJ_S_INX;
24097 FSGNJXOpc = RISCV::FSGNJX_S_INX;
24098 RC = &RISCV::GPRF32RegClass;
24099 break;
24100 case RISCV::PseudoFROUND_D:
24101 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
24102 CmpOpc = RISCV::FLT_D;
24103 F2IOpc = RISCV::FCVT_L_D;
24104 I2FOpc = RISCV::FCVT_D_L;
24105 FSGNJOpc = RISCV::FSGNJ_D;
24106 FSGNJXOpc = RISCV::FSGNJX_D;
24107 RC = &RISCV::FPR64RegClass;
24108 break;
24109 case RISCV::PseudoFROUND_D_INX:
24110 assert(Subtarget.is64Bit() && "Expected 64-bit GPR.");
24111 CmpOpc = RISCV::FLT_D_INX;
24112 F2IOpc = RISCV::FCVT_L_D_INX;
24113 I2FOpc = RISCV::FCVT_D_L_INX;
24114 FSGNJOpc = RISCV::FSGNJ_D_INX;
24115 FSGNJXOpc = RISCV::FSGNJX_D_INX;
24116 RC = &RISCV::GPRRegClass;
24117 break;
24118 }
24119
24120 const BasicBlock *BB = MBB->getBasicBlock();
24121 DebugLoc DL = MI.getDebugLoc();
24122 MachineFunction::iterator I = ++MBB->getIterator();
24123
24124 MachineFunction *F = MBB->getParent();
24125 MachineBasicBlock *CvtMBB = F->CreateMachineBasicBlock(BB);
24126 MachineBasicBlock *DoneMBB = F->CreateMachineBasicBlock(BB);
24127
24128 F->insert(MBBI: I, MBB: CvtMBB);
24129 F->insert(MBBI: I, MBB: DoneMBB);
24130 // Move all instructions after the sequence to DoneMBB.
24131 DoneMBB->splice(Where: DoneMBB->end(), Other: MBB, From: MachineBasicBlock::iterator(MI),
24132 To: MBB->end());
24133 // Update machine-CFG edges by transferring all successors of the current
24134 // block to the new block which will contain the Phi nodes for the selects.
24135 DoneMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
24136 // Set the successors for MBB.
24137 MBB->addSuccessor(Succ: CvtMBB);
24138 MBB->addSuccessor(Succ: DoneMBB);
24139
24140 Register DstReg = MI.getOperand(i: 0).getReg();
24141 Register SrcReg = MI.getOperand(i: 1).getReg();
24142 Register MaxReg = MI.getOperand(i: 2).getReg();
24143 int64_t FRM = MI.getOperand(i: 3).getImm();
24144
24145 const RISCVInstrInfo &TII = *Subtarget.getInstrInfo();
24146 MachineRegisterInfo &MRI = MBB->getParent()->getRegInfo();
24147
24148 Register FabsReg = MRI.createVirtualRegister(RegClass: RC);
24149 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: FSGNJXOpc), DestReg: FabsReg).addReg(RegNo: SrcReg).addReg(RegNo: SrcReg);
24150
24151 // Compare the FP value to the max value.
24152 Register CmpReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24153 auto MIB =
24154 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: CmpOpc), DestReg: CmpReg).addReg(RegNo: FabsReg).addReg(RegNo: MaxReg);
24155 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
24156 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
24157
24158 // Insert branch.
24159 BuildMI(BB: MBB, MIMD: DL, MCID: TII.get(Opcode: RISCV::BEQ))
24160 .addReg(RegNo: CmpReg)
24161 .addReg(RegNo: RISCV::X0)
24162 .addMBB(MBB: DoneMBB);
24163
24164 CvtMBB->addSuccessor(Succ: DoneMBB);
24165
24166 // Convert to integer.
24167 Register F2IReg = MRI.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24168 MIB = BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: F2IOpc), DestReg: F2IReg).addReg(RegNo: SrcReg).addImm(Val: FRM);
24169 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
24170 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
24171
24172 // Convert back to FP.
24173 Register I2FReg = MRI.createVirtualRegister(RegClass: RC);
24174 MIB = BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: I2FOpc), DestReg: I2FReg).addReg(RegNo: F2IReg).addImm(Val: FRM);
24175 if (MI.getFlag(Flag: MachineInstr::MIFlag::NoFPExcept))
24176 MIB->setFlag(MachineInstr::MIFlag::NoFPExcept);
24177
24178 // Restore the sign bit.
24179 Register CvtReg = MRI.createVirtualRegister(RegClass: RC);
24180 BuildMI(BB: CvtMBB, MIMD: DL, MCID: TII.get(Opcode: FSGNJOpc), DestReg: CvtReg).addReg(RegNo: I2FReg).addReg(RegNo: SrcReg);
24181
24182 // Merge the results.
24183 BuildMI(BB&: *DoneMBB, I: DoneMBB->begin(), MIMD: DL, MCID: TII.get(Opcode: RISCV::PHI), DestReg: DstReg)
24184 .addReg(RegNo: SrcReg)
24185 .addMBB(MBB)
24186 .addReg(RegNo: CvtReg)
24187 .addMBB(MBB: CvtMBB);
24188
24189 MI.eraseFromParent();
24190 return DoneMBB;
24191}
24192
24193MachineBasicBlock *
24194RISCVTargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
24195 MachineBasicBlock *BB) const {
24196 switch (MI.getOpcode()) {
24197 default:
24198 llvm_unreachable("Unexpected instr type to insert");
24199 case RISCV::ReadCounterWide:
24200 assert(!Subtarget.is64Bit() &&
24201 "ReadCounterWide is only to be used on riscv32");
24202 return emitReadCounterWidePseudo(MI, BB);
24203 case RISCV::Select_GPR_Using_CC_GPR:
24204 case RISCV::Select_GPR_Using_CC_Imm5_Zibi:
24205 case RISCV::Select_GPR_Using_CC_SImm5_CV:
24206 case RISCV::Select_GPRNoX0_Using_CC_SImm5NonZero_QC:
24207 case RISCV::Select_GPRNoX0_Using_CC_UImm5NonZero_QC:
24208 case RISCV::Select_GPRNoX0_Using_CC_SImm16NonZero_QC:
24209 case RISCV::Select_GPRNoX0_Using_CC_UImm16NonZero_QC:
24210 case RISCV::Select_GPR_Using_CC_UImmLog2XLen_NDS:
24211 case RISCV::Select_GPR_Using_CC_UImm7_NDS:
24212 case RISCV::Select_FPR16_Using_CC_GPR:
24213 case RISCV::Select_FPR16INX_Using_CC_GPR:
24214 case RISCV::Select_FPR32_Using_CC_GPR:
24215 case RISCV::Select_FPR32INX_Using_CC_GPR:
24216 case RISCV::Select_FPR64_Using_CC_GPR:
24217 case RISCV::Select_FPR64INX_Using_CC_GPR:
24218 case RISCV::Select_FPR64IN32X_Using_CC_GPR:
24219 return emitSelectPseudo(MI, BB, Subtarget);
24220 case RISCV::BuildPairF64Pseudo:
24221 return emitBuildPairF64Pseudo(MI, BB, Subtarget);
24222 case RISCV::SplitF64Pseudo:
24223 return emitSplitF64Pseudo(MI, BB, Subtarget);
24224 case RISCV::PseudoQuietFLE_H:
24225 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_H, EqOpcode: RISCV::FEQ_H, Subtarget);
24226 case RISCV::PseudoQuietFLE_H_INX:
24227 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_H_INX, EqOpcode: RISCV::FEQ_H_INX, Subtarget);
24228 case RISCV::PseudoQuietFLT_H:
24229 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_H, EqOpcode: RISCV::FEQ_H, Subtarget);
24230 case RISCV::PseudoQuietFLT_H_INX:
24231 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_H_INX, EqOpcode: RISCV::FEQ_H_INX, Subtarget);
24232 case RISCV::PseudoQuietFLE_S:
24233 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_S, EqOpcode: RISCV::FEQ_S, Subtarget);
24234 case RISCV::PseudoQuietFLE_S_INX:
24235 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_S_INX, EqOpcode: RISCV::FEQ_S_INX, Subtarget);
24236 case RISCV::PseudoQuietFLT_S:
24237 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_S, EqOpcode: RISCV::FEQ_S, Subtarget);
24238 case RISCV::PseudoQuietFLT_S_INX:
24239 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_S_INX, EqOpcode: RISCV::FEQ_S_INX, Subtarget);
24240 case RISCV::PseudoQuietFLE_D:
24241 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D, EqOpcode: RISCV::FEQ_D, Subtarget);
24242 case RISCV::PseudoQuietFLE_D_INX:
24243 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D_INX, EqOpcode: RISCV::FEQ_D_INX, Subtarget);
24244 case RISCV::PseudoQuietFLE_D_IN32X:
24245 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLE_D_IN32X, EqOpcode: RISCV::FEQ_D_IN32X,
24246 Subtarget);
24247 case RISCV::PseudoQuietFLT_D:
24248 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D, EqOpcode: RISCV::FEQ_D, Subtarget);
24249 case RISCV::PseudoQuietFLT_D_INX:
24250 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D_INX, EqOpcode: RISCV::FEQ_D_INX, Subtarget);
24251 case RISCV::PseudoQuietFLT_D_IN32X:
24252 return emitQuietFCMP(MI, BB, RelOpcode: RISCV::FLT_D_IN32X, EqOpcode: RISCV::FEQ_D_IN32X,
24253 Subtarget);
24254
24255 case RISCV::PseudoVFROUND_NOEXCEPT_V_M1_MASK:
24256 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M1_MASK);
24257 case RISCV::PseudoVFROUND_NOEXCEPT_V_M2_MASK:
24258 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M2_MASK);
24259 case RISCV::PseudoVFROUND_NOEXCEPT_V_M4_MASK:
24260 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M4_MASK);
24261 case RISCV::PseudoVFROUND_NOEXCEPT_V_M8_MASK:
24262 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_M8_MASK);
24263 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF2_MASK:
24264 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_MF2_MASK);
24265 case RISCV::PseudoVFROUND_NOEXCEPT_V_MF4_MASK:
24266 return emitVFROUND_NOEXCEPT_MASK(MI, BB, CVTXOpc: RISCV::PseudoVFCVT_X_F_V_MF4_MASK);
24267 case RISCV::PseudoFROUND_H:
24268 case RISCV::PseudoFROUND_H_INX:
24269 case RISCV::PseudoFROUND_S:
24270 case RISCV::PseudoFROUND_S_INX:
24271 case RISCV::PseudoFROUND_D:
24272 case RISCV::PseudoFROUND_D_INX:
24273 case RISCV::PseudoFROUND_D_IN32X:
24274 return emitFROUND(MI, MBB: BB, Subtarget);
24275 case RISCV::PROBED_STACKALLOC_DYN:
24276 return emitDynamicProbedAlloc(MI, MBB: BB);
24277 case TargetOpcode::STATEPOINT:
24278 // STATEPOINT is a pseudo instruction which has no implicit defs/uses
24279 // while jal call instruction (where statepoint will be lowered at the end)
24280 // has implicit def. This def is early-clobber as it will be set at
24281 // the moment of the call and earlier than any use is read.
24282 // Add this implicit dead def here as a workaround.
24283 MI.addOperand(MF&: *MI.getMF(),
24284 Op: MachineOperand::CreateReg(
24285 Reg: RISCV::X1, /*isDef*/ true,
24286 /*isImp*/ true, /*isKill*/ false, /*isDead*/ true,
24287 /*isUndef*/ false, /*isEarlyClobber*/ true));
24288 [[fallthrough]];
24289 case TargetOpcode::STACKMAP:
24290 case TargetOpcode::PATCHPOINT:
24291 if (!Subtarget.is64Bit())
24292 reportFatalUsageError(reason: "STACKMAP, PATCHPOINT and STATEPOINT are only "
24293 "supported on 64-bit targets");
24294 return emitPatchPoint(MI, MBB: BB);
24295 }
24296}
24297
24298void RISCVTargetLowering::AdjustInstrPostInstrSelection(MachineInstr &MI,
24299 SDNode *Node) const {
24300 // If instruction defines FRM operand, conservatively set it as non-dead to
24301 // express data dependency with FRM users and prevent incorrect instruction
24302 // reordering.
24303 if (auto *FRMDef = MI.findRegisterDefOperand(Reg: RISCV::FRM, /*TRI=*/nullptr)) {
24304 FRMDef->setIsDead(false);
24305 return;
24306 }
24307 // Add FRM dependency to any instructions with dynamic rounding mode.
24308 int Idx = RISCV::getNamedOperandIdx(Opcode: MI.getOpcode(), Name: RISCV::OpName::frm);
24309 if (Idx < 0) {
24310 // Vector pseudos have FRM index indicated by TSFlags.
24311 Idx = RISCVII::getFRMOpNum(Desc: MI.getDesc());
24312 if (Idx < 0)
24313 return;
24314 }
24315 if (MI.getOperand(i: Idx).getImm() != RISCVFPRndMode::DYN)
24316 return;
24317 // If the instruction already reads FRM, don't add another read.
24318 if (MI.readsRegister(Reg: RISCV::FRM, /*TRI=*/nullptr))
24319 return;
24320 MI.addOperand(
24321 Op: MachineOperand::CreateReg(Reg: RISCV::FRM, /*isDef*/ false, /*isImp*/ true));
24322}
24323
24324void RISCVTargetLowering::analyzeInputArgs(
24325 MachineFunction &MF, CCState &CCInfo,
24326 const SmallVectorImpl<ISD::InputArg> &Ins, bool IsRet,
24327 RISCVCCAssignFn Fn) const {
24328 for (const auto &[Idx, In] : enumerate(First: Ins)) {
24329 MVT ArgVT = In.VT;
24330 ISD::ArgFlagsTy ArgFlags = In.Flags;
24331
24332 if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, IsRet,
24333 In.OrigTy)) {
24334 LLVM_DEBUG(dbgs() << "InputArg #" << Idx << " has unhandled type "
24335 << ArgVT << '\n');
24336 llvm_unreachable(nullptr);
24337 }
24338 }
24339}
24340
24341void RISCVTargetLowering::analyzeOutputArgs(
24342 MachineFunction &MF, CCState &CCInfo,
24343 const SmallVectorImpl<ISD::OutputArg> &Outs, bool IsRet,
24344 CallLoweringInfo *CLI, RISCVCCAssignFn Fn) const {
24345 for (const auto &[Idx, Out] : enumerate(First: Outs)) {
24346 MVT ArgVT = Out.VT;
24347 ISD::ArgFlagsTy ArgFlags = Out.Flags;
24348
24349 if (Fn(Idx, ArgVT, ArgVT, CCValAssign::Full, ArgFlags, CCInfo, IsRet,
24350 Out.OrigTy)) {
24351 LLVM_DEBUG(dbgs() << "OutputArg #" << Idx << " has unhandled type "
24352 << ArgVT << "\n");
24353 llvm_unreachable(nullptr);
24354 }
24355 }
24356}
24357
24358// Convert Val to a ValVT. Should not be called for CCValAssign::Indirect
24359// values.
24360static SDValue convertLocVTToValVT(SelectionDAG &DAG, SDValue Val,
24361 const CCValAssign &VA, const SDLoc &DL,
24362 const RISCVSubtarget &Subtarget) {
24363 if (VA.needsCustom()) {
24364 if (VA.getLocVT().isInteger() &&
24365 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
24366 return DAG.getNode(Opcode: RISCVISD::FMV_H_X, DL, VT: VA.getValVT(), Operand: Val);
24367 if (VA.getLocVT() == MVT::i64 && VA.getValVT() == MVT::f32)
24368 return DAG.getNode(Opcode: RISCVISD::FMV_W_X_RV64, DL, VT: MVT::f32, Operand: Val);
24369 if (VA.getValVT().isFixedLengthVector() && VA.getLocVT().isScalableVector())
24370 return convertFromScalableVector(VT: VA.getValVT(), V: Val, DAG, Subtarget);
24371 llvm_unreachable("Unexpected Custom handling.");
24372 }
24373
24374 switch (VA.getLocInfo()) {
24375 default:
24376 llvm_unreachable("Unexpected CCValAssign::LocInfo");
24377 case CCValAssign::Full:
24378 break;
24379 case CCValAssign::BCvt:
24380 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: VA.getValVT(), Operand: Val);
24381 break;
24382 }
24383 return Val;
24384}
24385
24386// The caller is responsible for loading the full value if the argument is
24387// passed with CCValAssign::Indirect.
24388static SDValue unpackFromRegLoc(SelectionDAG &DAG, SDValue Chain,
24389 const CCValAssign &VA, const SDLoc &DL,
24390 const ISD::InputArg &In,
24391 const RISCVTargetLowering &TLI) {
24392 MachineFunction &MF = DAG.getMachineFunction();
24393 MachineRegisterInfo &RegInfo = MF.getRegInfo();
24394 EVT LocVT = VA.getLocVT();
24395 SDValue Val;
24396 const TargetRegisterClass *RC = TLI.getRegClassFor(VT: LocVT.getSimpleVT());
24397 Register VReg = RegInfo.createVirtualRegister(RegClass: RC);
24398 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: VReg);
24399 Val = DAG.getCopyFromReg(Chain, dl: DL, Reg: VReg, VT: LocVT);
24400
24401 // If input is sign extended from 32 bits, note it for the SExtWRemoval pass.
24402 if (In.isOrigArg()) {
24403 Argument *OrigArg = MF.getFunction().getArg(i: In.getOrigArgIndex());
24404 if (OrigArg->getType()->isIntegerTy()) {
24405 unsigned BitWidth = OrigArg->getType()->getIntegerBitWidth();
24406 // An input zero extended from i31 can also be considered sign extended.
24407 if ((BitWidth <= 32 && In.Flags.isSExt()) ||
24408 (BitWidth < 32 && In.Flags.isZExt())) {
24409 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
24410 RVFI->addSExt32Register(Reg: VReg);
24411 }
24412 }
24413 }
24414
24415 if (VA.getLocInfo() == CCValAssign::Indirect)
24416 return Val;
24417
24418 return convertLocVTToValVT(DAG, Val, VA, DL, Subtarget: TLI.getSubtarget());
24419}
24420
24421static SDValue convertValVTToLocVT(SelectionDAG &DAG, SDValue Val,
24422 const CCValAssign &VA, const SDLoc &DL,
24423 const RISCVSubtarget &Subtarget) {
24424 EVT LocVT = VA.getLocVT();
24425
24426 if (VA.needsCustom()) {
24427 if (LocVT.isInteger() &&
24428 (VA.getValVT() == MVT::f16 || VA.getValVT() == MVT::bf16))
24429 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTH, DL, VT: LocVT, Operand: Val);
24430 if (LocVT == MVT::i64 && VA.getValVT() == MVT::f32)
24431 return DAG.getNode(Opcode: RISCVISD::FMV_X_ANYEXTW_RV64, DL, VT: MVT::i64, Operand: Val);
24432 if (VA.getValVT().isFixedLengthVector() && LocVT.isScalableVector())
24433 return convertToScalableVector(VT: LocVT, V: Val, DAG, Subtarget);
24434 llvm_unreachable("Unexpected Custom handling.");
24435 }
24436
24437 switch (VA.getLocInfo()) {
24438 default:
24439 llvm_unreachable("Unexpected CCValAssign::LocInfo");
24440 case CCValAssign::Full:
24441 break;
24442 case CCValAssign::BCvt:
24443 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: LocVT, Operand: Val);
24444 break;
24445 }
24446 return Val;
24447}
24448
24449// The caller is responsible for loading the full value if the argument is
24450// passed with CCValAssign::Indirect.
24451static SDValue unpackFromMemLoc(SelectionDAG &DAG, SDValue Chain,
24452 const CCValAssign &VA, const SDLoc &DL,
24453 const RISCVTargetLowering &TLI) {
24454 MachineFunction &MF = DAG.getMachineFunction();
24455 MachineFrameInfo &MFI = MF.getFrameInfo();
24456 EVT LocVT = VA.getLocVT();
24457 EVT PtrVT = MVT::getIntegerVT(BitWidth: DAG.getDataLayout().getPointerSizeInBits(AS: 0));
24458 int FI = MFI.CreateFixedObject(Size: LocVT.getStoreSize(), SPOffset: VA.getLocMemOffset(),
24459 /*IsImmutable=*/true);
24460 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
24461 SDValue Val = DAG.getLoad(
24462 VT: LocVT, dl: DL, Chain, Ptr: FIN,
24463 PtrInfo: MachinePointerInfo::getFixedStack(MF&: DAG.getMachineFunction(), FI));
24464
24465 if (VA.getLocInfo() == CCValAssign::Indirect)
24466 return Val;
24467
24468 return convertLocVTToValVT(DAG, Val, VA, DL, Subtarget: TLI.getSubtarget());
24469}
24470
24471static SDValue unpackF64OnRV32DSoftABI(SelectionDAG &DAG, SDValue Chain,
24472 const CCValAssign &VA,
24473 const CCValAssign &HiVA,
24474 const SDLoc &DL) {
24475 assert(VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64 &&
24476 "Unexpected VA");
24477 MachineFunction &MF = DAG.getMachineFunction();
24478 MachineFrameInfo &MFI = MF.getFrameInfo();
24479 MachineRegisterInfo &RegInfo = MF.getRegInfo();
24480
24481 assert(VA.isRegLoc() && "Expected register VA assignment");
24482
24483 Register LoVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24484 RegInfo.addLiveIn(Reg: VA.getLocReg(), vreg: LoVReg);
24485 SDValue Lo = DAG.getCopyFromReg(Chain, dl: DL, Reg: LoVReg, VT: MVT::i32);
24486 SDValue Hi;
24487 if (HiVA.isMemLoc()) {
24488 // Second half of f64 is passed on the stack.
24489 int FI = MFI.CreateFixedObject(Size: 4, SPOffset: HiVA.getLocMemOffset(),
24490 /*IsImmutable=*/true);
24491 SDValue FIN = DAG.getFrameIndex(FI, VT: MVT::i32);
24492 Hi = DAG.getLoad(VT: MVT::i32, dl: DL, Chain, Ptr: FIN,
24493 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI));
24494 } else {
24495 // Second half of f64 is passed in another GPR.
24496 Register HiVReg = RegInfo.createVirtualRegister(RegClass: &RISCV::GPRRegClass);
24497 RegInfo.addLiveIn(Reg: HiVA.getLocReg(), vreg: HiVReg);
24498 Hi = DAG.getCopyFromReg(Chain, dl: DL, Reg: HiVReg, VT: MVT::i32);
24499 }
24500
24501 // For big-endian, swap the order of Lo and Hi when building the pair.
24502 const RISCVSubtarget &Subtarget = DAG.getSubtarget<RISCVSubtarget>();
24503 if (!Subtarget.isLittleEndian())
24504 std::swap(a&: Lo, b&: Hi);
24505
24506 return DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
24507}
24508
24509// Transform physical registers into virtual registers.
24510SDValue RISCVTargetLowering::LowerFormalArguments(
24511 SDValue Chain, CallingConv::ID CallConv, bool IsVarArg,
24512 const SmallVectorImpl<ISD::InputArg> &Ins, const SDLoc &DL,
24513 SelectionDAG &DAG, SmallVectorImpl<SDValue> &InVals) const {
24514
24515 MachineFunction &MF = DAG.getMachineFunction();
24516 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
24517
24518 switch (CallConv) {
24519 default:
24520 reportFatalUsageError(reason: "Unsupported calling convention");
24521 case CallingConv::C:
24522 case CallingConv::Fast:
24523 case CallingConv::SPIR_KERNEL:
24524 case CallingConv::PreserveMost:
24525 case CallingConv::GRAAL:
24526 case CallingConv::RISCV_VectorCall:
24527#define CC_VLS_CASE(ABI_VLEN) case CallingConv::RISCV_VLSCall_##ABI_VLEN:
24528 CC_VLS_CASE(32)
24529 CC_VLS_CASE(64)
24530 CC_VLS_CASE(128)
24531 CC_VLS_CASE(256)
24532 CC_VLS_CASE(512)
24533 CC_VLS_CASE(1024)
24534 CC_VLS_CASE(2048)
24535 CC_VLS_CASE(4096)
24536 CC_VLS_CASE(8192)
24537 CC_VLS_CASE(16384)
24538 CC_VLS_CASE(32768)
24539 CC_VLS_CASE(65536)
24540#undef CC_VLS_CASE
24541 break;
24542 case CallingConv::GHC:
24543 if (Subtarget.hasStdExtE())
24544 reportFatalUsageError(reason: "GHC calling convention is not supported on RVE!");
24545 if (!Subtarget.hasStdExtFOrZfinx() || !Subtarget.hasStdExtDOrZdinx())
24546 reportFatalUsageError(reason: "GHC calling convention requires the (Zfinx/F) and "
24547 "(Zdinx/D) instruction set extensions");
24548 }
24549
24550 const Function &Func = MF.getFunction();
24551 if (Func.hasFnAttribute(Kind: "interrupt")) {
24552 if (!Func.arg_empty())
24553 reportFatalUsageError(
24554 reason: "Functions with the interrupt attribute cannot have arguments!");
24555
24556 StringRef Kind =
24557 MF.getFunction().getFnAttribute(Kind: "interrupt").getValueAsString();
24558
24559 constexpr StringLiteral SupportedInterruptKinds[] = {
24560 "machine",
24561 "supervisor",
24562 "rnmi",
24563 "qci-nest",
24564 "qci-nonest",
24565 "SiFive-CLIC-preemptible",
24566 "SiFive-CLIC-stack-swap",
24567 "SiFive-CLIC-preemptible-stack-swap",
24568 };
24569 if (!llvm::is_contained(Range: SupportedInterruptKinds, Element: Kind))
24570 reportFatalUsageError(
24571 reason: "Function interrupt attribute argument not supported!");
24572
24573 if (Kind.starts_with(Prefix: "qci-") && !Subtarget.hasVendorXqciint())
24574 reportFatalUsageError(
24575 reason: "'qci-*' interrupt kinds require Xqciint extension");
24576
24577 if (Kind.starts_with(Prefix: "SiFive-CLIC-") && !Subtarget.hasVendorXSfmclic())
24578 reportFatalUsageError(
24579 reason: "'SiFive-CLIC-*' interrupt kinds require XSfmclic extension");
24580
24581 if (Kind == "rnmi" && !Subtarget.hasStdExtSmrnmi())
24582 reportFatalUsageError(reason: "'rnmi' interrupt kind requires Srnmi extension");
24583 const TargetFrameLowering *TFI = Subtarget.getFrameLowering();
24584 if (Kind.starts_with(Prefix: "SiFive-CLIC-preemptible") && TFI->hasFP(MF))
24585 reportFatalUsageError(reason: "'SiFive-CLIC-preemptible' interrupt kinds cannot "
24586 "have a frame pointer");
24587 }
24588
24589 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
24590 MVT XLenVT = Subtarget.getXLenVT();
24591 unsigned XLenInBytes = Subtarget.getXLen() / 8;
24592 // Used with vargs to accumulate store chains.
24593 std::vector<SDValue> OutChains;
24594
24595 // Assign locations to all of the incoming arguments.
24596 SmallVector<CCValAssign, 16> ArgLocs;
24597 CCState CCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
24598
24599 if (CallConv == CallingConv::GHC)
24600 CCInfo.AnalyzeFormalArguments(Ins, Fn: CC_RISCV_GHC);
24601 else
24602 analyzeInputArgs(MF, CCInfo, Ins, /*IsRet=*/false,
24603 Fn: CallConv == CallingConv::Fast ? CC_RISCV_FastCC
24604 : CC_RISCV);
24605
24606 for (unsigned i = 0, e = ArgLocs.size(), InsIdx = 0; i != e; ++i, ++InsIdx) {
24607 CCValAssign &VA = ArgLocs[i];
24608 SDValue ArgValue;
24609 // Passing f64 on RV32D with a soft float ABI must be handled as a special
24610 // case.
24611 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
24612 assert(VA.needsCustom());
24613 ArgValue = unpackF64OnRV32DSoftABI(DAG, Chain, VA, HiVA: ArgLocs[++i], DL);
24614 } else if (VA.isRegLoc())
24615 ArgValue = unpackFromRegLoc(DAG, Chain, VA, DL, In: Ins[InsIdx], TLI: *this);
24616 else
24617 ArgValue = unpackFromMemLoc(DAG, Chain, VA, DL, TLI: *this);
24618
24619 if (VA.getLocInfo() == CCValAssign::Indirect) {
24620 // If the original argument was split and passed by reference (e.g. i128
24621 // on RV32), we need to load all parts of it here (using the same
24622 // address). Vectors may be partly split to registers and partly to the
24623 // stack, in which case the base address is partly offset and subsequent
24624 // stores are relative to that.
24625 InVals.push_back(Elt: DAG.getLoad(VT: VA.getValVT(), dl: DL, Chain, Ptr: ArgValue,
24626 PtrInfo: MachinePointerInfo()));
24627 unsigned ArgIndex = Ins[InsIdx].OrigArgIndex;
24628 unsigned ArgPartOffset = Ins[InsIdx].PartOffset;
24629 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
24630 while (i + 1 != e && Ins[InsIdx + 1].OrigArgIndex == ArgIndex) {
24631 CCValAssign &PartVA = ArgLocs[i + 1];
24632 unsigned PartOffset = Ins[InsIdx + 1].PartOffset - ArgPartOffset;
24633 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
24634 if (PartVA.getValVT().isScalableVector())
24635 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
24636 SDValue Address = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: ArgValue, N2: Offset);
24637 InVals.push_back(Elt: DAG.getLoad(VT: PartVA.getValVT(), dl: DL, Chain, Ptr: Address,
24638 PtrInfo: MachinePointerInfo()));
24639 ++i;
24640 ++InsIdx;
24641 }
24642 continue;
24643 }
24644 InVals.push_back(Elt: ArgValue);
24645 if (Ins[InsIdx].Flags.isByVal())
24646 RVFI->addIncomingByValArgs(Val: ArgValue);
24647 }
24648
24649 if (any_of(Range&: ArgLocs,
24650 P: [](CCValAssign &VA) { return VA.getLocVT().isScalableVector(); }))
24651 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
24652
24653 if (IsVarArg) {
24654 ArrayRef<MCPhysReg> ArgRegs = RISCV::getArgGPRs(ABI: Subtarget.getTargetABI());
24655 unsigned Idx = CCInfo.getFirstUnallocated(Regs: ArgRegs);
24656 const TargetRegisterClass *RC = &RISCV::GPRRegClass;
24657 MachineFrameInfo &MFI = MF.getFrameInfo();
24658 MachineRegisterInfo &RegInfo = MF.getRegInfo();
24659
24660 // Size of the vararg save area. For now, the varargs save area is either
24661 // zero or large enough to hold a0-a7.
24662 int VarArgsSaveSize = XLenInBytes * (ArgRegs.size() - Idx);
24663 int FI;
24664
24665 // If all registers are allocated, then all varargs must be passed on the
24666 // stack and we don't need to save any argregs.
24667 if (VarArgsSaveSize == 0) {
24668 int VaArgOffset = CCInfo.getStackSize();
24669 FI = MFI.CreateFixedObject(Size: XLenInBytes, SPOffset: VaArgOffset, IsImmutable: true);
24670 } else {
24671 int VaArgOffset = -VarArgsSaveSize;
24672 FI = MFI.CreateFixedObject(Size: VarArgsSaveSize, SPOffset: VaArgOffset, IsImmutable: true);
24673
24674 // If saving an odd number of registers then create an extra stack slot to
24675 // ensure that the frame pointer is 2*XLEN-aligned, which in turn ensures
24676 // offsets to even-numbered registered remain 2*XLEN-aligned.
24677 if (Idx % 2) {
24678 MFI.CreateFixedObject(
24679 Size: XLenInBytes, SPOffset: VaArgOffset - static_cast<int>(XLenInBytes), IsImmutable: true);
24680 VarArgsSaveSize += XLenInBytes;
24681 }
24682
24683 SDValue FIN = DAG.getFrameIndex(FI, VT: PtrVT);
24684
24685 // Copy the integer registers that may have been used for passing varargs
24686 // to the vararg save area.
24687 for (unsigned I = Idx; I < ArgRegs.size(); ++I) {
24688 const Register Reg = RegInfo.createVirtualRegister(RegClass: RC);
24689 RegInfo.addLiveIn(Reg: ArgRegs[I], vreg: Reg);
24690 SDValue ArgValue = DAG.getCopyFromReg(Chain, dl: DL, Reg, VT: XLenVT);
24691 SDValue Store = DAG.getStore(
24692 Chain, dl: DL, Val: ArgValue, Ptr: FIN,
24693 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI, Offset: (I - Idx) * XLenInBytes));
24694 OutChains.push_back(x: Store);
24695 FIN =
24696 DAG.getMemBasePlusOffset(Base: FIN, Offset: TypeSize::getFixed(ExactSize: XLenInBytes), DL);
24697 }
24698 }
24699
24700 // Record the frame index of the first variable argument
24701 // which is a value necessary to VASTART.
24702 RVFI->setVarArgsFrameIndex(FI);
24703 RVFI->setVarArgsSaveSize(VarArgsSaveSize);
24704 }
24705
24706 RVFI->setArgumentStackSize(CCInfo.getStackSize());
24707
24708 // All stores are grouped in one node to allow the matching between
24709 // the size of Ins and InVals. This only happens for vararg functions.
24710 if (!OutChains.empty()) {
24711 OutChains.push_back(x: Chain);
24712 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: OutChains);
24713 }
24714
24715 return Chain;
24716}
24717
24718/// isEligibleForTailCallOptimization - Check whether the call is eligible
24719/// for tail call optimization.
24720/// Note: This is modelled after ARM's IsEligibleForTailCallOptimization.
24721bool RISCVTargetLowering::isEligibleForTailCallOptimization(
24722 CCState &CCInfo, CallLoweringInfo &CLI, MachineFunction &MF,
24723 const SmallVector<CCValAssign, 16> &ArgLocs) const {
24724
24725 auto CalleeCC = CLI.CallConv;
24726 auto &Outs = CLI.Outs;
24727 auto &Caller = MF.getFunction();
24728 auto CallerCC = Caller.getCallingConv();
24729 auto *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
24730
24731 // Exception-handling functions need a special set of instructions to
24732 // indicate a return to the hardware. Tail-calling another function would
24733 // probably break this.
24734 // TODO: The "interrupt" attribute isn't currently defined by RISC-V. This
24735 // should be expanded as new function attributes are introduced.
24736 if (Caller.hasFnAttribute(Kind: "interrupt"))
24737 return false;
24738
24739 // If the stack arguments for this call do not fit into our own save area then
24740 // the call cannot be made tail.
24741 if (CCInfo.getStackSize() > RVFI->getArgumentStackSize())
24742 return false;
24743
24744 // Do not tail call opt if either caller or callee uses struct return
24745 // semantics.
24746 auto IsCallerStructRet = Caller.hasStructRetAttr();
24747 auto IsCalleeStructRet = Outs.empty() ? false : Outs[0].Flags.isSRet();
24748 if (IsCallerStructRet != IsCalleeStructRet)
24749 return false;
24750
24751 // Do not tail call opt if caller's and callee's byval arguments do not match.
24752 for (unsigned i = 0, j = 0, e = Outs.size(); i != e; ++i) {
24753 if (!Outs[i].Flags.isByVal())
24754 continue;
24755 if (j++ >= RVFI->getIncomingByValArgsSize())
24756 return false;
24757 if (RVFI->getIncomingByValArgs(Idx: i).getValueType() != Outs[i].ArgVT)
24758 return false;
24759 }
24760
24761 // The callee has to preserve all registers the caller needs to preserve.
24762 const RISCVRegisterInfo *TRI = Subtarget.getRegisterInfo();
24763 const uint32_t *CallerPreserved = TRI->getCallPreservedMask(MF, CallerCC);
24764 if (CalleeCC != CallerCC) {
24765 const uint32_t *CalleePreserved = TRI->getCallPreservedMask(MF, CalleeCC);
24766 if (!TRI->regmaskSubsetEqual(mask0: CallerPreserved, mask1: CalleePreserved))
24767 return false;
24768 }
24769
24770 // If the callee takes no arguments then go on to check the results of the
24771 // call.
24772 const MachineRegisterInfo &MRI = MF.getRegInfo();
24773 const SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
24774 if (!parametersInCSRMatch(MRI, CallerPreservedMask: CallerPreserved, ArgLocs, OutVals))
24775 return false;
24776
24777 return true;
24778}
24779
24780static Align getPrefTypeAlign(EVT VT, SelectionDAG &DAG) {
24781 return DAG.getDataLayout().getPrefTypeAlign(
24782 Ty: VT.getTypeForEVT(Context&: *DAG.getContext()));
24783}
24784
24785// Lower a call to a callseq_start + CALL + callseq_end chain, and add input
24786// and output parameter nodes.
24787SDValue RISCVTargetLowering::LowerCall(CallLoweringInfo &CLI,
24788 SmallVectorImpl<SDValue> &InVals) const {
24789 SelectionDAG &DAG = CLI.DAG;
24790 SDLoc &DL = CLI.DL;
24791 SmallVectorImpl<ISD::OutputArg> &Outs = CLI.Outs;
24792 SmallVectorImpl<SDValue> &OutVals = CLI.OutVals;
24793 SmallVectorImpl<ISD::InputArg> &Ins = CLI.Ins;
24794 SDValue Chain = CLI.Chain;
24795 SDValue Callee = CLI.Callee;
24796 bool &IsTailCall = CLI.IsTailCall;
24797 CallingConv::ID CallConv = CLI.CallConv;
24798 bool IsVarArg = CLI.IsVarArg;
24799 EVT PtrVT = getPointerTy(DL: DAG.getDataLayout());
24800 MVT XLenVT = Subtarget.getXLenVT();
24801 const CallBase *CB = CLI.CB;
24802
24803 MachineFunction &MF = DAG.getMachineFunction();
24804 RISCVMachineFunctionInfo *RVFI = MF.getInfo<RISCVMachineFunctionInfo>();
24805 MachineFunction::CallSiteInfo CSInfo;
24806
24807 // Set type id for call site info.
24808 setTypeIdForCallsiteInfo(CB, MF, CSInfo);
24809
24810 // Analyze the operands of the call, assigning locations to each operand.
24811 SmallVector<CCValAssign, 16> ArgLocs;
24812 CCState ArgCCInfo(CallConv, IsVarArg, MF, ArgLocs, *DAG.getContext());
24813
24814 if (CallConv == CallingConv::GHC) {
24815 if (Subtarget.hasStdExtE())
24816 reportFatalUsageError(reason: "GHC calling convention is not supported on RVE!");
24817 ArgCCInfo.AnalyzeCallOperands(Outs, Fn: CC_RISCV_GHC);
24818 } else
24819 analyzeOutputArgs(MF, CCInfo&: ArgCCInfo, Outs, /*IsRet=*/false, CLI: &CLI,
24820 Fn: CallConv == CallingConv::Fast ? CC_RISCV_FastCC
24821 : CC_RISCV);
24822
24823 // Check if it's really possible to do a tail call.
24824 if (IsTailCall)
24825 IsTailCall = isEligibleForTailCallOptimization(CCInfo&: ArgCCInfo, CLI, MF, ArgLocs);
24826
24827 if (IsTailCall)
24828 ++NumTailCalls;
24829 else if (CLI.CB && CLI.CB->isMustTailCall())
24830 reportFatalInternalError(reason: "failed to perform tail call elimination on a "
24831 "call site marked musttail");
24832
24833 // Get a count of how many bytes are to be pushed on the stack.
24834 unsigned NumBytes = ArgCCInfo.getStackSize();
24835
24836 // Create local copies for byval args
24837 SmallVector<SDValue, 8> ByValArgs;
24838 for (unsigned i = 0, j = 0, e = Outs.size(); i != e; ++i) {
24839 ISD::ArgFlagsTy Flags = Outs[i].Flags;
24840 if (!Flags.isByVal())
24841 continue;
24842
24843 SDValue Arg = OutVals[i];
24844 unsigned Size = Flags.getByValSize();
24845 Align Alignment = Flags.getNonZeroByValAlign();
24846
24847 SDValue SizeNode = DAG.getConstant(Val: Size, DL, VT: XLenVT);
24848 SDValue Dst;
24849
24850 if (IsTailCall) {
24851 SDValue CallerArg = RVFI->getIncomingByValArgs(Idx: j++);
24852 if (isa<GlobalAddressSDNode>(Val: Arg) || isa<ExternalSymbolSDNode>(Val: Arg) ||
24853 isa<FrameIndexSDNode>(Val: Arg))
24854 Dst = CallerArg;
24855 } else {
24856 int FI =
24857 MF.getFrameInfo().CreateStackObject(Size, Alignment, /*isSS=*/isSpillSlot: false);
24858 Dst = DAG.getFrameIndex(FI, VT: getPointerTy(DL: DAG.getDataLayout()));
24859 }
24860 if (Dst) {
24861 Chain =
24862 DAG.getMemcpy(Chain, dl: DL, Dst, Src: Arg, Size: SizeNode, Alignment,
24863 /*IsVolatile=*/isVol: false,
24864 /*AlwaysInline=*/false, /*CI=*/nullptr, OverrideTailCall: std::nullopt,
24865 DstPtrInfo: MachinePointerInfo(), SrcPtrInfo: MachinePointerInfo());
24866 ByValArgs.push_back(Elt: Dst);
24867 }
24868 }
24869
24870 if (!IsTailCall)
24871 Chain = DAG.getCALLSEQ_START(Chain, InSize: NumBytes, OutSize: 0, DL: CLI.DL);
24872
24873 // During a tail call, stores to the argument area must happen after all of
24874 // the function's incoming arguments have been loaded because they may alias.
24875 // This is done by folding in a TokenFactor from LowerFormalArguments, but
24876 // there's no point in doing so repeatedly so this tracks whether that's
24877 // happened yet.
24878 bool AfterFormalArgLoads = false;
24879
24880 // Copy argument values to their designated locations.
24881 SmallVector<std::pair<Register, SDValue>, 8> RegsToPass;
24882 SmallVector<SDValue, 8> MemOpChains;
24883 SDValue StackPtr;
24884 for (unsigned i = 0, j = 0, e = ArgLocs.size(), OutIdx = 0; i != e;
24885 ++i, ++OutIdx) {
24886 CCValAssign &VA = ArgLocs[i];
24887 SDValue ArgValue = OutVals[OutIdx];
24888 ISD::ArgFlagsTy Flags = Outs[OutIdx].Flags;
24889
24890 // Handle passing f64 on RV32D with a soft float ABI as a special case.
24891 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
24892 assert(VA.isRegLoc() && "Expected register VA assignment");
24893 assert(VA.needsCustom());
24894 SDValue SplitF64 = DAG.getNode(
24895 Opcode: RISCVISD::SplitF64, DL, VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: ArgValue);
24896 SDValue Lo = SplitF64.getValue(R: 0);
24897 SDValue Hi = SplitF64.getValue(R: 1);
24898
24899 // For big-endian, swap the order of Lo and Hi when passing.
24900 if (!Subtarget.isLittleEndian())
24901 std::swap(a&: Lo, b&: Hi);
24902
24903 Register RegLo = VA.getLocReg();
24904 RegsToPass.push_back(Elt: std::make_pair(x&: RegLo, y&: Lo));
24905
24906 // Get the CCValAssign for the Hi part.
24907 CCValAssign &HiVA = ArgLocs[++i];
24908
24909 if (HiVA.isMemLoc()) {
24910 // Second half of f64 is passed on the stack.
24911 if (!StackPtr.getNode())
24912 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
24913 SDValue Address =
24914 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr,
24915 N2: DAG.getIntPtrConstant(Val: HiVA.getLocMemOffset(), DL));
24916 // Emit the store.
24917 MemOpChains.push_back(Elt: DAG.getStore(
24918 Chain, dl: DL, Val: Hi, Ptr: Address,
24919 PtrInfo: MachinePointerInfo::getStack(MF, Offset: HiVA.getLocMemOffset())));
24920 } else {
24921 // Second half of f64 is passed in another GPR.
24922 Register RegHigh = HiVA.getLocReg();
24923 RegsToPass.push_back(Elt: std::make_pair(x&: RegHigh, y&: Hi));
24924 }
24925 continue;
24926 }
24927
24928 // Promote the value if needed.
24929 // For now, only handle fully promoted and indirect arguments.
24930 if (VA.getLocInfo() == CCValAssign::Indirect) {
24931 // Store the argument in a stack slot and pass its address.
24932 Align StackAlign =
24933 std::max(a: getPrefTypeAlign(VT: Outs[OutIdx].ArgVT, DAG),
24934 b: getPrefTypeAlign(VT: ArgValue.getValueType(), DAG));
24935 TypeSize StoredSize = ArgValue.getValueType().getStoreSize();
24936 // If the original argument was split (e.g. i128), we need
24937 // to store the required parts of it here (and pass just one address).
24938 // Vectors may be partly split to registers and partly to the stack, in
24939 // which case the base address is partly offset and subsequent stores are
24940 // relative to that.
24941 unsigned ArgIndex = Outs[OutIdx].OrigArgIndex;
24942 unsigned ArgPartOffset = Outs[OutIdx].PartOffset;
24943 assert(VA.getValVT().isVector() || ArgPartOffset == 0);
24944 // Calculate the total size to store. We don't have access to what we're
24945 // actually storing other than performing the loop and collecting the
24946 // info.
24947 SmallVector<std::pair<SDValue, SDValue>> Parts;
24948 while (i + 1 != e && Outs[OutIdx + 1].OrigArgIndex == ArgIndex) {
24949 SDValue PartValue = OutVals[OutIdx + 1];
24950 unsigned PartOffset = Outs[OutIdx + 1].PartOffset - ArgPartOffset;
24951 SDValue Offset = DAG.getIntPtrConstant(Val: PartOffset, DL);
24952 EVT PartVT = PartValue.getValueType();
24953 if (PartVT.isScalableVector())
24954 Offset = DAG.getNode(Opcode: ISD::VSCALE, DL, VT: XLenVT, Operand: Offset);
24955 StoredSize += PartVT.getStoreSize();
24956 StackAlign = std::max(a: StackAlign, b: getPrefTypeAlign(VT: PartVT, DAG));
24957 Parts.push_back(Elt: std::make_pair(x&: PartValue, y&: Offset));
24958 ++i;
24959 ++OutIdx;
24960 }
24961 SDValue SpillSlot = DAG.CreateStackTemporary(Bytes: StoredSize, Alignment: StackAlign);
24962 int FI = cast<FrameIndexSDNode>(Val&: SpillSlot)->getIndex();
24963 MemOpChains.push_back(
24964 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: SpillSlot,
24965 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
24966 for (const auto &Part : Parts) {
24967 SDValue PartValue = Part.first;
24968 SDValue PartOffset = Part.second;
24969 SDValue Address =
24970 DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: SpillSlot, N2: PartOffset);
24971 MemOpChains.push_back(
24972 Elt: DAG.getStore(Chain, dl: DL, Val: PartValue, Ptr: Address,
24973 PtrInfo: MachinePointerInfo::getFixedStack(MF, FI)));
24974 }
24975 ArgValue = SpillSlot;
24976 } else {
24977 ArgValue = convertValVTToLocVT(DAG, Val: ArgValue, VA, DL, Subtarget);
24978 }
24979
24980 // Use local copy if it is a byval arg.
24981 if (Flags.isByVal()) {
24982 if (!IsTailCall || (isa<GlobalAddressSDNode>(Val: ArgValue) ||
24983 isa<ExternalSymbolSDNode>(Val: ArgValue) ||
24984 isa<FrameIndexSDNode>(Val: ArgValue)))
24985 ArgValue = ByValArgs[j++];
24986 }
24987
24988 if (VA.isRegLoc()) {
24989 // Queue up the argument copies and emit them at the end.
24990 RegsToPass.push_back(Elt: std::make_pair(x: VA.getLocReg(), y&: ArgValue));
24991
24992 const TargetOptions &Options = DAG.getTarget().Options;
24993 if (Options.EmitCallSiteInfo)
24994 CSInfo.ArgRegPairs.emplace_back(Args: VA.getLocReg(), Args&: i);
24995 } else {
24996 assert(VA.isMemLoc() && "Argument not register or memory");
24997 SDValue DstAddr;
24998 MachinePointerInfo DstInfo;
24999 int32_t Offset = VA.getLocMemOffset();
25000
25001 // Work out the address of the stack slot.
25002 if (!StackPtr.getNode())
25003 StackPtr = DAG.getCopyFromReg(Chain, dl: DL, Reg: RISCV::X2, VT: PtrVT);
25004
25005 if (IsTailCall) {
25006 unsigned OpSize = divideCeil(Numerator: VA.getValVT().getSizeInBits(), Denominator: 8);
25007 int FI = MF.getFrameInfo().CreateFixedObject(Size: OpSize, SPOffset: Offset, IsImmutable: true);
25008 DstAddr = DAG.getFrameIndex(FI, VT: PtrVT);
25009 DstInfo = MachinePointerInfo::getFixedStack(MF, FI);
25010 if (!AfterFormalArgLoads) {
25011 Chain = DAG.getStackArgumentTokenFactor(Chain);
25012 AfterFormalArgLoads = true;
25013 }
25014 } else {
25015 SDValue PtrOff = DAG.getIntPtrConstant(Val: Offset, DL);
25016 DstAddr = DAG.getNode(Opcode: ISD::ADD, DL, VT: PtrVT, N1: StackPtr, N2: PtrOff);
25017 DstInfo = MachinePointerInfo::getStack(MF, Offset);
25018 }
25019
25020 // Emit the store.
25021 MemOpChains.push_back(
25022 Elt: DAG.getStore(Chain, dl: DL, Val: ArgValue, Ptr: DstAddr, PtrInfo: DstInfo));
25023 }
25024 }
25025
25026 // Join the stores, which are independent of one another.
25027 if (!MemOpChains.empty())
25028 Chain = DAG.getNode(Opcode: ISD::TokenFactor, DL, VT: MVT::Other, Ops: MemOpChains);
25029
25030 SDValue Glue;
25031
25032 // Build a sequence of copy-to-reg nodes, chained and glued together.
25033 for (auto &Reg : RegsToPass) {
25034 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: Reg.first, N: Reg.second, Glue);
25035 Glue = Chain.getValue(R: 1);
25036 }
25037
25038 // Validate that none of the argument registers have been marked as
25039 // reserved, if so report an error. Do the same for the return address if this
25040 // is not a tailcall.
25041 validateCCReservedRegs(Regs: RegsToPass, MF);
25042 if (!IsTailCall && MF.getSubtarget().isRegisterReservedByUser(R: RISCV::X1))
25043 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
25044 MF.getFunction(),
25045 "Return address register required, but has been reserved."});
25046
25047 // If the callee is a GlobalAddress/ExternalSymbol node, turn it into a
25048 // TargetGlobalAddress/TargetExternalSymbol node so that legalize won't
25049 // split it and then direct call can be matched by PseudoCALL.
25050 bool CalleeIsLargeExternalSymbol = false;
25051 if (getTargetMachine().getCodeModel() == CodeModel::Large) {
25052 if (auto *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee))
25053 Callee = getLargeGlobalAddress(N: S, DL, Ty: PtrVT, DAG);
25054 else if (auto *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
25055 Callee = getLargeExternalSymbol(N: S, DL, Ty: PtrVT, DAG);
25056 CalleeIsLargeExternalSymbol = true;
25057 }
25058 } else if (GlobalAddressSDNode *S = dyn_cast<GlobalAddressSDNode>(Val&: Callee)) {
25059 const GlobalValue *GV = S->getGlobal();
25060 Callee = DAG.getTargetGlobalAddress(GV, DL, VT: PtrVT, offset: 0, TargetFlags: RISCVII::MO_CALL);
25061 } else if (ExternalSymbolSDNode *S = dyn_cast<ExternalSymbolSDNode>(Val&: Callee)) {
25062 Callee = DAG.getTargetExternalSymbol(Sym: S->getSymbol(), VT: PtrVT, TargetFlags: RISCVII::MO_CALL);
25063 }
25064
25065 // The first call operand is the chain and the second is the target address.
25066 SmallVector<SDValue, 8> Ops;
25067 Ops.push_back(Elt: Chain);
25068 Ops.push_back(Elt: Callee);
25069
25070 // Add argument registers to the end of the list so that they are
25071 // known live into the call.
25072 for (auto &Reg : RegsToPass)
25073 Ops.push_back(Elt: DAG.getRegister(Reg: Reg.first, VT: Reg.second.getValueType()));
25074
25075 // Add a register mask operand representing the call-preserved registers.
25076 const TargetRegisterInfo *TRI = Subtarget.getRegisterInfo();
25077 const uint32_t *Mask = TRI->getCallPreservedMask(MF, CallConv);
25078 assert(Mask && "Missing call preserved mask for calling convention");
25079 Ops.push_back(Elt: DAG.getRegisterMask(RegMask: Mask));
25080
25081 // Glue the call to the argument copies, if any.
25082 if (Glue.getNode())
25083 Ops.push_back(Elt: Glue);
25084
25085 assert((!CLI.CFIType || CLI.CB->isIndirectCall()) &&
25086 "Unexpected CFI type for a direct call");
25087
25088 // Emit the call.
25089 SDVTList NodeTys = DAG.getVTList(VT1: MVT::Other, VT2: MVT::Glue);
25090
25091 // Use software guarded branch for large code model non-indirect calls
25092 // Tail call to external symbol will have a null CLI.CB and we need another
25093 // way to determine the callsite type
25094 bool NeedSWGuarded = false;
25095 if (getTargetMachine().getCodeModel() == CodeModel::Large &&
25096 Subtarget.hasStdExtZicfilp() &&
25097 ((CLI.CB && !CLI.CB->isIndirectCall()) || CalleeIsLargeExternalSymbol))
25098 NeedSWGuarded = true;
25099
25100 if (IsTailCall) {
25101 MF.getFrameInfo().setHasTailCall();
25102 unsigned CallOpc =
25103 NeedSWGuarded ? RISCVISD::SW_GUARDED_TAIL : RISCVISD::TAIL;
25104 SDValue Ret = DAG.getNode(Opcode: CallOpc, DL, VTList: NodeTys, Ops);
25105 if (CLI.CFIType)
25106 Ret.getNode()->setCFIType(CLI.CFIType->getZExtValue());
25107 DAG.addNoMergeSiteInfo(Node: Ret.getNode(), NoMerge: CLI.NoMerge);
25108 DAG.addCallSiteInfo(Node: Ret.getNode(), CallInfo: std::move(CSInfo));
25109 return Ret;
25110 }
25111
25112 unsigned CallOpc = NeedSWGuarded ? RISCVISD::SW_GUARDED_CALL : RISCVISD::CALL;
25113 Chain = DAG.getNode(Opcode: CallOpc, DL, VTList: NodeTys, Ops);
25114 if (CLI.CFIType)
25115 Chain.getNode()->setCFIType(CLI.CFIType->getZExtValue());
25116
25117 DAG.addNoMergeSiteInfo(Node: Chain.getNode(), NoMerge: CLI.NoMerge);
25118 DAG.addCallSiteInfo(Node: Chain.getNode(), CallInfo: std::move(CSInfo));
25119 Glue = Chain.getValue(R: 1);
25120
25121 // Mark the end of the call, which is glued to the call itself.
25122 Chain = DAG.getCALLSEQ_END(Chain, Size1: NumBytes, Size2: 0, Glue, DL);
25123 Glue = Chain.getValue(R: 1);
25124
25125 // Assign locations to each value returned by this call.
25126 SmallVector<CCValAssign, 16> RVLocs;
25127 CCState RetCCInfo(CallConv, IsVarArg, MF, RVLocs, *DAG.getContext());
25128 analyzeInputArgs(MF, CCInfo&: RetCCInfo, Ins, /*IsRet=*/true, Fn: CC_RISCV);
25129
25130 // Copy all of the result registers out of their specified physreg.
25131 for (unsigned i = 0, e = RVLocs.size(); i != e; ++i) {
25132 auto &VA = RVLocs[i];
25133 // Copy the value out
25134 SDValue RetValue =
25135 DAG.getCopyFromReg(Chain, dl: DL, Reg: VA.getLocReg(), VT: VA.getLocVT(), Glue);
25136 // Glue the RetValue to the end of the call sequence
25137 Chain = RetValue.getValue(R: 1);
25138 Glue = RetValue.getValue(R: 2);
25139
25140 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
25141 assert(VA.needsCustom());
25142 SDValue RetValue2 = DAG.getCopyFromReg(Chain, dl: DL, Reg: RVLocs[++i].getLocReg(),
25143 VT: MVT::i32, Glue);
25144 Chain = RetValue2.getValue(R: 1);
25145 Glue = RetValue2.getValue(R: 2);
25146
25147 // For big-endian, swap the order when building the pair.
25148 SDValue Lo = RetValue;
25149 SDValue Hi = RetValue2;
25150 if (!Subtarget.isLittleEndian())
25151 std::swap(a&: Lo, b&: Hi);
25152
25153 RetValue = DAG.getNode(Opcode: RISCVISD::BuildPairF64, DL, VT: MVT::f64, N1: Lo, N2: Hi);
25154 } else
25155 RetValue = convertLocVTToValVT(DAG, Val: RetValue, VA, DL, Subtarget);
25156
25157 InVals.push_back(Elt: RetValue);
25158 }
25159
25160 return Chain;
25161}
25162
25163bool RISCVTargetLowering::CanLowerReturn(
25164 CallingConv::ID CallConv, MachineFunction &MF, bool IsVarArg,
25165 const SmallVectorImpl<ISD::OutputArg> &Outs, LLVMContext &Context,
25166 const Type *RetTy) const {
25167 SmallVector<CCValAssign, 16> RVLocs;
25168 CCState CCInfo(CallConv, IsVarArg, MF, RVLocs, Context);
25169
25170 for (unsigned i = 0, e = Outs.size(); i != e; ++i) {
25171 MVT VT = Outs[i].VT;
25172 ISD::ArgFlagsTy ArgFlags = Outs[i].Flags;
25173 if (CC_RISCV(ValNo: i, ValVT: VT, LocVT: VT, LocInfo: CCValAssign::Full, ArgFlags, State&: CCInfo,
25174 /*IsRet=*/true, OrigTy: Outs[i].OrigTy))
25175 return false;
25176 }
25177 return true;
25178}
25179
25180SDValue
25181RISCVTargetLowering::LowerReturn(SDValue Chain, CallingConv::ID CallConv,
25182 bool IsVarArg,
25183 const SmallVectorImpl<ISD::OutputArg> &Outs,
25184 const SmallVectorImpl<SDValue> &OutVals,
25185 const SDLoc &DL, SelectionDAG &DAG) const {
25186 MachineFunction &MF = DAG.getMachineFunction();
25187
25188 // Stores the assignment of the return value to a location.
25189 SmallVector<CCValAssign, 16> RVLocs;
25190
25191 // Info about the registers and stack slot.
25192 CCState CCInfo(CallConv, IsVarArg, DAG.getMachineFunction(), RVLocs,
25193 *DAG.getContext());
25194
25195 analyzeOutputArgs(MF&: DAG.getMachineFunction(), CCInfo, Outs, /*IsRet=*/true,
25196 CLI: nullptr, Fn: CC_RISCV);
25197
25198 if (CallConv == CallingConv::GHC && !RVLocs.empty())
25199 reportFatalUsageError(reason: "GHC functions return void only");
25200
25201 SDValue Glue;
25202 SmallVector<SDValue, 4> RetOps(1, Chain);
25203
25204 // Copy the result values into the output registers.
25205 for (unsigned i = 0, e = RVLocs.size(), OutIdx = 0; i < e; ++i, ++OutIdx) {
25206 SDValue Val = OutVals[OutIdx];
25207 CCValAssign &VA = RVLocs[i];
25208 assert(VA.isRegLoc() && "Can only return in registers!");
25209
25210 if (VA.getLocVT() == MVT::i32 && VA.getValVT() == MVT::f64) {
25211 // Handle returning f64 on RV32D with a soft float ABI.
25212 assert(VA.isRegLoc() && "Expected return via registers");
25213 assert(VA.needsCustom());
25214 SDValue SplitF64 = DAG.getNode(Opcode: RISCVISD::SplitF64, DL,
25215 VTList: DAG.getVTList(VT1: MVT::i32, VT2: MVT::i32), N: Val);
25216 SDValue Lo = SplitF64.getValue(R: 0);
25217 SDValue Hi = SplitF64.getValue(R: 1);
25218
25219 // For big-endian, swap the order of Lo and Hi when returning.
25220 if (!Subtarget.isLittleEndian())
25221 std::swap(a&: Lo, b&: Hi);
25222
25223 Register RegLo = VA.getLocReg();
25224 Register RegHi = RVLocs[++i].getLocReg();
25225
25226 if (Subtarget.isRegisterReservedByUser(i: RegLo) ||
25227 Subtarget.isRegisterReservedByUser(i: RegHi))
25228 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
25229 MF.getFunction(),
25230 "Return value register required, but has been reserved."});
25231
25232 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegLo, N: Lo, Glue);
25233 Glue = Chain.getValue(R: 1);
25234 RetOps.push_back(Elt: DAG.getRegister(Reg: RegLo, VT: MVT::i32));
25235 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: RegHi, N: Hi, Glue);
25236 Glue = Chain.getValue(R: 1);
25237 RetOps.push_back(Elt: DAG.getRegister(Reg: RegHi, VT: MVT::i32));
25238 } else {
25239 // Handle a 'normal' return.
25240 Val = convertValVTToLocVT(DAG, Val, VA, DL, Subtarget);
25241 Chain = DAG.getCopyToReg(Chain, dl: DL, Reg: VA.getLocReg(), N: Val, Glue);
25242
25243 if (Subtarget.isRegisterReservedByUser(i: VA.getLocReg()))
25244 MF.getFunction().getContext().diagnose(DI: DiagnosticInfoUnsupported{
25245 MF.getFunction(),
25246 "Return value register required, but has been reserved."});
25247
25248 // Guarantee that all emitted copies are stuck together.
25249 Glue = Chain.getValue(R: 1);
25250 RetOps.push_back(Elt: DAG.getRegister(Reg: VA.getLocReg(), VT: VA.getLocVT()));
25251 }
25252 }
25253
25254 RetOps[0] = Chain; // Update chain.
25255
25256 // Add the glue node if we have it.
25257 if (Glue.getNode()) {
25258 RetOps.push_back(Elt: Glue);
25259 }
25260
25261 if (any_of(Range&: RVLocs,
25262 P: [](CCValAssign &VA) { return VA.getLocVT().isScalableVector(); }))
25263 MF.getInfo<RISCVMachineFunctionInfo>()->setIsVectorCall();
25264
25265 unsigned RetOpc = RISCVISD::RET_GLUE;
25266 // Interrupt service routines use different return instructions.
25267 const Function &Func = DAG.getMachineFunction().getFunction();
25268 if (Func.hasFnAttribute(Kind: "interrupt")) {
25269 if (!Func.getReturnType()->isVoidTy())
25270 reportFatalUsageError(
25271 reason: "Functions with the interrupt attribute must have void return type!");
25272
25273 MachineFunction &MF = DAG.getMachineFunction();
25274 StringRef Kind =
25275 MF.getFunction().getFnAttribute(Kind: "interrupt").getValueAsString();
25276
25277 if (Kind == "supervisor")
25278 RetOpc = RISCVISD::SRET_GLUE;
25279 else if (Kind == "rnmi") {
25280 assert(Subtarget.hasFeature(RISCV::FeatureStdExtSmrnmi) &&
25281 "Need Smrnmi extension for rnmi");
25282 RetOpc = RISCVISD::MNRET_GLUE;
25283 } else if (Kind == "qci-nest" || Kind == "qci-nonest") {
25284 assert(Subtarget.hasFeature(RISCV::FeatureVendorXqciint) &&
25285 "Need Xqciint for qci-(no)nest");
25286 RetOpc = RISCVISD::QC_C_MILEAVERET_GLUE;
25287 } else
25288 RetOpc = RISCVISD::MRET_GLUE;
25289 }
25290
25291 return DAG.getNode(Opcode: RetOpc, DL, VT: MVT::Other, Ops: RetOps);
25292}
25293
25294void RISCVTargetLowering::validateCCReservedRegs(
25295 const SmallVectorImpl<std::pair<llvm::Register, llvm::SDValue>> &Regs,
25296 MachineFunction &MF) const {
25297 const Function &F = MF.getFunction();
25298
25299 if (llvm::any_of(Range: Regs, P: [this](auto Reg) {
25300 return Subtarget.isRegisterReservedByUser(i: Reg.first);
25301 }))
25302 F.getContext().diagnose(DI: DiagnosticInfoUnsupported{
25303 F, "Argument register required, but has been reserved."});
25304}
25305
25306// Check if the result of the node is only used as a return value, as
25307// otherwise we can't perform a tail-call.
25308bool RISCVTargetLowering::isUsedByReturnOnly(SDNode *N, SDValue &Chain) const {
25309 if (N->getNumValues() != 1)
25310 return false;
25311 if (!N->hasNUsesOfValue(NUses: 1, Value: 0))
25312 return false;
25313
25314 SDNode *Copy = *N->user_begin();
25315
25316 if (Copy->getOpcode() == ISD::BITCAST) {
25317 return isUsedByReturnOnly(N: Copy, Chain);
25318 }
25319
25320 // TODO: Handle additional opcodes in order to support tail-calling libcalls
25321 // with soft float ABIs.
25322 if (Copy->getOpcode() != ISD::CopyToReg) {
25323 return false;
25324 }
25325
25326 // If the ISD::CopyToReg has a glue operand, we conservatively assume it
25327 // isn't safe to perform a tail call.
25328 if (Copy->getOperand(Num: Copy->getNumOperands() - 1).getValueType() == MVT::Glue)
25329 return false;
25330
25331 // The copy must be used by a RISCVISD::RET_GLUE, and nothing else.
25332 bool HasRet = false;
25333 for (SDNode *Node : Copy->users()) {
25334 if (Node->getOpcode() != RISCVISD::RET_GLUE)
25335 return false;
25336 HasRet = true;
25337 }
25338 if (!HasRet)
25339 return false;
25340
25341 Chain = Copy->getOperand(Num: 0);
25342 return true;
25343}
25344
25345bool RISCVTargetLowering::mayBeEmittedAsTailCall(const CallInst *CI) const {
25346 return CI->isTailCall();
25347}
25348
25349/// getConstraintType - Given a constraint letter, return the type of
25350/// constraint it is for this target.
25351RISCVTargetLowering::ConstraintType
25352RISCVTargetLowering::getConstraintType(StringRef Constraint) const {
25353 if (Constraint.size() == 1) {
25354 switch (Constraint[0]) {
25355 default:
25356 break;
25357 case 'f':
25358 case 'R':
25359 return C_RegisterClass;
25360 case 'I':
25361 case 'J':
25362 case 'K':
25363 return C_Immediate;
25364 case 'A':
25365 return C_Memory;
25366 case 's':
25367 case 'S': // A symbolic address
25368 return C_Other;
25369 }
25370 } else {
25371 if (Constraint == "vr" || Constraint == "vd" || Constraint == "vm")
25372 return C_RegisterClass;
25373 if (Constraint == "cr" || Constraint == "cR" || Constraint == "cf")
25374 return C_RegisterClass;
25375 }
25376 return TargetLowering::getConstraintType(Constraint);
25377}
25378
25379std::pair<unsigned, const TargetRegisterClass *>
25380RISCVTargetLowering::getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
25381 StringRef Constraint,
25382 MVT VT) const {
25383 // First, see if this is a constraint that directly corresponds to a RISC-V
25384 // register class.
25385 if (Constraint.size() == 1) {
25386 switch (Constraint[0]) {
25387 case 'r':
25388 // TODO: Support fixed vectors up to XLen for P extension?
25389 if (VT.isVector())
25390 break;
25391 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
25392 return std::make_pair(x: 0U, y: &RISCV::GPRF16NoX0RegClass);
25393 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
25394 return std::make_pair(x: 0U, y: &RISCV::GPRF32NoX0RegClass);
25395 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
25396 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
25397 return std::make_pair(x: 0U, y: &RISCV::GPRNoX0RegClass);
25398 case 'f':
25399 if (VT == MVT::f16) {
25400 if (Subtarget.hasStdExtZfhmin())
25401 return std::make_pair(x: 0U, y: &RISCV::FPR16RegClass);
25402 if (Subtarget.hasStdExtZhinxmin())
25403 return std::make_pair(x: 0U, y: &RISCV::GPRF16NoX0RegClass);
25404 } else if (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
25405 return std::make_pair(x: 0U, y: &RISCV::FPR16RegClass);
25406 } else if (VT == MVT::f32) {
25407 if (Subtarget.hasStdExtF())
25408 return std::make_pair(x: 0U, y: &RISCV::FPR32RegClass);
25409 if (Subtarget.hasStdExtZfinx())
25410 return std::make_pair(x: 0U, y: &RISCV::GPRF32NoX0RegClass);
25411 } else if (VT == MVT::f64) {
25412 if (Subtarget.hasStdExtD())
25413 return std::make_pair(x: 0U, y: &RISCV::FPR64RegClass);
25414 if (Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
25415 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
25416 if (Subtarget.hasStdExtZdinx() && Subtarget.is64Bit())
25417 return std::make_pair(x: 0U, y: &RISCV::GPRNoX0RegClass);
25418 }
25419 break;
25420 case 'R':
25421 if (((VT == MVT::i64 || VT == MVT::f64) && !Subtarget.is64Bit()) ||
25422 (VT == MVT::i128 && Subtarget.is64Bit()))
25423 return std::make_pair(x: 0U, y: &RISCV::GPRPairNoX0RegClass);
25424 break;
25425 default:
25426 break;
25427 }
25428 } else if (Constraint == "vr") {
25429 // Check VM and fractional LMUL first so that those types will use that
25430 // class instead of VR.
25431 for (const auto *RC :
25432 {&RISCV::ZZZ_VMRegClass, &RISCV::ZZZ_VRMF8RegClass,
25433 &RISCV::ZZZ_VRMF4RegClass, &RISCV::ZZZ_VRMF2RegClass,
25434 &RISCV::VRRegClass, &RISCV::VRM2RegClass, &RISCV::VRM4RegClass,
25435 &RISCV::VRM8RegClass, &RISCV::VRN2M1RegClass, &RISCV::VRN3M1RegClass,
25436 &RISCV::VRN4M1RegClass, &RISCV::VRN5M1RegClass,
25437 &RISCV::VRN6M1RegClass, &RISCV::VRN7M1RegClass,
25438 &RISCV::VRN8M1RegClass, &RISCV::VRN2M2RegClass,
25439 &RISCV::VRN3M2RegClass, &RISCV::VRN4M2RegClass,
25440 &RISCV::VRN2M4RegClass}) {
25441 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy))
25442 return std::make_pair(x: 0U, y&: RC);
25443
25444 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
25445 MVT ContainerVT = getContainerForFixedLengthVector(VT);
25446 if (TRI->isTypeLegalForClass(RC: *RC, T: ContainerVT))
25447 return std::make_pair(x: 0U, y&: RC);
25448 }
25449 }
25450 } else if (Constraint == "vd") {
25451 // Check VMNoV0 and fractional LMUL first so that those types will use that
25452 // class instead of VRNoV0.
25453 for (const auto *RC :
25454 {&RISCV::ZZZ_VMNoV0RegClass, &RISCV::ZZZ_VRMF8NoV0RegClass,
25455 &RISCV::ZZZ_VRMF4NoV0RegClass, &RISCV::ZZZ_VRMF2NoV0RegClass,
25456 &RISCV::VRNoV0RegClass, &RISCV::VRM2NoV0RegClass,
25457 &RISCV::VRM4NoV0RegClass, &RISCV::VRM8NoV0RegClass,
25458 &RISCV::VRN2M1NoV0RegClass, &RISCV::VRN3M1NoV0RegClass,
25459 &RISCV::VRN4M1NoV0RegClass, &RISCV::VRN5M1NoV0RegClass,
25460 &RISCV::VRN6M1NoV0RegClass, &RISCV::VRN7M1NoV0RegClass,
25461 &RISCV::VRN8M1NoV0RegClass, &RISCV::VRN2M2NoV0RegClass,
25462 &RISCV::VRN3M2NoV0RegClass, &RISCV::VRN4M2NoV0RegClass,
25463 &RISCV::VRN2M4NoV0RegClass}) {
25464 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy))
25465 return std::make_pair(x: 0U, y&: RC);
25466
25467 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
25468 MVT ContainerVT = getContainerForFixedLengthVector(VT);
25469 if (TRI->isTypeLegalForClass(RC: *RC, T: ContainerVT))
25470 return std::make_pair(x: 0U, y&: RC);
25471 }
25472 }
25473 } else if (Constraint == "vm") {
25474 if (TRI->isTypeLegalForClass(RC: RISCV::VMV0RegClass, T: VT.SimpleTy))
25475 return std::make_pair(x: 0U, y: &RISCV::VMV0RegClass);
25476
25477 if (VT.isFixedLengthVector() && useRVVForFixedLengthVectorVT(VT)) {
25478 MVT ContainerVT = getContainerForFixedLengthVector(VT);
25479 // VT here might be coerced to vector with i8 elements, so we need to
25480 // check if this is a M1 register here instead of checking VMV0RegClass.
25481 if (TRI->isTypeLegalForClass(RC: RISCV::VRRegClass, T: ContainerVT))
25482 return std::make_pair(x: 0U, y: &RISCV::VMV0RegClass);
25483 }
25484 } else if (Constraint == "cr") {
25485 if (VT == MVT::f16 && Subtarget.hasStdExtZhinxmin())
25486 return std::make_pair(x: 0U, y: &RISCV::GPRF16CRegClass);
25487 if (VT == MVT::f32 && Subtarget.hasStdExtZfinx())
25488 return std::make_pair(x: 0U, y: &RISCV::GPRF32CRegClass);
25489 if (VT == MVT::f64 && Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
25490 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
25491 if (!VT.isVector())
25492 return std::make_pair(x: 0U, y: &RISCV::GPRCRegClass);
25493 } else if (Constraint == "cR") {
25494 if (((VT == MVT::i64 || VT == MVT::f64) && !Subtarget.is64Bit()) ||
25495 (VT == MVT::i128 && Subtarget.is64Bit()))
25496 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
25497 } else if (Constraint == "cf") {
25498 if (VT == MVT::f16) {
25499 if (Subtarget.hasStdExtZfhmin())
25500 return std::make_pair(x: 0U, y: &RISCV::FPR16CRegClass);
25501 if (Subtarget.hasStdExtZhinxmin())
25502 return std::make_pair(x: 0U, y: &RISCV::GPRF16CRegClass);
25503 } else if (VT == MVT::bf16 && Subtarget.hasStdExtZfbfmin()) {
25504 return std::make_pair(x: 0U, y: &RISCV::FPR16CRegClass);
25505 } else if (VT == MVT::f32) {
25506 if (Subtarget.hasStdExtF())
25507 return std::make_pair(x: 0U, y: &RISCV::FPR32CRegClass);
25508 if (Subtarget.hasStdExtZfinx())
25509 return std::make_pair(x: 0U, y: &RISCV::GPRF32CRegClass);
25510 } else if (VT == MVT::f64) {
25511 if (Subtarget.hasStdExtD())
25512 return std::make_pair(x: 0U, y: &RISCV::FPR64CRegClass);
25513 if (Subtarget.hasStdExtZdinx() && !Subtarget.is64Bit())
25514 return std::make_pair(x: 0U, y: &RISCV::GPRPairCRegClass);
25515 if (Subtarget.hasStdExtZdinx() && Subtarget.is64Bit())
25516 return std::make_pair(x: 0U, y: &RISCV::GPRCRegClass);
25517 }
25518 }
25519
25520 // Clang will correctly decode the usage of register name aliases into their
25521 // official names. However, other frontends like `rustc` do not. This allows
25522 // users of these frontends to use the ABI names for registers in LLVM-style
25523 // register constraints.
25524 unsigned XRegFromAlias = StringSwitch<unsigned>(Constraint.lower())
25525 .Case(S: "{zero}", Value: RISCV::X0)
25526 .Case(S: "{ra}", Value: RISCV::X1)
25527 .Case(S: "{sp}", Value: RISCV::X2)
25528 .Case(S: "{gp}", Value: RISCV::X3)
25529 .Case(S: "{tp}", Value: RISCV::X4)
25530 .Case(S: "{t0}", Value: RISCV::X5)
25531 .Case(S: "{t1}", Value: RISCV::X6)
25532 .Case(S: "{t2}", Value: RISCV::X7)
25533 .Cases(CaseStrings: {"{s0}", "{fp}"}, Value: RISCV::X8)
25534 .Case(S: "{s1}", Value: RISCV::X9)
25535 .Case(S: "{a0}", Value: RISCV::X10)
25536 .Case(S: "{a1}", Value: RISCV::X11)
25537 .Case(S: "{a2}", Value: RISCV::X12)
25538 .Case(S: "{a3}", Value: RISCV::X13)
25539 .Case(S: "{a4}", Value: RISCV::X14)
25540 .Case(S: "{a5}", Value: RISCV::X15)
25541 .Case(S: "{a6}", Value: RISCV::X16)
25542 .Case(S: "{a7}", Value: RISCV::X17)
25543 .Case(S: "{s2}", Value: RISCV::X18)
25544 .Case(S: "{s3}", Value: RISCV::X19)
25545 .Case(S: "{s4}", Value: RISCV::X20)
25546 .Case(S: "{s5}", Value: RISCV::X21)
25547 .Case(S: "{s6}", Value: RISCV::X22)
25548 .Case(S: "{s7}", Value: RISCV::X23)
25549 .Case(S: "{s8}", Value: RISCV::X24)
25550 .Case(S: "{s9}", Value: RISCV::X25)
25551 .Case(S: "{s10}", Value: RISCV::X26)
25552 .Case(S: "{s11}", Value: RISCV::X27)
25553 .Case(S: "{t3}", Value: RISCV::X28)
25554 .Case(S: "{t4}", Value: RISCV::X29)
25555 .Case(S: "{t5}", Value: RISCV::X30)
25556 .Case(S: "{t6}", Value: RISCV::X31)
25557 .Default(Value: RISCV::NoRegister);
25558 if (XRegFromAlias != RISCV::NoRegister)
25559 return std::make_pair(x&: XRegFromAlias, y: &RISCV::GPRRegClass);
25560
25561 // Since TargetLowering::getRegForInlineAsmConstraint uses the name of the
25562 // TableGen record rather than the AsmName to choose registers for InlineAsm
25563 // constraints, plus we want to match those names to the widest floating point
25564 // register type available, manually select floating point registers here.
25565 //
25566 // The second case is the ABI name of the register, so that frontends can also
25567 // use the ABI names in register constraint lists.
25568 if (Subtarget.hasStdExtF()) {
25569 unsigned FReg = StringSwitch<unsigned>(Constraint.lower())
25570 .Cases(CaseStrings: {"{f0}", "{ft0}"}, Value: RISCV::F0_F)
25571 .Cases(CaseStrings: {"{f1}", "{ft1}"}, Value: RISCV::F1_F)
25572 .Cases(CaseStrings: {"{f2}", "{ft2}"}, Value: RISCV::F2_F)
25573 .Cases(CaseStrings: {"{f3}", "{ft3}"}, Value: RISCV::F3_F)
25574 .Cases(CaseStrings: {"{f4}", "{ft4}"}, Value: RISCV::F4_F)
25575 .Cases(CaseStrings: {"{f5}", "{ft5}"}, Value: RISCV::F5_F)
25576 .Cases(CaseStrings: {"{f6}", "{ft6}"}, Value: RISCV::F6_F)
25577 .Cases(CaseStrings: {"{f7}", "{ft7}"}, Value: RISCV::F7_F)
25578 .Cases(CaseStrings: {"{f8}", "{fs0}"}, Value: RISCV::F8_F)
25579 .Cases(CaseStrings: {"{f9}", "{fs1}"}, Value: RISCV::F9_F)
25580 .Cases(CaseStrings: {"{f10}", "{fa0}"}, Value: RISCV::F10_F)
25581 .Cases(CaseStrings: {"{f11}", "{fa1}"}, Value: RISCV::F11_F)
25582 .Cases(CaseStrings: {"{f12}", "{fa2}"}, Value: RISCV::F12_F)
25583 .Cases(CaseStrings: {"{f13}", "{fa3}"}, Value: RISCV::F13_F)
25584 .Cases(CaseStrings: {"{f14}", "{fa4}"}, Value: RISCV::F14_F)
25585 .Cases(CaseStrings: {"{f15}", "{fa5}"}, Value: RISCV::F15_F)
25586 .Cases(CaseStrings: {"{f16}", "{fa6}"}, Value: RISCV::F16_F)
25587 .Cases(CaseStrings: {"{f17}", "{fa7}"}, Value: RISCV::F17_F)
25588 .Cases(CaseStrings: {"{f18}", "{fs2}"}, Value: RISCV::F18_F)
25589 .Cases(CaseStrings: {"{f19}", "{fs3}"}, Value: RISCV::F19_F)
25590 .Cases(CaseStrings: {"{f20}", "{fs4}"}, Value: RISCV::F20_F)
25591 .Cases(CaseStrings: {"{f21}", "{fs5}"}, Value: RISCV::F21_F)
25592 .Cases(CaseStrings: {"{f22}", "{fs6}"}, Value: RISCV::F22_F)
25593 .Cases(CaseStrings: {"{f23}", "{fs7}"}, Value: RISCV::F23_F)
25594 .Cases(CaseStrings: {"{f24}", "{fs8}"}, Value: RISCV::F24_F)
25595 .Cases(CaseStrings: {"{f25}", "{fs9}"}, Value: RISCV::F25_F)
25596 .Cases(CaseStrings: {"{f26}", "{fs10}"}, Value: RISCV::F26_F)
25597 .Cases(CaseStrings: {"{f27}", "{fs11}"}, Value: RISCV::F27_F)
25598 .Cases(CaseStrings: {"{f28}", "{ft8}"}, Value: RISCV::F28_F)
25599 .Cases(CaseStrings: {"{f29}", "{ft9}"}, Value: RISCV::F29_F)
25600 .Cases(CaseStrings: {"{f30}", "{ft10}"}, Value: RISCV::F30_F)
25601 .Cases(CaseStrings: {"{f31}", "{ft11}"}, Value: RISCV::F31_F)
25602 .Default(Value: RISCV::NoRegister);
25603 if (FReg != RISCV::NoRegister) {
25604 assert(RISCV::F0_F <= FReg && FReg <= RISCV::F31_F && "Unknown fp-reg");
25605 if (Subtarget.hasStdExtD() && (VT == MVT::f64 || VT == MVT::Other)) {
25606 unsigned RegNo = FReg - RISCV::F0_F;
25607 unsigned DReg = RISCV::F0_D + RegNo;
25608 return std::make_pair(x&: DReg, y: &RISCV::FPR64RegClass);
25609 }
25610 if (VT == MVT::f32 || VT == MVT::Other)
25611 return std::make_pair(x&: FReg, y: &RISCV::FPR32RegClass);
25612 if (Subtarget.hasStdExtZfhmin() && VT == MVT::f16) {
25613 unsigned RegNo = FReg - RISCV::F0_F;
25614 unsigned HReg = RISCV::F0_H + RegNo;
25615 return std::make_pair(x&: HReg, y: &RISCV::FPR16RegClass);
25616 }
25617 }
25618 }
25619
25620 if (Subtarget.hasVInstructions()) {
25621 Register VReg = StringSwitch<Register>(Constraint.lower())
25622 .Case(S: "{v0}", Value: RISCV::V0)
25623 .Case(S: "{v1}", Value: RISCV::V1)
25624 .Case(S: "{v2}", Value: RISCV::V2)
25625 .Case(S: "{v3}", Value: RISCV::V3)
25626 .Case(S: "{v4}", Value: RISCV::V4)
25627 .Case(S: "{v5}", Value: RISCV::V5)
25628 .Case(S: "{v6}", Value: RISCV::V6)
25629 .Case(S: "{v7}", Value: RISCV::V7)
25630 .Case(S: "{v8}", Value: RISCV::V8)
25631 .Case(S: "{v9}", Value: RISCV::V9)
25632 .Case(S: "{v10}", Value: RISCV::V10)
25633 .Case(S: "{v11}", Value: RISCV::V11)
25634 .Case(S: "{v12}", Value: RISCV::V12)
25635 .Case(S: "{v13}", Value: RISCV::V13)
25636 .Case(S: "{v14}", Value: RISCV::V14)
25637 .Case(S: "{v15}", Value: RISCV::V15)
25638 .Case(S: "{v16}", Value: RISCV::V16)
25639 .Case(S: "{v17}", Value: RISCV::V17)
25640 .Case(S: "{v18}", Value: RISCV::V18)
25641 .Case(S: "{v19}", Value: RISCV::V19)
25642 .Case(S: "{v20}", Value: RISCV::V20)
25643 .Case(S: "{v21}", Value: RISCV::V21)
25644 .Case(S: "{v22}", Value: RISCV::V22)
25645 .Case(S: "{v23}", Value: RISCV::V23)
25646 .Case(S: "{v24}", Value: RISCV::V24)
25647 .Case(S: "{v25}", Value: RISCV::V25)
25648 .Case(S: "{v26}", Value: RISCV::V26)
25649 .Case(S: "{v27}", Value: RISCV::V27)
25650 .Case(S: "{v28}", Value: RISCV::V28)
25651 .Case(S: "{v29}", Value: RISCV::V29)
25652 .Case(S: "{v30}", Value: RISCV::V30)
25653 .Case(S: "{v31}", Value: RISCV::V31)
25654 .Default(Value: RISCV::NoRegister);
25655 if (VReg != RISCV::NoRegister) {
25656 if (TRI->isTypeLegalForClass(RC: RISCV::ZZZ_VMRegClass, T: VT.SimpleTy))
25657 return std::make_pair(x&: VReg, y: &RISCV::ZZZ_VMRegClass);
25658 if (TRI->isTypeLegalForClass(RC: RISCV::VRRegClass, T: VT.SimpleTy))
25659 return std::make_pair(x&: VReg, y: &RISCV::VRRegClass);
25660 for (const auto *RC :
25661 {&RISCV::VRM2RegClass, &RISCV::VRM4RegClass, &RISCV::VRM8RegClass}) {
25662 if (TRI->isTypeLegalForClass(RC: *RC, T: VT.SimpleTy)) {
25663 VReg = TRI->getMatchingSuperReg(Reg: VReg, SubIdx: RISCV::sub_vrm1_0, RC);
25664 return std::make_pair(x&: VReg, y&: RC);
25665 }
25666 }
25667 }
25668 }
25669
25670 return TargetLowering::getRegForInlineAsmConstraint(TRI, Constraint, VT);
25671}
25672
25673InlineAsm::ConstraintCode
25674RISCVTargetLowering::getInlineAsmMemConstraint(StringRef ConstraintCode) const {
25675 // Currently only support length 1 constraints.
25676 if (ConstraintCode.size() == 1) {
25677 switch (ConstraintCode[0]) {
25678 case 'A':
25679 return InlineAsm::ConstraintCode::A;
25680 default:
25681 break;
25682 }
25683 }
25684
25685 return TargetLowering::getInlineAsmMemConstraint(ConstraintCode);
25686}
25687
25688void RISCVTargetLowering::LowerAsmOperandForConstraint(
25689 SDValue Op, StringRef Constraint, std::vector<SDValue> &Ops,
25690 SelectionDAG &DAG) const {
25691 // Currently only support length 1 constraints.
25692 if (Constraint.size() == 1) {
25693 switch (Constraint[0]) {
25694 case 'I':
25695 // Validate & create a 12-bit signed immediate operand.
25696 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
25697 uint64_t CVal = C->getSExtValue();
25698 if (isInt<12>(x: CVal))
25699 Ops.push_back(x: DAG.getSignedTargetConstant(Val: CVal, DL: SDLoc(Op),
25700 VT: Subtarget.getXLenVT()));
25701 }
25702 return;
25703 case 'J':
25704 // Validate & create an integer zero operand.
25705 if (isNullConstant(V: Op))
25706 Ops.push_back(
25707 x: DAG.getTargetConstant(Val: 0, DL: SDLoc(Op), VT: Subtarget.getXLenVT()));
25708 return;
25709 case 'K':
25710 // Validate & create a 5-bit unsigned immediate operand.
25711 if (auto *C = dyn_cast<ConstantSDNode>(Val&: Op)) {
25712 uint64_t CVal = C->getZExtValue();
25713 if (isUInt<5>(x: CVal))
25714 Ops.push_back(
25715 x: DAG.getTargetConstant(Val: CVal, DL: SDLoc(Op), VT: Subtarget.getXLenVT()));
25716 }
25717 return;
25718 case 'S':
25719 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint: "s", Ops, DAG);
25720 return;
25721 default:
25722 break;
25723 }
25724 }
25725 TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG);
25726}
25727
25728Instruction *RISCVTargetLowering::emitLeadingFence(IRBuilderBase &Builder,
25729 Instruction *Inst,
25730 AtomicOrdering Ord) const {
25731 if (Subtarget.hasStdExtZtso()) {
25732 if (isa<LoadInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
25733 return Builder.CreateFence(Ordering: Ord);
25734 return nullptr;
25735 }
25736
25737 if (isa<LoadInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
25738 return Builder.CreateFence(Ordering: Ord);
25739 if (isa<StoreInst>(Val: Inst) && isReleaseOrStronger(AO: Ord))
25740 return Builder.CreateFence(Ordering: AtomicOrdering::Release);
25741 return nullptr;
25742}
25743
25744Instruction *RISCVTargetLowering::emitTrailingFence(IRBuilderBase &Builder,
25745 Instruction *Inst,
25746 AtomicOrdering Ord) const {
25747 if (Subtarget.hasStdExtZtso()) {
25748 if (isa<StoreInst>(Val: Inst) && Ord == AtomicOrdering::SequentiallyConsistent)
25749 return Builder.CreateFence(Ordering: Ord);
25750 return nullptr;
25751 }
25752
25753 if (isa<LoadInst>(Val: Inst) && isAcquireOrStronger(AO: Ord))
25754 return Builder.CreateFence(Ordering: AtomicOrdering::Acquire);
25755 if (Subtarget.enableTrailingSeqCstFence() && isa<StoreInst>(Val: Inst) &&
25756 Ord == AtomicOrdering::SequentiallyConsistent)
25757 return Builder.CreateFence(Ordering: AtomicOrdering::SequentiallyConsistent);
25758 return nullptr;
25759}
25760
25761TargetLowering::AtomicExpansionKind
25762RISCVTargetLowering::shouldExpandAtomicRMWInIR(const AtomicRMWInst *AI) const {
25763 // atomicrmw {fadd,fsub} must be expanded to use compare-exchange, as floating
25764 // point operations can't be used in an lr/sc sequence without breaking the
25765 // forward-progress guarantee.
25766 if (AI->isFloatingPointOperation() ||
25767 AI->getOperation() == AtomicRMWInst::UIncWrap ||
25768 AI->getOperation() == AtomicRMWInst::UDecWrap ||
25769 AI->getOperation() == AtomicRMWInst::USubCond ||
25770 AI->getOperation() == AtomicRMWInst::USubSat)
25771 return AtomicExpansionKind::CmpXChg;
25772
25773 // Don't expand forced atomics, we want to have __sync libcalls instead.
25774 if (Subtarget.hasForcedAtomics())
25775 return AtomicExpansionKind::None;
25776
25777 unsigned Size = AI->getType()->getPrimitiveSizeInBits();
25778 if (AI->getOperation() == AtomicRMWInst::Nand) {
25779 if (Subtarget.hasStdExtZacas() &&
25780 (Size >= 32 || Subtarget.hasStdExtZabha()))
25781 return AtomicExpansionKind::CmpXChg;
25782 if (Size < 32)
25783 return AtomicExpansionKind::MaskedIntrinsic;
25784 }
25785
25786 if (Size < 32 && !Subtarget.hasStdExtZabha())
25787 return AtomicExpansionKind::MaskedIntrinsic;
25788
25789 return AtomicExpansionKind::None;
25790}
25791
25792static Intrinsic::ID
25793getIntrinsicForMaskedAtomicRMWBinOp(unsigned XLen, AtomicRMWInst::BinOp BinOp) {
25794 switch (BinOp) {
25795 default:
25796 llvm_unreachable("Unexpected AtomicRMW BinOp");
25797 case AtomicRMWInst::Xchg:
25798 return Intrinsic::riscv_masked_atomicrmw_xchg;
25799 case AtomicRMWInst::Add:
25800 return Intrinsic::riscv_masked_atomicrmw_add;
25801 case AtomicRMWInst::Sub:
25802 return Intrinsic::riscv_masked_atomicrmw_sub;
25803 case AtomicRMWInst::Nand:
25804 return Intrinsic::riscv_masked_atomicrmw_nand;
25805 case AtomicRMWInst::Max:
25806 return Intrinsic::riscv_masked_atomicrmw_max;
25807 case AtomicRMWInst::Min:
25808 return Intrinsic::riscv_masked_atomicrmw_min;
25809 case AtomicRMWInst::UMax:
25810 return Intrinsic::riscv_masked_atomicrmw_umax;
25811 case AtomicRMWInst::UMin:
25812 return Intrinsic::riscv_masked_atomicrmw_umin;
25813 }
25814}
25815
25816Value *RISCVTargetLowering::emitMaskedAtomicRMWIntrinsic(
25817 IRBuilderBase &Builder, AtomicRMWInst *AI, Value *AlignedAddr, Value *Incr,
25818 Value *Mask, Value *ShiftAmt, AtomicOrdering Ord) const {
25819 // In the case of an atomicrmw xchg with a constant 0/-1 operand, replace
25820 // the atomic instruction with an AtomicRMWInst::And/Or with appropriate
25821 // mask, as this produces better code than the LR/SC loop emitted by
25822 // int_riscv_masked_atomicrmw_xchg.
25823 if (AI->getOperation() == AtomicRMWInst::Xchg &&
25824 isa<ConstantInt>(Val: AI->getValOperand())) {
25825 ConstantInt *CVal = cast<ConstantInt>(Val: AI->getValOperand());
25826 if (CVal->isZero())
25827 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::And, Ptr: AlignedAddr,
25828 Val: Builder.CreateNot(V: Mask, Name: "Inv_Mask"),
25829 Align: AI->getAlign(), Ordering: Ord);
25830 if (CVal->isMinusOne())
25831 return Builder.CreateAtomicRMW(Op: AtomicRMWInst::Or, Ptr: AlignedAddr, Val: Mask,
25832 Align: AI->getAlign(), Ordering: Ord);
25833 }
25834
25835 unsigned XLen = Subtarget.getXLen();
25836 Value *Ordering =
25837 Builder.getIntN(N: XLen, C: static_cast<uint64_t>(AI->getOrdering()));
25838 Type *Tys[] = {Builder.getIntNTy(N: XLen), AlignedAddr->getType()};
25839 Function *LrwOpScwLoop = Intrinsic::getOrInsertDeclaration(
25840 M: AI->getModule(),
25841 id: getIntrinsicForMaskedAtomicRMWBinOp(XLen, BinOp: AI->getOperation()), Tys);
25842
25843 if (XLen == 64) {
25844 Incr = Builder.CreateSExt(V: Incr, DestTy: Builder.getInt64Ty());
25845 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
25846 ShiftAmt = Builder.CreateSExt(V: ShiftAmt, DestTy: Builder.getInt64Ty());
25847 }
25848
25849 Value *Result;
25850
25851 // Must pass the shift amount needed to sign extend the loaded value prior
25852 // to performing a signed comparison for min/max. ShiftAmt is the number of
25853 // bits to shift the value into position. Pass XLen-ShiftAmt-ValWidth, which
25854 // is the number of bits to left+right shift the value in order to
25855 // sign-extend.
25856 if (AI->getOperation() == AtomicRMWInst::Min ||
25857 AI->getOperation() == AtomicRMWInst::Max) {
25858 const DataLayout &DL = AI->getDataLayout();
25859 unsigned ValWidth =
25860 DL.getTypeStoreSizeInBits(Ty: AI->getValOperand()->getType());
25861 Value *SextShamt =
25862 Builder.CreateSub(LHS: Builder.getIntN(N: XLen, C: XLen - ValWidth), RHS: ShiftAmt);
25863 Result = Builder.CreateCall(Callee: LrwOpScwLoop,
25864 Args: {AlignedAddr, Incr, Mask, SextShamt, Ordering});
25865 } else {
25866 Result =
25867 Builder.CreateCall(Callee: LrwOpScwLoop, Args: {AlignedAddr, Incr, Mask, Ordering});
25868 }
25869
25870 if (XLen == 64)
25871 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
25872 return Result;
25873}
25874
25875TargetLowering::AtomicExpansionKind
25876RISCVTargetLowering::shouldExpandAtomicCmpXchgInIR(
25877 const AtomicCmpXchgInst *CI) const {
25878 // Don't expand forced atomics, we want to have __sync libcalls instead.
25879 if (Subtarget.hasForcedAtomics())
25880 return AtomicExpansionKind::None;
25881
25882 unsigned Size = CI->getCompareOperand()->getType()->getPrimitiveSizeInBits();
25883 if (!(Subtarget.hasStdExtZabha() && Subtarget.hasStdExtZacas()) &&
25884 (Size == 8 || Size == 16))
25885 return AtomicExpansionKind::MaskedIntrinsic;
25886 return AtomicExpansionKind::None;
25887}
25888
25889Value *RISCVTargetLowering::emitMaskedAtomicCmpXchgIntrinsic(
25890 IRBuilderBase &Builder, AtomicCmpXchgInst *CI, Value *AlignedAddr,
25891 Value *CmpVal, Value *NewVal, Value *Mask, AtomicOrdering Ord) const {
25892 unsigned XLen = Subtarget.getXLen();
25893 Value *Ordering = Builder.getIntN(N: XLen, C: static_cast<uint64_t>(Ord));
25894 Intrinsic::ID CmpXchgIntrID = Intrinsic::riscv_masked_cmpxchg;
25895 if (XLen == 64) {
25896 CmpVal = Builder.CreateSExt(V: CmpVal, DestTy: Builder.getInt64Ty());
25897 NewVal = Builder.CreateSExt(V: NewVal, DestTy: Builder.getInt64Ty());
25898 Mask = Builder.CreateSExt(V: Mask, DestTy: Builder.getInt64Ty());
25899 }
25900 Type *Tys[] = {Builder.getIntNTy(N: XLen), AlignedAddr->getType()};
25901 Value *Result = Builder.CreateIntrinsic(
25902 ID: CmpXchgIntrID, Types: Tys, Args: {AlignedAddr, CmpVal, NewVal, Mask, Ordering});
25903 if (XLen == 64)
25904 Result = Builder.CreateTrunc(V: Result, DestTy: Builder.getInt32Ty());
25905 return Result;
25906}
25907
25908bool RISCVTargetLowering::shouldRemoveExtendFromGSIndex(SDValue Extend,
25909 EVT DataVT) const {
25910 // We have indexed loads for all supported EEW types. Indices are always
25911 // zero extended.
25912 return Extend.getOpcode() == ISD::ZERO_EXTEND &&
25913 isTypeLegal(VT: Extend.getValueType()) &&
25914 isTypeLegal(VT: Extend.getOperand(i: 0).getValueType()) &&
25915 Extend.getOperand(i: 0).getValueType().getVectorElementType() != MVT::i1;
25916}
25917
25918bool RISCVTargetLowering::shouldConvertFpToSat(unsigned Op, EVT FPVT,
25919 EVT VT) const {
25920 if (!isOperationLegalOrCustom(Op, VT) || !FPVT.isSimple())
25921 return false;
25922
25923 switch (FPVT.getSimpleVT().SimpleTy) {
25924 case MVT::f16:
25925 return Subtarget.hasStdExtZfhmin();
25926 case MVT::f32:
25927 return Subtarget.hasStdExtF();
25928 case MVT::f64:
25929 return Subtarget.hasStdExtD();
25930 default:
25931 return false;
25932 }
25933}
25934
25935unsigned RISCVTargetLowering::getJumpTableEncoding() const {
25936 // If we are using the small code model, we can reduce size of jump table
25937 // entry to 4 bytes.
25938 if (Subtarget.is64Bit() && !isPositionIndependent() &&
25939 getTargetMachine().getCodeModel() == CodeModel::Small) {
25940 return MachineJumpTableInfo::EK_Custom32;
25941 }
25942 return TargetLowering::getJumpTableEncoding();
25943}
25944
25945const MCExpr *RISCVTargetLowering::LowerCustomJumpTableEntry(
25946 const MachineJumpTableInfo *MJTI, const MachineBasicBlock *MBB,
25947 unsigned uid, MCContext &Ctx) const {
25948 assert(Subtarget.is64Bit() && !isPositionIndependent() &&
25949 getTargetMachine().getCodeModel() == CodeModel::Small);
25950 return MCSymbolRefExpr::create(Symbol: MBB->getSymbol(), Ctx);
25951}
25952
25953bool RISCVTargetLowering::getIndexedAddressParts(SDNode *Op, SDValue &Base,
25954 SDValue &Offset,
25955 ISD::MemIndexedMode &AM,
25956 SelectionDAG &DAG) const {
25957 // Target does not support indexed loads.
25958 if (!Subtarget.hasVendorXTHeadMemIdx())
25959 return false;
25960
25961 if (Op->getOpcode() != ISD::ADD && Op->getOpcode() != ISD::SUB)
25962 return false;
25963
25964 Base = Op->getOperand(Num: 0);
25965 if (ConstantSDNode *RHS = dyn_cast<ConstantSDNode>(Val: Op->getOperand(Num: 1))) {
25966 int64_t RHSC = RHS->getSExtValue();
25967 if (Op->getOpcode() == ISD::SUB)
25968 RHSC = -(uint64_t)RHSC;
25969
25970 // The constants that can be encoded in the THeadMemIdx instructions
25971 // are of the form (sign_extend(imm5) << imm2).
25972 bool isLegalIndexedOffset = false;
25973 for (unsigned i = 0; i < 4; i++)
25974 if (isInt<5>(x: RHSC >> i) && ((RHSC % (1LL << i)) == 0)) {
25975 isLegalIndexedOffset = true;
25976 break;
25977 }
25978
25979 if (!isLegalIndexedOffset)
25980 return false;
25981
25982 Offset = Op->getOperand(Num: 1);
25983 return true;
25984 }
25985
25986 return false;
25987}
25988
25989bool RISCVTargetLowering::getPreIndexedAddressParts(SDNode *N, SDValue &Base,
25990 SDValue &Offset,
25991 ISD::MemIndexedMode &AM,
25992 SelectionDAG &DAG) const {
25993 EVT VT;
25994 SDValue Ptr;
25995 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
25996 VT = LD->getMemoryVT();
25997 Ptr = LD->getBasePtr();
25998 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
25999 VT = ST->getMemoryVT();
26000 Ptr = ST->getBasePtr();
26001 } else
26002 return false;
26003
26004 if (!getIndexedAddressParts(Op: Ptr.getNode(), Base, Offset, AM, DAG))
26005 return false;
26006
26007 AM = ISD::PRE_INC;
26008 return true;
26009}
26010
26011bool RISCVTargetLowering::getPostIndexedAddressParts(SDNode *N, SDNode *Op,
26012 SDValue &Base,
26013 SDValue &Offset,
26014 ISD::MemIndexedMode &AM,
26015 SelectionDAG &DAG) const {
26016 if (Subtarget.hasVendorXCVmem() && !Subtarget.is64Bit()) {
26017 if (Op->getOpcode() != ISD::ADD)
26018 return false;
26019
26020 if (LSBaseSDNode *LS = dyn_cast<LSBaseSDNode>(Val: N))
26021 Base = LS->getBasePtr();
26022 else
26023 return false;
26024
26025 if (Base == Op->getOperand(Num: 0))
26026 Offset = Op->getOperand(Num: 1);
26027 else if (Base == Op->getOperand(Num: 1))
26028 Offset = Op->getOperand(Num: 0);
26029 else
26030 return false;
26031
26032 AM = ISD::POST_INC;
26033 return true;
26034 }
26035
26036 EVT VT;
26037 SDValue Ptr;
26038 if (LoadSDNode *LD = dyn_cast<LoadSDNode>(Val: N)) {
26039 VT = LD->getMemoryVT();
26040 Ptr = LD->getBasePtr();
26041 } else if (StoreSDNode *ST = dyn_cast<StoreSDNode>(Val: N)) {
26042 VT = ST->getMemoryVT();
26043 Ptr = ST->getBasePtr();
26044 } else
26045 return false;
26046
26047 if (!getIndexedAddressParts(Op, Base, Offset, AM, DAG))
26048 return false;
26049 // Post-indexing updates the base, so it's not a valid transform
26050 // if that's not the same as the load's pointer.
26051 if (Ptr != Base)
26052 return false;
26053
26054 AM = ISD::POST_INC;
26055 return true;
26056}
26057
26058bool RISCVTargetLowering::isFMAFasterThanFMulAndFAdd(const MachineFunction &MF,
26059 EVT VT) const {
26060 EVT SVT = VT.getScalarType();
26061
26062 if (!SVT.isSimple())
26063 return false;
26064
26065 switch (SVT.getSimpleVT().SimpleTy) {
26066 case MVT::f16:
26067 return VT.isVector() ? Subtarget.hasVInstructionsF16()
26068 : Subtarget.hasStdExtZfhOrZhinx();
26069 case MVT::f32:
26070 return Subtarget.hasStdExtFOrZfinx();
26071 case MVT::f64:
26072 return Subtarget.hasStdExtDOrZdinx();
26073 default:
26074 break;
26075 }
26076
26077 return false;
26078}
26079
26080ISD::NodeType RISCVTargetLowering::getExtendForAtomicCmpSwapArg() const {
26081 // Zacas will use amocas.w which does not require extension.
26082 return Subtarget.hasStdExtZacas() ? ISD::ANY_EXTEND : ISD::SIGN_EXTEND;
26083}
26084
26085ISD::NodeType RISCVTargetLowering::getExtendForAtomicRMWArg(unsigned Op) const {
26086 // Zaamo will use amo<op>.w which does not require extension.
26087 if (Subtarget.hasStdExtZaamo() || Subtarget.hasForcedAtomics())
26088 return ISD::ANY_EXTEND;
26089
26090 // Zalrsc pseudo expansions with comparison require sign-extension.
26091 assert(Subtarget.hasStdExtZalrsc());
26092 switch (Op) {
26093 case ISD::ATOMIC_LOAD_MIN:
26094 case ISD::ATOMIC_LOAD_MAX:
26095 case ISD::ATOMIC_LOAD_UMIN:
26096 case ISD::ATOMIC_LOAD_UMAX:
26097 return ISD::SIGN_EXTEND;
26098 default:
26099 break;
26100 }
26101 return ISD::ANY_EXTEND;
26102}
26103
26104Register RISCVTargetLowering::getExceptionPointerRegister(
26105 const Constant *PersonalityFn) const {
26106 return RISCV::X10;
26107}
26108
26109Register RISCVTargetLowering::getExceptionSelectorRegister(
26110 const Constant *PersonalityFn) const {
26111 return RISCV::X11;
26112}
26113
26114bool RISCVTargetLowering::shouldExtendTypeInLibCall(EVT Type) const {
26115 // Return false to suppress the unnecessary extensions if the LibCall
26116 // arguments or return value is a float narrower than XLEN on a soft FP ABI.
26117 if (Subtarget.isSoftFPABI() && (Type.isFloatingPoint() && !Type.isVector() &&
26118 Type.getSizeInBits() < Subtarget.getXLen()))
26119 return false;
26120
26121 return true;
26122}
26123
26124bool RISCVTargetLowering::shouldSignExtendTypeInLibCall(Type *Ty,
26125 bool IsSigned) const {
26126 if (Subtarget.is64Bit() && Ty->isIntegerTy(Bitwidth: 32))
26127 return true;
26128
26129 return IsSigned;
26130}
26131
26132bool RISCVTargetLowering::decomposeMulByConstant(LLVMContext &Context, EVT VT,
26133 SDValue C) const {
26134 // Check integral scalar types.
26135 if (!VT.isScalarInteger())
26136 return false;
26137
26138 // Omit the optimization if the sub target has the M extension and the data
26139 // size exceeds XLen.
26140 const bool HasZmmul = Subtarget.hasStdExtZmmul();
26141 if (HasZmmul && VT.getSizeInBits() > Subtarget.getXLen())
26142 return false;
26143
26144 auto *ConstNode = cast<ConstantSDNode>(Val&: C);
26145 const APInt &Imm = ConstNode->getAPIntValue();
26146
26147 // Don't do this if the Xqciac extension is enabled and the Imm in simm12.
26148 if (Subtarget.hasVendorXqciac() && Imm.isSignedIntN(N: 12))
26149 return false;
26150
26151 // Break the MUL to a SLLI and an ADD/SUB.
26152 if ((Imm + 1).isPowerOf2() || (Imm - 1).isPowerOf2() ||
26153 (1 - Imm).isPowerOf2() || (-1 - Imm).isPowerOf2())
26154 return true;
26155
26156 // Optimize the MUL to (SH*ADD x, (SLLI x, bits)) if Imm is not simm12.
26157 if (Subtarget.hasShlAdd(ShAmt: 3) && !Imm.isSignedIntN(N: 12) &&
26158 ((Imm - 2).isPowerOf2() || (Imm - 4).isPowerOf2() ||
26159 (Imm - 8).isPowerOf2()))
26160 return true;
26161
26162 // Break the MUL to two SLLI instructions and an ADD/SUB, if Imm needs
26163 // a pair of LUI/ADDI.
26164 if (!Imm.isSignedIntN(N: 12) && Imm.countr_zero() < 12 &&
26165 ConstNode->hasOneUse()) {
26166 APInt ImmS = Imm.ashr(ShiftAmt: Imm.countr_zero());
26167 if ((ImmS + 1).isPowerOf2() || (ImmS - 1).isPowerOf2() ||
26168 (1 - ImmS).isPowerOf2())
26169 return true;
26170 }
26171
26172 return false;
26173}
26174
26175bool RISCVTargetLowering::isMulAddWithConstProfitable(SDValue AddNode,
26176 SDValue ConstNode) const {
26177 // Let the DAGCombiner decide for vectors.
26178 EVT VT = AddNode.getValueType();
26179 if (VT.isVector())
26180 return true;
26181
26182 // Let the DAGCombiner decide for larger types.
26183 if (VT.getScalarSizeInBits() > Subtarget.getXLen())
26184 return true;
26185
26186 // It is worse if c1 is simm12 while c1*c2 is not.
26187 ConstantSDNode *C1Node = cast<ConstantSDNode>(Val: AddNode.getOperand(i: 1));
26188 ConstantSDNode *C2Node = cast<ConstantSDNode>(Val&: ConstNode);
26189 const APInt &C1 = C1Node->getAPIntValue();
26190 const APInt &C2 = C2Node->getAPIntValue();
26191 if (C1.isSignedIntN(N: 12) && !(C1 * C2).isSignedIntN(N: 12))
26192 return false;
26193
26194 // Default to true and let the DAGCombiner decide.
26195 return true;
26196}
26197
26198bool RISCVTargetLowering::allowsMisalignedMemoryAccesses(
26199 EVT VT, unsigned AddrSpace, Align Alignment, MachineMemOperand::Flags Flags,
26200 unsigned *Fast) const {
26201 if (!VT.isVector() || Subtarget.hasStdExtP()) {
26202 if (Fast)
26203 *Fast = Subtarget.enableUnalignedScalarMem();
26204 return Subtarget.enableUnalignedScalarMem();
26205 }
26206
26207 // All vector implementations must support element alignment
26208 EVT ElemVT = VT.getVectorElementType();
26209 if (Alignment >= ElemVT.getStoreSize()) {
26210 if (Fast)
26211 *Fast = 1;
26212 return true;
26213 }
26214
26215 // Note: We lower an unmasked unaligned vector access to an equally sized
26216 // e8 element type access. Given this, we effectively support all unmasked
26217 // misaligned accesses. TODO: Work through the codegen implications of
26218 // allowing such accesses to be formed, and considered fast.
26219 if (Fast)
26220 *Fast = Subtarget.enableUnalignedVectorMem();
26221 return Subtarget.enableUnalignedVectorMem();
26222}
26223
26224EVT RISCVTargetLowering::getOptimalMemOpType(
26225 LLVMContext &Context, const MemOp &Op,
26226 const AttributeList &FuncAttributes) const {
26227 if (!Subtarget.hasVInstructions())
26228 return MVT::Other;
26229
26230 if (FuncAttributes.hasFnAttr(Kind: Attribute::NoImplicitFloat))
26231 return MVT::Other;
26232
26233 // We use LMUL1 memory operations here for a non-obvious reason. Our caller
26234 // has an expansion threshold, and we want the number of hardware memory
26235 // operations to correspond roughly to that threshold. LMUL>1 operations
26236 // are typically expanded linearly internally, and thus correspond to more
26237 // than one actual memory operation. Note that store merging and load
26238 // combining will typically form larger LMUL operations from the LMUL1
26239 // operations emitted here, and that's okay because combining isn't
26240 // introducing new memory operations; it's just merging existing ones.
26241 // NOTE: We limit to 1024 bytes to avoid creating an invalid MVT.
26242 const unsigned MinVLenInBytes =
26243 std::min(a: Subtarget.getRealMinVLen() / 8, b: 1024U);
26244
26245 if (Op.size() < MinVLenInBytes)
26246 // TODO: Figure out short memops. For the moment, do the default thing
26247 // which ends up using scalar sequences.
26248 return MVT::Other;
26249
26250 // If the minimum VLEN is less than RISCV::RVVBitsPerBlock we don't support
26251 // fixed vectors.
26252 if (MinVLenInBytes <= RISCV::RVVBytesPerBlock)
26253 return MVT::Other;
26254
26255 // Prefer i8 for non-zero memset as it allows us to avoid materializing
26256 // a large scalar constant and instead use vmv.v.x/i to do the
26257 // broadcast. For everything else, prefer ELenVT to minimize VL and thus
26258 // maximize the chance we can encode the size in the vsetvli.
26259 MVT ELenVT = MVT::getIntegerVT(BitWidth: Subtarget.getELen());
26260 MVT PreferredVT = (Op.isMemset() && !Op.isZeroMemset()) ? MVT::i8 : ELenVT;
26261
26262 // Do we have sufficient alignment for our preferred VT? If not, revert
26263 // to largest size allowed by our alignment criteria.
26264 if (PreferredVT != MVT::i8 && !Subtarget.enableUnalignedVectorMem()) {
26265 Align RequiredAlign(PreferredVT.getStoreSize());
26266 if (Op.isFixedDstAlign())
26267 RequiredAlign = std::min(a: RequiredAlign, b: Op.getDstAlign());
26268 if (Op.isMemcpy())
26269 RequiredAlign = std::min(a: RequiredAlign, b: Op.getSrcAlign());
26270 PreferredVT = MVT::getIntegerVT(BitWidth: RequiredAlign.value() * 8);
26271 }
26272 return MVT::getVectorVT(VT: PreferredVT, NumElements: MinVLenInBytes/PreferredVT.getStoreSize());
26273}
26274
26275bool RISCVTargetLowering::splitValueIntoRegisterParts(
26276 SelectionDAG &DAG, const SDLoc &DL, SDValue Val, SDValue *Parts,
26277 unsigned NumParts, MVT PartVT, std::optional<CallingConv::ID> CC) const {
26278 bool IsABIRegCopy = CC.has_value();
26279 EVT ValueVT = Val.getValueType();
26280
26281 MVT PairVT = Subtarget.is64Bit() ? MVT::i128 : MVT::i64;
26282 if ((ValueVT == PairVT ||
26283 (!Subtarget.is64Bit() && Subtarget.hasStdExtZdinx() &&
26284 ValueVT == MVT::f64)) &&
26285 NumParts == 1 && PartVT == MVT::Untyped) {
26286 // Pairs in Inline Assembly, f64 in Inline assembly on rv32_zdinx
26287 MVT XLenVT = Subtarget.getXLenVT();
26288 if (ValueVT == MVT::f64)
26289 Val = DAG.getBitcast(VT: MVT::i64, V: Val);
26290 auto [Lo, Hi] = DAG.SplitScalar(N: Val, DL, LoVT: XLenVT, HiVT: XLenVT);
26291 // Always creating an MVT::Untyped part, so always use
26292 // RISCVISD::BuildGPRPair.
26293 Parts[0] = DAG.getNode(Opcode: RISCVISD::BuildGPRPair, DL, VT: PartVT, N1: Lo, N2: Hi);
26294 return true;
26295 }
26296
26297 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
26298 PartVT == MVT::f32) {
26299 // Cast the [b]f16 to i16, extend to i32, pad with ones to make a float
26300 // nan, and cast to f32.
26301 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i16, Operand: Val);
26302 Val = DAG.getNode(Opcode: ISD::ANY_EXTEND, DL, VT: MVT::i32, Operand: Val);
26303 Val = DAG.getNode(Opcode: ISD::OR, DL, VT: MVT::i32, N1: Val,
26304 N2: DAG.getConstant(Val: 0xFFFF0000, DL, VT: MVT::i32));
26305 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
26306 Parts[0] = Val;
26307 return true;
26308 }
26309
26310 if (ValueVT.isRISCVVectorTuple() && PartVT.isRISCVVectorTuple()) {
26311#ifndef NDEBUG
26312 unsigned ValNF = ValueVT.getRISCVVectorTupleNumFields();
26313 [[maybe_unused]] unsigned ValLMUL =
26314 divideCeil(ValueVT.getSizeInBits().getKnownMinValue(),
26315 ValNF * RISCV::RVVBitsPerBlock);
26316 unsigned PartNF = PartVT.getRISCVVectorTupleNumFields();
26317 [[maybe_unused]] unsigned PartLMUL =
26318 divideCeil(PartVT.getSizeInBits().getKnownMinValue(),
26319 PartNF * RISCV::RVVBitsPerBlock);
26320 assert(ValNF == PartNF && ValLMUL == PartLMUL &&
26321 "RISC-V vector tuple type only accepts same register class type "
26322 "TUPLE_INSERT");
26323#endif
26324
26325 Val = DAG.getNode(Opcode: RISCVISD::TUPLE_INSERT, DL, VT: PartVT, N1: DAG.getUNDEF(VT: PartVT),
26326 N2: Val, N3: DAG.getTargetConstant(Val: 0, DL, VT: MVT::i32));
26327 Parts[0] = Val;
26328 return true;
26329 }
26330
26331 if (ValueVT.isFixedLengthVector() && PartVT.isScalableVector()) {
26332 ValueVT = getContainerForFixedLengthVector(VT: ValueVT.getSimpleVT());
26333 Val = convertToScalableVector(VT: ValueVT, V: Val, DAG, Subtarget);
26334
26335 LLVMContext &Context = *DAG.getContext();
26336 EVT ValueEltVT = ValueVT.getVectorElementType();
26337 EVT PartEltVT = PartVT.getVectorElementType();
26338 unsigned ValueVTBitSize = ValueVT.getSizeInBits().getKnownMinValue();
26339 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinValue();
26340 if (PartVTBitSize % ValueVTBitSize == 0) {
26341 assert(PartVTBitSize >= ValueVTBitSize);
26342 // If the element types are different, bitcast to the same element type of
26343 // PartVT first.
26344 // Give an example here, we want copy a <vscale x 1 x i8> value to
26345 // <vscale x 4 x i16>.
26346 // We need to convert <vscale x 1 x i8> to <vscale x 8 x i8> by insert
26347 // subvector, then we can bitcast to <vscale x 4 x i16>.
26348 if (ValueEltVT != PartEltVT) {
26349 if (PartVTBitSize > ValueVTBitSize) {
26350 unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
26351 assert(Count != 0 && "The number of element should not be zero.");
26352 EVT SameEltTypeVT =
26353 EVT::getVectorVT(Context, VT: ValueEltVT, NumElements: Count, /*IsScalable=*/true);
26354 Val = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: SameEltTypeVT), SubVec: Val, Idx: 0);
26355 }
26356 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: PartVT, Operand: Val);
26357 } else {
26358 Val = DAG.getInsertSubvector(DL, Vec: DAG.getUNDEF(VT: PartVT), SubVec: Val, Idx: 0);
26359 }
26360 Parts[0] = Val;
26361 return true;
26362 }
26363 }
26364
26365 return false;
26366}
26367
26368SDValue RISCVTargetLowering::joinRegisterPartsIntoValue(
26369 SelectionDAG &DAG, const SDLoc &DL, const SDValue *Parts, unsigned NumParts,
26370 MVT PartVT, EVT ValueVT, std::optional<CallingConv::ID> CC) const {
26371 bool IsABIRegCopy = CC.has_value();
26372
26373 MVT PairVT = Subtarget.is64Bit() ? MVT::i128 : MVT::i64;
26374 if ((ValueVT == PairVT ||
26375 (!Subtarget.is64Bit() && Subtarget.hasStdExtZdinx() &&
26376 ValueVT == MVT::f64)) &&
26377 NumParts == 1 && PartVT == MVT::Untyped) {
26378 // Pairs in Inline Assembly, f64 in Inline assembly on rv32_zdinx
26379 MVT XLenVT = Subtarget.getXLenVT();
26380
26381 SDValue Val = Parts[0];
26382 // Always starting with an MVT::Untyped part, so always use
26383 // RISCVISD::SplitGPRPair
26384 Val = DAG.getNode(Opcode: RISCVISD::SplitGPRPair, DL, VTList: DAG.getVTList(VT1: XLenVT, VT2: XLenVT),
26385 N: Val);
26386 Val = DAG.getNode(Opcode: ISD::BUILD_PAIR, DL, VT: PairVT, N1: Val.getValue(R: 0),
26387 N2: Val.getValue(R: 1));
26388 if (ValueVT == MVT::f64)
26389 Val = DAG.getBitcast(VT: ValueVT, V: Val);
26390 return Val;
26391 }
26392
26393 if (IsABIRegCopy && (ValueVT == MVT::f16 || ValueVT == MVT::bf16) &&
26394 PartVT == MVT::f32) {
26395 SDValue Val = Parts[0];
26396
26397 // Cast the f32 to i32, truncate to i16, and cast back to [b]f16.
26398 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: MVT::i32, Operand: Val);
26399 Val = DAG.getNode(Opcode: ISD::TRUNCATE, DL, VT: MVT::i16, Operand: Val);
26400 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: ValueVT, Operand: Val);
26401 return Val;
26402 }
26403
26404 if (ValueVT.isFixedLengthVector() && PartVT.isScalableVector()) {
26405 LLVMContext &Context = *DAG.getContext();
26406 SDValue Val = Parts[0];
26407 EVT ValueEltVT = ValueVT.getVectorElementType();
26408 EVT PartEltVT = PartVT.getVectorElementType();
26409
26410 unsigned ValueVTBitSize =
26411 getContainerForFixedLengthVector(VT: ValueVT.getSimpleVT())
26412 .getSizeInBits()
26413 .getKnownMinValue();
26414
26415 unsigned PartVTBitSize = PartVT.getSizeInBits().getKnownMinValue();
26416 if (PartVTBitSize % ValueVTBitSize == 0) {
26417 assert(PartVTBitSize >= ValueVTBitSize);
26418 EVT SameEltTypeVT = ValueVT;
26419 // If the element types are different, convert it to the same element type
26420 // of PartVT.
26421 // Give an example here, we want copy a <vscale x 1 x i8> value from
26422 // <vscale x 4 x i16>.
26423 // We need to convert <vscale x 4 x i16> to <vscale x 8 x i8> first,
26424 // then we can extract <vscale x 1 x i8>.
26425 if (ValueEltVT != PartEltVT) {
26426 unsigned Count = PartVTBitSize / ValueEltVT.getFixedSizeInBits();
26427 assert(Count != 0 && "The number of element should not be zero.");
26428 SameEltTypeVT =
26429 EVT::getVectorVT(Context, VT: ValueEltVT, NumElements: Count, /*IsScalable=*/true);
26430 Val = DAG.getNode(Opcode: ISD::BITCAST, DL, VT: SameEltTypeVT, Operand: Val);
26431 }
26432 if (ValueVT.isFixedLengthVector())
26433 Val = convertFromScalableVector(VT: ValueVT, V: Val, DAG, Subtarget);
26434 else
26435 Val = DAG.getExtractSubvector(DL, VT: ValueVT, Vec: Val, Idx: 0);
26436 return Val;
26437 }
26438 }
26439 return SDValue();
26440}
26441
26442bool RISCVTargetLowering::isIntDivCheap(EVT VT, AttributeList Attr) const {
26443 // When aggressively optimizing for code size, we prefer to use a div
26444 // instruction, as it is usually smaller than the alternative sequence.
26445 // TODO: Add vector division?
26446 bool OptSize = Attr.hasFnAttr(Kind: Attribute::MinSize);
26447 return OptSize && !VT.isVector() &&
26448 VT.getSizeInBits() <= getMaxDivRemBitWidthSupported();
26449}
26450
26451void RISCVTargetLowering::finalizeLowering(MachineFunction &MF) const {
26452 MF.getFrameInfo().computeMaxCallFrameSize(MF);
26453 TargetLoweringBase::finalizeLowering(MF);
26454}
26455
26456bool RISCVTargetLowering::preferScalarizeSplat(SDNode *N) const {
26457 // Scalarize zero_ext and sign_ext might stop match to widening instruction in
26458 // some situation.
26459 unsigned Opc = N->getOpcode();
26460 if (Opc == ISD::ZERO_EXTEND || Opc == ISD::SIGN_EXTEND)
26461 return false;
26462 return true;
26463}
26464
26465static Value *useTpOffset(IRBuilderBase &IRB, unsigned Offset) {
26466 Module *M = IRB.GetInsertBlock()->getModule();
26467 Function *ThreadPointerFunc = Intrinsic::getOrInsertDeclaration(
26468 M, id: Intrinsic::thread_pointer, Tys: IRB.getPtrTy());
26469 return IRB.CreateConstGEP1_32(Ty: IRB.getInt8Ty(),
26470 Ptr: IRB.CreateCall(Callee: ThreadPointerFunc), Idx0: Offset);
26471}
26472
26473Value *RISCVTargetLowering::getIRStackGuard(
26474 IRBuilderBase &IRB, const LibcallLoweringInfo &Libcalls) const {
26475 // Fuchsia provides a fixed TLS slot for the stack cookie.
26476 // <zircon/tls.h> defines ZX_TLS_STACK_GUARD_OFFSET with this value.
26477 if (Subtarget.isTargetFuchsia())
26478 return useTpOffset(IRB, Offset: -0x10);
26479
26480 // Android provides a fixed TLS slot for the stack cookie. See the definition
26481 // of TLS_SLOT_STACK_GUARD in
26482 // https://android.googlesource.com/platform/bionic/+/main/libc/platform/bionic/tls_defines.h
26483 if (Subtarget.isTargetAndroid())
26484 return useTpOffset(IRB, Offset: -0x18);
26485
26486 Module *M = IRB.GetInsertBlock()->getModule();
26487
26488 if (M->getStackProtectorGuard() == "tls") {
26489 // Users must specify the offset explicitly
26490 int Offset = M->getStackProtectorGuardOffset();
26491 return useTpOffset(IRB, Offset);
26492 }
26493
26494 return TargetLowering::getIRStackGuard(IRB, Libcalls);
26495}
26496
26497bool RISCVTargetLowering::isLegalStridedLoadStore(EVT DataType,
26498 Align Alignment) const {
26499 if (!Subtarget.hasVInstructions())
26500 return false;
26501
26502 // Only support fixed vectors if we know the minimum vector size.
26503 if (DataType.isFixedLengthVector() && !Subtarget.useRVVForFixedLengthVectors())
26504 return false;
26505
26506 EVT ScalarType = DataType.getScalarType();
26507 if (!isLegalElementTypeForRVV(ScalarTy: ScalarType))
26508 return false;
26509
26510 if (!Subtarget.enableUnalignedVectorMem() &&
26511 Alignment < ScalarType.getStoreSize())
26512 return false;
26513
26514 return true;
26515}
26516
26517bool RISCVTargetLowering::isLegalFirstFaultLoad(EVT DataType,
26518 Align Alignment) const {
26519 if (!Subtarget.hasVInstructions())
26520 return false;
26521
26522 EVT ScalarType = DataType.getScalarType();
26523 if (!isLegalElementTypeForRVV(ScalarTy: ScalarType))
26524 return false;
26525
26526 if (!Subtarget.enableUnalignedVectorMem() &&
26527 Alignment < ScalarType.getStoreSize())
26528 return false;
26529
26530 return true;
26531}
26532
26533MachineInstr *
26534RISCVTargetLowering::EmitKCFICheck(MachineBasicBlock &MBB,
26535 MachineBasicBlock::instr_iterator &MBBI,
26536 const TargetInstrInfo *TII) const {
26537 assert(MBBI->isCall() && MBBI->getCFIType() &&
26538 "Invalid call instruction for a KCFI check");
26539 assert(is_contained({RISCV::PseudoCALLIndirect, RISCV::PseudoTAILIndirect},
26540 MBBI->getOpcode()));
26541
26542 MachineOperand &Target = MBBI->getOperand(i: 0);
26543 Target.setIsRenamable(false);
26544
26545 return BuildMI(BB&: MBB, I: MBBI, MIMD: MBBI->getDebugLoc(), MCID: TII->get(Opcode: RISCV::KCFI_CHECK))
26546 .addReg(RegNo: Target.getReg())
26547 .addImm(Val: MBBI->getCFIType())
26548 .getInstr();
26549}
26550
26551#define GET_REGISTER_MATCHER
26552#include "RISCVGenAsmMatcher.inc"
26553
26554Register
26555RISCVTargetLowering::getRegisterByName(const char *RegName, LLT VT,
26556 const MachineFunction &MF) const {
26557 Register Reg = MatchRegisterAltName(Name: RegName);
26558 if (!Reg)
26559 Reg = MatchRegisterName(Name: RegName);
26560 if (!Reg)
26561 return Reg;
26562
26563 BitVector ReservedRegs = Subtarget.getRegisterInfo()->getReservedRegs(MF);
26564 if (!ReservedRegs.test(Idx: Reg) && !Subtarget.isRegisterReservedByUser(i: Reg))
26565 reportFatalUsageError(reason: Twine("Trying to obtain non-reserved register \"" +
26566 StringRef(RegName) + "\"."));
26567 return Reg;
26568}
26569
26570MachineMemOperand::Flags
26571RISCVTargetLowering::getTargetMMOFlags(const Instruction &I) const {
26572 const MDNode *NontemporalInfo = I.getMetadata(KindID: LLVMContext::MD_nontemporal);
26573
26574 if (NontemporalInfo == nullptr)
26575 return MachineMemOperand::MONone;
26576
26577 // 1 for default value work as __RISCV_NTLH_ALL
26578 // 2 -> __RISCV_NTLH_INNERMOST_PRIVATE
26579 // 3 -> __RISCV_NTLH_ALL_PRIVATE
26580 // 4 -> __RISCV_NTLH_INNERMOST_SHARED
26581 // 5 -> __RISCV_NTLH_ALL
26582 int NontemporalLevel = 5;
26583 const MDNode *RISCVNontemporalInfo =
26584 I.getMetadata(Kind: "riscv-nontemporal-domain");
26585 if (RISCVNontemporalInfo != nullptr)
26586 NontemporalLevel =
26587 cast<ConstantInt>(
26588 Val: cast<ConstantAsMetadata>(Val: RISCVNontemporalInfo->getOperand(I: 0))
26589 ->getValue())
26590 ->getZExtValue();
26591
26592 assert((1 <= NontemporalLevel && NontemporalLevel <= 5) &&
26593 "RISC-V target doesn't support this non-temporal domain.");
26594
26595 NontemporalLevel -= 2;
26596 MachineMemOperand::Flags Flags = MachineMemOperand::MONone;
26597 if (NontemporalLevel & 0b1)
26598 Flags |= MONontemporalBit0;
26599 if (NontemporalLevel & 0b10)
26600 Flags |= MONontemporalBit1;
26601
26602 return Flags;
26603}
26604
26605MachineMemOperand::Flags
26606RISCVTargetLowering::getTargetMMOFlags(const MemSDNode &Node) const {
26607
26608 MachineMemOperand::Flags NodeFlags = Node.getMemOperand()->getFlags();
26609 MachineMemOperand::Flags TargetFlags = MachineMemOperand::MONone;
26610 TargetFlags |= (NodeFlags & MONontemporalBit0);
26611 TargetFlags |= (NodeFlags & MONontemporalBit1);
26612 return TargetFlags;
26613}
26614
26615bool RISCVTargetLowering::areTwoSDNodeTargetMMOFlagsMergeable(
26616 const MemSDNode &NodeX, const MemSDNode &NodeY) const {
26617 return getTargetMMOFlags(Node: NodeX) == getTargetMMOFlags(Node: NodeY);
26618}
26619
26620bool RISCVTargetLowering::isCtpopFast(EVT VT) const {
26621 if (VT.isVector()) {
26622 EVT SVT = VT.getVectorElementType();
26623 // If the element type is legal we can use cpop.v if it is enabled.
26624 if (isLegalElementTypeForRVV(ScalarTy: SVT))
26625 return Subtarget.hasStdExtZvbb();
26626 // Don't consider it fast if the type needs to be legalized or scalarized.
26627 return false;
26628 }
26629
26630 return Subtarget.hasCPOPLike() && (VT == MVT::i32 || VT == MVT::i64);
26631}
26632
26633unsigned RISCVTargetLowering::getCustomCtpopCost(EVT VT,
26634 ISD::CondCode Cond) const {
26635 return isCtpopFast(VT) ? 0 : 1;
26636}
26637
26638bool RISCVTargetLowering::shouldInsertFencesForAtomic(
26639 const Instruction *I) const {
26640 if (Subtarget.hasStdExtZalasr()) {
26641 if (Subtarget.hasStdExtZtso()) {
26642 // Zalasr + TSO means that atomic_load_acquire and atomic_store_release
26643 // should be lowered to plain load/store. The easiest way to do this is
26644 // to say we should insert fences for them, and the fence insertion code
26645 // will just not insert any fences
26646 auto *LI = dyn_cast<LoadInst>(Val: I);
26647 auto *SI = dyn_cast<StoreInst>(Val: I);
26648 if ((LI &&
26649 (LI->getOrdering() == AtomicOrdering::SequentiallyConsistent)) ||
26650 (SI &&
26651 (SI->getOrdering() == AtomicOrdering::SequentiallyConsistent))) {
26652 // Here, this is a load or store which is seq_cst, and needs a .aq or
26653 // .rl therefore we shouldn't try to insert fences
26654 return false;
26655 }
26656 // Here, we are a TSO inst that isn't a seq_cst load/store
26657 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
26658 }
26659 return false;
26660 }
26661 // Note that one specific case requires fence insertion for an
26662 // AtomicCmpXchgInst but is handled via the RISCVZacasABIFix pass rather
26663 // than this hook due to limitations in the interface here.
26664 return isa<LoadInst>(Val: I) || isa<StoreInst>(Val: I);
26665}
26666
26667bool RISCVTargetLowering::fallBackToDAGISel(const Instruction &Inst) const {
26668
26669 // GISel support is in progress or complete for these opcodes.
26670 unsigned Op = Inst.getOpcode();
26671 if (Op == Instruction::Add || Op == Instruction::Sub ||
26672 Op == Instruction::And || Op == Instruction::Or ||
26673 Op == Instruction::Xor || Op == Instruction::InsertElement ||
26674 Op == Instruction::ShuffleVector || Op == Instruction::Load ||
26675 Op == Instruction::Freeze || Op == Instruction::Store)
26676 return false;
26677
26678 if (auto *II = dyn_cast<IntrinsicInst>(Val: &Inst)) {
26679 // Mark RVV intrinsic as supported.
26680 if (RISCVVIntrinsicsTable::getRISCVVIntrinsicInfo(IntrinsicID: II->getIntrinsicID())) {
26681 // GISel doesn't support tuple types yet. It also doesn't suport returning
26682 // a struct containing a scalable vector like vleff.
26683 if (Inst.getType()->isRISCVVectorTupleTy() ||
26684 Inst.getType()->isStructTy())
26685 return true;
26686
26687 for (unsigned i = 0; i < II->arg_size(); ++i)
26688 if (II->getArgOperand(i)->getType()->isRISCVVectorTupleTy())
26689 return true;
26690
26691 return false;
26692 }
26693 if (II->getIntrinsicID() == Intrinsic::vector_extract)
26694 return false;
26695 }
26696
26697 if (Inst.getType()->isScalableTy())
26698 return true;
26699
26700 for (unsigned i = 0; i < Inst.getNumOperands(); ++i)
26701 if (Inst.getOperand(i)->getType()->isScalableTy() &&
26702 !isa<ReturnInst>(Val: &Inst))
26703 return true;
26704
26705 return false;
26706}
26707
26708SDValue
26709RISCVTargetLowering::BuildSDIVPow2(SDNode *N, const APInt &Divisor,
26710 SelectionDAG &DAG,
26711 SmallVectorImpl<SDNode *> &Created) const {
26712 AttributeList Attr = DAG.getMachineFunction().getFunction().getAttributes();
26713 if (isIntDivCheap(VT: N->getValueType(ResNo: 0), Attr))
26714 return SDValue(N, 0); // Lower SDIV as SDIV
26715
26716 // Only perform this transform if short forward branch opt is supported.
26717 if (!Subtarget.hasShortForwardBranchIALU())
26718 return SDValue();
26719 EVT VT = N->getValueType(ResNo: 0);
26720 if (!(VT == MVT::i32 || (VT == MVT::i64 && Subtarget.is64Bit())))
26721 return SDValue();
26722
26723 // Ensure 2**k-1 < 2048 so that we can just emit a single addi/addiw.
26724 if (Divisor.sgt(RHS: 2048) || Divisor.slt(RHS: -2048))
26725 return SDValue();
26726 return TargetLowering::buildSDIVPow2WithCMov(N, Divisor, DAG, Created);
26727}
26728
26729bool RISCVTargetLowering::shouldFoldSelectWithSingleBitTest(
26730 EVT VT, const APInt &AndMask) const {
26731 if (Subtarget.hasCZEROLike() || Subtarget.hasVendorXTHeadCondMov())
26732 return !Subtarget.hasBEXTILike() && AndMask.ugt(RHS: 1024);
26733 return TargetLowering::shouldFoldSelectWithSingleBitTest(VT, AndMask);
26734}
26735
26736unsigned RISCVTargetLowering::getMinimumJumpTableEntries() const {
26737 return Subtarget.getMinimumJumpTableEntries();
26738}
26739
26740SDValue RISCVTargetLowering::expandIndirectJTBranch(const SDLoc &dl,
26741 SDValue Value, SDValue Addr,
26742 int JTI,
26743 SelectionDAG &DAG) const {
26744 if (Subtarget.hasStdExtZicfilp()) {
26745 // When Zicfilp enabled, we need to use software guarded branch for jump
26746 // table branch.
26747 SDValue Chain = Value;
26748 // Jump table debug info is only needed if CodeView is enabled.
26749 if (DAG.getTarget().getTargetTriple().isOSBinFormatCOFF())
26750 Chain = DAG.getJumpTableDebugInfo(JTI, Chain, DL: dl);
26751 return DAG.getNode(Opcode: RISCVISD::SW_GUARDED_BRIND, DL: dl, VT: MVT::Other, N1: Chain, N2: Addr);
26752 }
26753 return TargetLowering::expandIndirectJTBranch(dl, Value, Addr, JTI, DAG);
26754}
26755
26756// If an output pattern produces multiple instructions tablegen may pick an
26757// arbitrary type from an instructions destination register class to use for the
26758// VT of that MachineSDNode. This VT may be used to look up the representative
26759// register class. If the type isn't legal, the default implementation will
26760// not find a register class.
26761//
26762// Some integer types smaller than XLen are listed in the GPR register class to
26763// support isel patterns for GISel, but are not legal in SelectionDAG. The
26764// arbitrary type tablegen picks may be one of these smaller types.
26765//
26766// f16 and bf16 are both valid for the FPR16 or GPRF16 register class. It's
26767// possible for tablegen to pick bf16 as the arbitrary type for an f16 pattern.
26768std::pair<const TargetRegisterClass *, uint8_t>
26769RISCVTargetLowering::findRepresentativeClass(const TargetRegisterInfo *TRI,
26770 MVT VT) const {
26771 switch (VT.SimpleTy) {
26772 default:
26773 break;
26774 case MVT::i8:
26775 case MVT::i16:
26776 case MVT::i32:
26777 return TargetLowering::findRepresentativeClass(TRI, VT: Subtarget.getXLenVT());
26778 case MVT::bf16:
26779 case MVT::f16:
26780 return TargetLowering::findRepresentativeClass(TRI, VT: MVT::f32);
26781 }
26782
26783 return TargetLowering::findRepresentativeClass(TRI, VT);
26784}
26785
26786namespace llvm::RISCVVIntrinsicsTable {
26787
26788#define GET_RISCVVIntrinsicsTable_IMPL
26789#include "RISCVGenSearchableTables.inc"
26790
26791} // namespace llvm::RISCVVIntrinsicsTable
26792
26793bool RISCVTargetLowering::hasInlineStackProbe(const MachineFunction &MF) const {
26794
26795 // If the function specifically requests inline stack probes, emit them.
26796 if (MF.getFunction().hasFnAttribute(Kind: "probe-stack"))
26797 return MF.getFunction().getFnAttribute(Kind: "probe-stack").getValueAsString() ==
26798 "inline-asm";
26799
26800 return false;
26801}
26802
26803unsigned RISCVTargetLowering::getStackProbeSize(const MachineFunction &MF,
26804 Align StackAlign) const {
26805 // The default stack probe size is 4096 if the function has no
26806 // stack-probe-size attribute.
26807 const Function &Fn = MF.getFunction();
26808 unsigned StackProbeSize =
26809 Fn.getFnAttributeAsParsedInteger(Kind: "stack-probe-size", Default: 4096);
26810 // Round down to the stack alignment.
26811 StackProbeSize = alignDown(Value: StackProbeSize, Align: StackAlign.value());
26812 return StackProbeSize ? StackProbeSize : StackAlign.value();
26813}
26814
26815SDValue RISCVTargetLowering::lowerDYNAMIC_STACKALLOC(SDValue Op,
26816 SelectionDAG &DAG) const {
26817 MachineFunction &MF = DAG.getMachineFunction();
26818 if (!hasInlineStackProbe(MF))
26819 return SDValue();
26820
26821 MVT XLenVT = Subtarget.getXLenVT();
26822 // Get the inputs.
26823 SDValue Chain = Op.getOperand(i: 0);
26824 SDValue Size = Op.getOperand(i: 1);
26825
26826 MaybeAlign Align =
26827 cast<ConstantSDNode>(Val: Op.getOperand(i: 2))->getMaybeAlignValue();
26828 SDLoc dl(Op);
26829 EVT VT = Op.getValueType();
26830
26831 // Construct the new SP value in a GPR.
26832 SDValue SP = DAG.getCopyFromReg(Chain, dl, Reg: RISCV::X2, VT: XLenVT);
26833 Chain = SP.getValue(R: 1);
26834 SP = DAG.getNode(Opcode: ISD::SUB, DL: dl, VT: XLenVT, N1: SP, N2: Size);
26835 if (Align)
26836 SP = DAG.getNode(Opcode: ISD::AND, DL: dl, VT, N1: SP.getValue(R: 0),
26837 N2: DAG.getSignedConstant(Val: -Align->value(), DL: dl, VT));
26838
26839 // Set the real SP to the new value with a probing loop.
26840 Chain = DAG.getNode(Opcode: RISCVISD::PROBED_ALLOCA, DL: dl, VT: MVT::Other, N1: Chain, N2: SP);
26841 return DAG.getMergeValues(Ops: {SP, Chain}, dl);
26842}
26843
26844MachineBasicBlock *
26845RISCVTargetLowering::emitDynamicProbedAlloc(MachineInstr &MI,
26846 MachineBasicBlock *MBB) const {
26847 MachineFunction &MF = *MBB->getParent();
26848 MachineBasicBlock::iterator MBBI = MI.getIterator();
26849 DebugLoc DL = MBB->findDebugLoc(MBBI);
26850 Register TargetReg = MI.getOperand(i: 0).getReg();
26851
26852 const RISCVInstrInfo *TII = Subtarget.getInstrInfo();
26853 bool IsRV64 = Subtarget.is64Bit();
26854 Align StackAlign = Subtarget.getFrameLowering()->getStackAlign();
26855 const RISCVTargetLowering *TLI = Subtarget.getTargetLowering();
26856 uint64_t ProbeSize = TLI->getStackProbeSize(MF, StackAlign);
26857
26858 MachineFunction::iterator MBBInsertPoint = std::next(x: MBB->getIterator());
26859 MachineBasicBlock *LoopTestMBB =
26860 MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
26861 MF.insert(MBBI: MBBInsertPoint, MBB: LoopTestMBB);
26862 MachineBasicBlock *ExitMBB = MF.CreateMachineBasicBlock(BB: MBB->getBasicBlock());
26863 MF.insert(MBBI: MBBInsertPoint, MBB: ExitMBB);
26864 Register SPReg = RISCV::X2;
26865 Register ScratchReg =
26866 MF.getRegInfo().createVirtualRegister(RegClass: &RISCV::GPRRegClass);
26867
26868 // ScratchReg = ProbeSize
26869 TII->movImm(MBB&: *MBB, MBBI, DL, DstReg: ScratchReg, Val: ProbeSize, Flag: MachineInstr::NoFlags);
26870
26871 // LoopTest:
26872 // SUB SP, SP, ProbeSize
26873 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::SUB), DestReg: SPReg)
26874 .addReg(RegNo: SPReg)
26875 .addReg(RegNo: ScratchReg);
26876
26877 // s[d|w] zero, 0(sp)
26878 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL,
26879 MCID: TII->get(Opcode: IsRV64 ? RISCV::SD : RISCV::SW))
26880 .addReg(RegNo: RISCV::X0)
26881 .addReg(RegNo: SPReg)
26882 .addImm(Val: 0);
26883
26884 // BLT TargetReg, SP, LoopTest
26885 BuildMI(BB&: *LoopTestMBB, I: LoopTestMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::BLT))
26886 .addReg(RegNo: TargetReg)
26887 .addReg(RegNo: SPReg)
26888 .addMBB(MBB: LoopTestMBB);
26889
26890 // Adjust with: MV SP, TargetReg.
26891 BuildMI(BB&: *ExitMBB, I: ExitMBB->end(), MIMD: DL, MCID: TII->get(Opcode: RISCV::ADDI), DestReg: SPReg)
26892 .addReg(RegNo: TargetReg)
26893 .addImm(Val: 0);
26894
26895 ExitMBB->splice(Where: ExitMBB->end(), Other: MBB, From: std::next(x: MBBI), To: MBB->end());
26896 ExitMBB->transferSuccessorsAndUpdatePHIs(FromMBB: MBB);
26897
26898 LoopTestMBB->addSuccessor(Succ: ExitMBB);
26899 LoopTestMBB->addSuccessor(Succ: LoopTestMBB);
26900 MBB->addSuccessor(Succ: LoopTestMBB);
26901
26902 MI.eraseFromParent();
26903 MF.getInfo<RISCVMachineFunctionInfo>()->setDynamicAllocation();
26904 return ExitMBB->begin()->getParent();
26905}
26906
26907ArrayRef<MCPhysReg> RISCVTargetLowering::getRoundingControlRegisters() const {
26908 if (Subtarget.hasStdExtFOrZfinx()) {
26909 static const MCPhysReg RCRegs[] = {RISCV::FRM, RISCV::FFLAGS};
26910 return RCRegs;
26911 }
26912 return {};
26913}
26914
26915bool RISCVTargetLowering::shouldFoldMaskToVariableShiftPair(SDValue Y) const {
26916 EVT VT = Y.getValueType();
26917
26918 if (VT.isVector())
26919 return false;
26920
26921 return VT.getSizeInBits() <= Subtarget.getXLen();
26922}
26923
26924bool RISCVTargetLowering::isReassocProfitable(SelectionDAG &DAG, SDValue N0,
26925 SDValue N1) const {
26926 if (!N0.hasOneUse())
26927 return false;
26928
26929 // Avoid reassociating expressions that can be lowered to vector
26930 // multiply accumulate (i.e. add (mul x, y), z)
26931 if (N0.getOpcode() == ISD::ADD && N1.getOpcode() == ISD::MUL &&
26932 (N0.getValueType().isVector() && Subtarget.hasVInstructions()))
26933 return false;
26934
26935 return true;
26936}
26937